From 0f8fc11c539769c266c36d3b61e070f8dab04471 Mon Sep 17 00:00:00 2001 From: rossirpaulo Date: Tue, 2 Jun 2026 11:24:52 -0500 Subject: [PATCH 1/7] Overhaul BAML language release pipeline --- .../workflows/build-python-sdk.reusable.yaml | 17 +- .github/workflows/ci.yaml | 4 +- ...lease-sdk.yaml => publish-python-pypi.yml} | 48 +- .../workflows/release-baml-language-alpha.yml | 629 ------- .github/workflows/release-baml-language.yml | 625 +++++++ .github/workflows/release-cli.yaml | 9 - .../workflows/release-pkg-boundaryml-com.yml | 36 - TASK/AMENDMENTS.md | 110 ++ TASK/COMMANDS.md | 54 + TASK/FAILURE-MODES.md | 58 + TASK/OPEN-QUESTIONS.md | 962 ++++++++++ TASK/PRIOR-ART.md | 552 ++++++ TASK/TASK.md | 1638 +++++++++++++++++ TASK/USER-PATHS.md | 417 +++++ TASK/docs/install.md | 85 + TASK/docs/release-maintainer.md | 66 + TASK/docs/toolchain-system.md | 68 + baml_language/Cargo.lock | 40 + baml_language/Cargo.toml | 2 + baml_language/crates/baml/Cargo.toml | 29 + baml_language/crates/baml/src/main.rs | 662 +++++++ baml_language/crates/baml_cli/Cargo.toml | 2 + baml_language/crates/baml_cli/src/commands.rs | 14 +- .../crates/baml_cli/src/ide_command.rs | 128 ++ baml_language/crates/baml_cli/src/lib.rs | 1 + baml_language/crates/baml_cli/src/main.rs | 20 + .../crates/baml_cli/src/pack_command.rs | 316 +--- .../crates/baml_lsp_server/Cargo.toml | 1 + .../crates/baml_lsp_server/src/lib.rs | 2 +- .../baml_lsp_server/src/playground_server.rs | 17 + .../baml_lsp_server/src/playground_ws.rs | 10 + baml_language/crates/baml_release/Cargo.toml | 31 + baml_language/crates/baml_release/src/lib.rs | 825 +++++++++ .../crates/baml_release/src/manifest.rs | 119 ++ baml_language/crates/baml_version/Cargo.toml | 20 + baml_language/crates/baml_version/src/lib.rs | 6 + baml_language/crates/bex_project/Cargo.toml | 1 + .../crates/bex_project/src/bex_lsp/mod.rs | 1 + .../src/bex_lsp/multi_project/request.rs | 14 +- .../bex_project/src/bex_lsp/protocol.rs | 10 + baml_language/crates/bridge_cffi/Cargo.toml | 1 + .../crates/bridge_cffi/src/ffi/runtime.rs | 4 +- baml_language/crates/bridge_wasm/Cargo.toml | 1 + baml_language/crates/bridge_wasm/src/lib.rs | 2 +- baml_language/release.toml | 5 +- .../sdks/nodejs/bridge_nodejs/Cargo.toml | 1 + .../sdks/nodejs/bridge_nodejs/package.json | 2 +- .../sdks/nodejs/bridge_nodejs/src/lib.rs | 2 +- baml_language/sdks/python/pyproject.toml | 2 +- .../sdks/python/rust/bridge_python/Cargo.toml | 1 + .../sdks/python/rust/bridge_python/src/lib.rs | 2 +- .../sdks/python/src/baml_core/__init__.py | 2 + baml_language/sdks/python/uv.lock | 2 +- baml_language/stow.toml | 8 +- packaging/aur/baml-bin/PKGBUILD | 9 +- packaging/aur/baml/PKGBUILD | 15 +- scripts/baml-language-version | 373 +++- scripts/baml-package-manager-artifacts | 178 ++ scripts/baml-release-manifests | 108 ++ scripts/baml-wrapper-version | 111 ++ scripts/install.ps1 | 107 ++ scripts/install.sh | 154 ++ tools/pkg_boundaryml_com/data/index.html | 4 +- typescript2/README.md | 24 +- typescript2/app-vscode-ext/package.json | 6 +- typescript2/app-vscode-ext/src/compat.ts | 22 + typescript2/app-vscode-ext/src/extension.ts | 216 ++- typescript2/package.json | 6 +- .../src/ports/WebSocketRuntimePort.ts | 15 + typescript2/pkg-playground/src/protocol.ts | 8 + 70 files changed, 7861 insertions(+), 1179 deletions(-) rename .github/workflows/{release-sdk.yaml => publish-python-pypi.yml} (55%) delete mode 100644 .github/workflows/release-baml-language-alpha.yml create mode 100644 .github/workflows/release-baml-language.yml delete mode 100644 .github/workflows/release-cli.yaml delete mode 100644 .github/workflows/release-pkg-boundaryml-com.yml create mode 100644 TASK/AMENDMENTS.md create mode 100644 TASK/COMMANDS.md create mode 100644 TASK/FAILURE-MODES.md create mode 100644 TASK/OPEN-QUESTIONS.md create mode 100644 TASK/PRIOR-ART.md create mode 100644 TASK/TASK.md create mode 100644 TASK/USER-PATHS.md create mode 100644 TASK/docs/install.md create mode 100644 TASK/docs/release-maintainer.md create mode 100644 TASK/docs/toolchain-system.md create mode 100644 baml_language/crates/baml/Cargo.toml create mode 100644 baml_language/crates/baml/src/main.rs create mode 100644 baml_language/crates/baml_cli/src/ide_command.rs create mode 100644 baml_language/crates/baml_release/Cargo.toml create mode 100644 baml_language/crates/baml_release/src/lib.rs create mode 100644 baml_language/crates/baml_release/src/manifest.rs create mode 100644 baml_language/crates/baml_version/Cargo.toml create mode 100644 baml_language/crates/baml_version/src/lib.rs create mode 100644 baml_language/crates/bex_project/src/bex_lsp/protocol.rs create mode 100755 scripts/baml-package-manager-artifacts create mode 100755 scripts/baml-release-manifests create mode 100755 scripts/baml-wrapper-version create mode 100644 scripts/install.ps1 create mode 100755 scripts/install.sh create mode 100644 typescript2/app-vscode-ext/src/compat.ts create mode 100644 typescript2/pkg-playground/src/protocol.ts diff --git a/.github/workflows/build-python-sdk.reusable.yaml b/.github/workflows/build-python-sdk.reusable.yaml index 8f2304165aa..45a281ce69c 100644 --- a/.github/workflows/build-python-sdk.reusable.yaml +++ b/.github/workflows/build-python-sdk.reusable.yaml @@ -1,7 +1,13 @@ name: BAML SDK Release - Build Python on: - workflow_call: {} + workflow_call: + inputs: + release_plan_json: + description: "Frozen BAML language release-plan.json content" + type: string + default: "" + required: false push: branches: - sam/publish-python @@ -69,6 +75,15 @@ jobs: - uses: actions/checkout@v6 + - name: Stamp release plan + if: ${{ inputs.release_plan_json != '' }} + env: + RELEASE_PLAN_JSON: ${{ inputs.release_plan_json }} + run: | + set -euo pipefail + printf '%s\n' "$RELEASE_PLAN_JSON" > release-plan.json + scripts/baml-language-version stamp --plan release-plan.json + # NOTE: we deliberately do NOT use ./.github/actions/setup-mise here. # The host Python version pins the abi3 floor of the resulting wheel, # so we want a specific Python (3.10, matching the abi3-py310 target) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ad4502cc202..370c1044dd6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -74,8 +74,10 @@ jobs: if git diff --quiet "${MERGE_BASE}...HEAD" -- \ ':baml_language/**' \ ':scripts/baml-language-version' \ + ':scripts/install.sh' \ + ':scripts/install.ps1' \ ':.github/workflows/ci.yaml' \ - ':.github/workflows/release-baml-language-alpha.yml' \ + ':.github/workflows/release-baml-language.yml' \ ':.github/workflows/cargo-tests.reusable.yaml' \ ':.github/workflows/size-gate.reusable.yaml' \ ':.github/workflows/wasm-pack-tests.reusable.yaml' \ diff --git a/.github/workflows/release-sdk.yaml b/.github/workflows/publish-python-pypi.yml similarity index 55% rename from .github/workflows/release-sdk.yaml rename to .github/workflows/publish-python-pypi.yml index 131ce73747f..98fdbadb4d3 100644 --- a/.github/workflows/release-sdk.yaml +++ b/.github/workflows/publish-python-pypi.yml @@ -1,24 +1,23 @@ -name: BAML SDK Release +name: Publish Python PyPI -# Release orchestrator for the BAML SDKs (currently: `baml_core` Python -# wheel via PyPI). Mirrors the shape of `release.yml`: determine-version -# → build → all-builds gate → publish-* jobs. -# -# TODO: tag-driven release. The convention isn't decided yet, so this -# workflow is `workflow_dispatch`-only for now. Once the tag scheme is -# locked in, add a `push: tags:` trigger and revive the tag-parsing -# branch in `determine-version`. -# -# TODO: merge the Homebrew formula dispatch from -# `release-baml-language-alpha.yml` into this workflow so a single -# release tag also bumps the tap. That means adding a `dispatch_homebrew` -# input, threading the homebrew payload through the build outputs, and -# adding a `dispatch-homebrew` job that fires `repository_dispatch` to -# `BoundaryML/homebrew-baml`. See the `Validate tap version before -# dispatch` and `Dispatch Homebrew update` steps in -# `release-baml-language-alpha.yml` for the existing implementation. +# Focused PyPI publisher for Python `baml_core`. The BAML language release +# graph owns version planning/stamping and calls this path after its build gate. +# Before production publishing with this renamed workflow, update or validate +# PyPI's trusted-publisher binding for project `baml-core`. on: + workflow_call: + inputs: + publish: + description: "Publish to PyPI after building" + type: boolean + default: false + required: false + release_plan_json: + description: "Frozen BAML language release-plan.json content" + type: string + default: "" + required: false workflow_dispatch: inputs: publish: @@ -28,18 +27,19 @@ on: required: false concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-release-sdk - cancel-in-progress: true + group: publish-python-pypi-${{ github.ref }} + cancel-in-progress: false permissions: contents: read id-token: write jobs: - # Version comes from `baml_language/sdks/python/pyproject.toml` - # as-is — bump the file there and merge before tagging a release run. + # Version comes from the stamped checkout prepared by the release graph. build-python-sdk: uses: ./.github/workflows/build-python-sdk.reusable.yaml + with: + release_plan_json: ${{ inputs.release_plan_json }} all-builds: name: Assert all builds passed @@ -66,8 +66,8 @@ jobs: - name: Verify wheel count run: | set -euo pipefail - ls -la dist/ - wheel_count=$(ls dist/*.whl 2>/dev/null | wc -l) + find dist -maxdepth 1 -type f -print + wheel_count=$(find dist -maxdepth 1 -type f -name '*.whl' | wc -l | tr -d ' ') if [ "$wheel_count" -lt 8 ]; then echo "::error::Expected at least 8 wheels, found $wheel_count" exit 1 diff --git a/.github/workflows/release-baml-language-alpha.yml b/.github/workflows/release-baml-language-alpha.yml deleted file mode 100644 index 973ec8ef2ce..00000000000 --- a/.github/workflows/release-baml-language-alpha.yml +++ /dev/null @@ -1,629 +0,0 @@ -name: BAML Language Alpha Release - -on: - workflow_run: - workflows: ["CI - BAML Language"] - types: [completed] - workflow_dispatch: - inputs: - ref: - description: "Branch, tag, or SHA to build" - required: false - alpha_number: - description: "Numeric alpha suffix override" - required: false - publish_github_release: - type: boolean - default: false - dispatch_homebrew: - type: boolean - default: false - -permissions: - contents: write - actions: read - -concurrency: - group: ${{ (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'push' && github.event.workflow_run.head_branch == 'canary') && 'baml-language-alpha-canary' || format('baml-language-alpha-{0}-{1}', github.event_name, github.run_id) }} - cancel-in-progress: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'push' && github.event.workflow_run.head_branch == 'canary' }} - -defaults: - run: - shell: bash - -env: - CARGO_INCREMENTAL: 0 - CARGO_NET_RETRY: 10 - CARGO_TERM_COLOR: always - RUSTUP_MAX_RETRIES: 10 - -jobs: - prepare: - name: Prepare Release - if: | - github.event_name == 'workflow_dispatch' || - (github.event.workflow_run.conclusion == 'success' && - github.event.workflow_run.event == 'push' && - github.event.workflow_run.head_branch == 'canary') - runs-on: blacksmith-4vcpu-ubuntu-2204 - outputs: - should_run: ${{ steps.gate.outputs.should_run }} - publish: ${{ steps.gate.outputs.publish }} - dispatch_homebrew: ${{ steps.gate.outputs.dispatch_homebrew }} - checkout_ref: ${{ steps.gate.outputs.checkout_ref }} - source_ref: ${{ steps.source.outputs.source_ref }} - source_sha: ${{ steps.source.outputs.source_sha }} - ci_run_url: ${{ steps.gate.outputs.ci_run_url }} - version: ${{ steps.version.outputs.version }} - release_tag: ${{ steps.version.outputs.release_tag }} - steps: - - name: Gate release mode - id: gate - env: - EVENT_NAME: ${{ github.event_name }} - WORKFLOW_CONCLUSION: ${{ github.event.workflow_run.conclusion }} - WORKFLOW_NAME: ${{ github.event.workflow_run.name }} - WORKFLOW_EVENT: ${{ github.event.workflow_run.event }} - WORKFLOW_HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} - WORKFLOW_HEAD_SHA: ${{ github.event.workflow_run.head_sha }} - WORKFLOW_RUN_NUMBER: ${{ github.event.workflow_run.run_number }} - WORKFLOW_RUN_URL: ${{ github.event.workflow_run.html_url }} - INPUT_REF: ${{ inputs.ref }} - INPUT_ALPHA_NUMBER: ${{ inputs.alpha_number }} - INPUT_PUBLISH: ${{ inputs.publish_github_release }} - INPUT_DISPATCH_HOMEBREW: ${{ inputs.dispatch_homebrew }} - run: | - set -euo pipefail - - should_run=false - publish=false - dispatch_homebrew=false - checkout_ref="${GITHUB_SHA}" - alpha_number="${GITHUB_RUN_NUMBER}" - ci_run_url="" - - if [[ "${EVENT_NAME}" == "workflow_run" ]]; then - if [[ "${WORKFLOW_CONCLUSION}" == "success" && - "${WORKFLOW_NAME}" == "CI - BAML Language" && - "${WORKFLOW_EVENT}" == "push" && - "${WORKFLOW_HEAD_BRANCH}" == "canary" ]]; then - should_run=true - publish=true - dispatch_homebrew=true - checkout_ref="${WORKFLOW_HEAD_SHA}" - alpha_number="${WORKFLOW_RUN_NUMBER}" - ci_run_url="${WORKFLOW_RUN_URL}" - fi - else - should_run=true - publish="${INPUT_PUBLISH:-false}" - dispatch_homebrew="${INPUT_DISPATCH_HOMEBREW:-false}" - checkout_ref="${INPUT_REF:-${GITHUB_SHA}}" - alpha_number="${INPUT_ALPHA_NUMBER:-${GITHUB_RUN_NUMBER}}" - fi - - if [[ "${dispatch_homebrew}" == "true" && "${publish}" != "true" ]]; then - echo "::error::dispatch_homebrew=true requires publish_github_release=true" - exit 1 - fi - - if [[ ! "${alpha_number}" =~ ^[0-9]+$ ]]; then - echo "::error::alpha_number must be numeric" - exit 1 - fi - - { - echo "should_run=${should_run}" - echo "publish=${publish}" - echo "dispatch_homebrew=${dispatch_homebrew}" - echo "checkout_ref=${checkout_ref}" - echo "alpha_number=${alpha_number}" - echo "ci_run_url=${ci_run_url}" - } >> "$GITHUB_OUTPUT" - - - name: Checkout - if: steps.gate.outputs.should_run == 'true' - uses: actions/checkout@v6 - with: - ref: ${{ steps.gate.outputs.checkout_ref }} - fetch-depth: 0 - persist-credentials: false - - - name: Resolve source - if: steps.gate.outputs.should_run == 'true' - id: source - env: - EVENT_NAME: ${{ github.event_name }} - WORKFLOW_HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} - INPUT_REF: ${{ inputs.ref }} - run: | - set -euo pipefail - source_sha="$(git rev-parse HEAD)" - if [[ "${EVENT_NAME}" == "workflow_run" ]]; then - source_ref="${WORKFLOW_HEAD_BRANCH}" - else - source_ref="${INPUT_REF:-${GITHUB_REF_NAME}}" - fi - { - echo "source_sha=${source_sha}" - echo "source_ref=${source_ref}" - } >> "$GITHUB_OUTPUT" - - - name: Compute release version - if: steps.gate.outputs.should_run == 'true' - id: version - env: - ALPHA_NUMBER: ${{ steps.gate.outputs.alpha_number }} - run: | - set -euo pipefail - scripts/baml-language-version check - version="$(scripts/baml-language-version compute --alpha-number "${ALPHA_NUMBER}")" - echo "version=${version}" >> "$GITHUB_OUTPUT" - echo "release_tag=baml-language-${version}" >> "$GITHUB_OUTPUT" - - build: - name: Build ${{ matrix.target }} - needs: prepare - if: needs.prepare.outputs.should_run == 'true' - strategy: - fail-fast: false - matrix: - include: - - target: aarch64-apple-darwin - os: macos-14 - binary: baml-cli - pack_host: baml-pack-host - archive_ext: tar.gz - - target: x86_64-apple-darwin - os: macos-15-intel - binary: baml-cli - pack_host: baml-pack-host - archive_ext: tar.gz - - target: x86_64-unknown-linux-gnu - os: blacksmith-4vcpu-ubuntu-2204 - binary: baml-cli - pack_host: baml-pack-host - archive_ext: tar.gz - - target: aarch64-unknown-linux-gnu - os: blacksmith-4vcpu-ubuntu-2204-arm - binary: baml-cli - pack_host: baml-pack-host - archive_ext: tar.gz - - target: x86_64-unknown-linux-musl - os: blacksmith-4vcpu-ubuntu-2204 - binary: baml-cli - pack_host: baml-pack-host - archive_ext: tar.gz - container: rust:alpine@sha256:606fd313a0f49743ee2a7bd49a0914bab7deedb12791f3a846a34a4711db7ed2 - - target: aarch64-unknown-linux-musl - os: blacksmith-4vcpu-ubuntu-2204 - binary: baml-cli - pack_host: baml-pack-host - archive_ext: tar.gz - cross: true - - target: x86_64-pc-windows-msvc - os: blacksmith-4vcpu-windows-2025 - binary: baml-cli.exe - pack_host: baml-pack-host.exe - archive_ext: zip - runs-on: ${{ matrix.os }} - container: ${{ matrix.container || '' }} - timeout-minutes: 45 - steps: - - name: Install Alpine build dependencies - if: matrix.container - shell: sh - run: apk add --no-cache bash git cmake clang openssl-dev openssl-libs-static perl make python3 - - - name: Checkout - uses: actions/checkout@v6 - with: - ref: ${{ needs.prepare.outputs.checkout_ref }} - persist-credentials: false - - - name: Setup Rust - if: ${{ !matrix.container }} - uses: ./.github/actions/setup-rust - with: - targets: ${{ matrix.target }} - workspace: baml_language - enable-cache: true - enable-cross: false - skip-protoc: true - - - name: Install cross - if: matrix.cross - run: cargo install cross - - - name: Build baml-cli - env: - BAML_RELEASE_VERSION: ${{ needs.prepare.outputs.version }} - TARGET: ${{ matrix.target }} - CARGO: ${{ matrix.cross && 'cross' || 'cargo' }} - run: $CARGO build --release --bin baml-cli --target "${TARGET}" - working-directory: baml_language - - - name: Build baml-pack-host - env: - BAML_RELEASE_VERSION: ${{ needs.prepare.outputs.version }} - TARGET: ${{ matrix.target }} - CARGO: ${{ matrix.cross && 'cross' || 'cargo' }} - run: $CARGO build --release -p baml_pack_host --bin baml-pack-host --target "${TARGET}" - working-directory: baml_language - - - name: Smoke test version - if: ${{ !matrix.cross && !matrix.cross_arch }} - env: - EXPECTED_VERSION: ${{ needs.prepare.outputs.version }} - TARGET: ${{ matrix.target }} - BINARY: ${{ matrix.binary }} - run: | - set -euo pipefail - binary="baml_language/target/${TARGET}/release/${BINARY}" - output="$("${binary}" --version)" - echo "${output}" - grep -F "${EXPECTED_VERSION}" <<< "${output}" - - - name: Package archive - env: - VERSION: ${{ needs.prepare.outputs.version }} - TARGET: ${{ matrix.target }} - BINARY: ${{ matrix.binary }} - PACK_HOST: ${{ matrix.pack_host }} - ARCHIVE_EXT: ${{ matrix.archive_ext }} - run: | - set -euo pipefail - python3 - <<'PY' - import hashlib - import os - import shutil - import tarfile - import zipfile - from pathlib import Path - - version = os.environ["VERSION"] - target = os.environ["TARGET"] - binary = os.environ["BINARY"] - pack_host = os.environ["PACK_HOST"] - archive_ext = os.environ["ARCHIVE_EXT"] - - root = Path.cwd() - staging = root / "dist" / target / "staging" - staging.mkdir(parents=True, exist_ok=True) - release_dir = root / "baml_language" / "target" / target / "release" - shutil.copy2(release_dir / binary, staging / binary) - shutil.copy2(release_dir / pack_host, staging / pack_host) - shutil.copy2(root / "README.md", staging / "README.md") - shutil.copy2(root / "LICENSE", staging / "LICENSE") - - archive = root / "dist" / target / f"baml-language-{version}-{target}.{archive_ext}" - contents = (binary, pack_host, "README.md", "LICENSE") - if archive_ext == "zip": - with zipfile.ZipFile(archive, "w", compression=zipfile.ZIP_DEFLATED) as zf: - for item in contents: - zf.write(staging / item, item) - else: - with tarfile.open(archive, "w:gz") as tf: - for item in contents: - tf.add(staging / item, item) - - digest = hashlib.sha256(archive.read_bytes()).hexdigest() - (archive.with_suffix(archive.suffix + ".sha256")).write_text(f"{digest} {archive.name}\n") - print(f"{digest} {archive.name}") - PY - - - name: Upload archive artifact - uses: actions/upload-artifact@v7 - with: - name: baml-language-archive-${{ matrix.target }} - path: | - dist/${{ matrix.target }}/* - !dist/${{ matrix.target }}/staging/** - if-no-files-found: error - - # ARM musl is cross-compiled from x86, so we can't smoke test it in the build job. - # GitHub Actions' JS-based actions (like actions/checkout) don't work in Alpine - # containers on ARM runners, so we can't use a container there either. - # Instead, we run a separate job on a native ARM runner to verify the binary works. - test-arm-musl: - name: Test ARM musl binary - needs: - - prepare - - build - if: needs.prepare.outputs.should_run == 'true' - runs-on: blacksmith-4vcpu-ubuntu-2204-arm - timeout-minutes: 5 - steps: - - name: Download ARM musl archive - uses: actions/download-artifact@v4 - with: - name: baml-language-archive-aarch64-unknown-linux-musl - path: dist - - - name: Extract and test - run: | - set -euo pipefail - tar -xzf dist/*.tar.gz - ./baml-cli --version - - publish: - name: Publish Release - needs: - - prepare - - build - - test-arm-musl - if: needs.prepare.outputs.should_run == 'true' && needs.prepare.outputs.publish == 'true' - runs-on: blacksmith-4vcpu-ubuntu-2204 - timeout-minutes: 15 - env: - GH_TOKEN: ${{ github.token }} - GH_REPO: ${{ github.repository }} - VERSION: ${{ needs.prepare.outputs.version }} - RELEASE_TAG: ${{ needs.prepare.outputs.release_tag }} - SOURCE_SHA: ${{ needs.prepare.outputs.source_sha }} - SOURCE_REF: ${{ needs.prepare.outputs.source_ref }} - CI_RUN_URL: ${{ needs.prepare.outputs.ci_run_url }} - DISPATCH_HOMEBREW: ${{ needs.prepare.outputs.dispatch_homebrew }} - steps: - - name: Download archives - uses: actions/download-artifact@v8 - with: - pattern: baml-language-archive-* - merge-multiple: true - path: release-assets - - - name: Prepare release notes and payload - env: - REPOSITORY: ${{ github.repository }} - RELEASE_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - run: | - set -euo pipefail - python3 - <<'PY' - import json - import os - from pathlib import Path - - version = os.environ["VERSION"] - tag = os.environ["RELEASE_TAG"] - repo = os.environ["REPOSITORY"] - source_sha = os.environ["SOURCE_SHA"] - source_ref = os.environ["SOURCE_REF"] - ci_run_url = os.environ["CI_RUN_URL"] - release_run_url = os.environ["RELEASE_RUN_URL"] - - assets_dir = Path("release-assets") - target_assets = {} - rows = [] - for checksum_file in sorted(assets_dir.glob("*.sha256")): - sha, filename = checksum_file.read_text().strip().split(maxsplit=1) - filename = filename.strip() - archive = assets_dir / filename - if not archive.exists(): - raise SystemExit(f"missing archive for checksum {filename}") - prefix = f"baml-language-{version}-" - suffix = ".zip" if filename.endswith(".zip") else ".tar.gz" - target = filename.removeprefix(prefix).removesuffix(suffix) - url = f"https://github.com/{repo}/releases/download/{tag}/{filename}" - target_assets[target] = {"url": url, "sha256": sha} - rows.append((target, filename, sha)) - - expected = { - "aarch64-apple-darwin", - "x86_64-apple-darwin", - "x86_64-unknown-linux-gnu", - "aarch64-unknown-linux-gnu", - "x86_64-unknown-linux-musl", - "aarch64-unknown-linux-musl", - "x86_64-pc-windows-msvc", - } - missing = expected - target_assets.keys() - if missing: - raise SystemExit(f"missing release assets for {sorted(missing)}") - - body = [ - f"Source commit: `{source_sha}`", - f"Source ref: `{source_ref}`", - ] - if ci_run_url: - body.append(f"Validated by CI run: {ci_run_url}") - body.extend([ - f"Release workflow run: {release_run_url}", - "", - "| Target | Asset | SHA256 |", - "| --- | --- | --- |", - ]) - for target, filename, sha in rows: - body.append(f"| `{target}` | `{filename}` | `{sha}` |") - Path("release-body.md").write_text("\n".join(body) + "\n") - - payload = { - "version": version, - "source_repo": repo, - "source_sha": source_sha, - "release_tag": tag, - "assets": target_assets, - } - Path("homebrew-payload.json").write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") - PY - - - name: Create or update GitHub release - run: | - set -euo pipefail - title="BAML Language ${VERSION}" - if gh release view "${RELEASE_TAG}" --repo "${GH_REPO}" >/dev/null 2>&1; then - gh release edit "${RELEASE_TAG}" \ - --repo "${GH_REPO}" \ - --title "${title}" \ - --notes-file release-body.md \ - --prerelease - else - gh release create "${RELEASE_TAG}" \ - --repo "${GH_REPO}" \ - --title "${title}" \ - --notes-file release-body.md \ - --prerelease \ - --latest=false \ - --target "${SOURCE_SHA}" - fi - gh release upload "${RELEASE_TAG}" release-assets/* --repo "${GH_REPO}" --clobber - - - name: Validate tap version before dispatch - if: env.DISPATCH_HOMEBREW == 'true' - env: - GH_TOKEN: ${{ secrets.HOMEBREW_BAML_DISPATCH_TOKEN }} - run: | - set -euo pipefail - if [[ -z "${GH_TOKEN}" ]]; then - echo "::error::HOMEBREW_BAML_DISPATCH_TOKEN is required when dispatch_homebrew=true" - exit 1 - fi - gh api repos/BoundaryML/homebrew-baml/contents/Formula/baml.rb -q .content | base64 --decode > current-baml.rb - python3 - <<'PY' - import json - import re - from pathlib import Path - - incoming = json.loads(Path("homebrew-payload.json").read_text())["version"] - content = Path("current-baml.rb").read_text() - - legacy = "GlooHQ/baml" in content or 'bin.install "baml"' in content - match = re.search(r'version\s+"([^"]+)"', content) - if not match or legacy: - print("tap formula is legacy or unversioned; allowing one-time migration") - raise SystemExit(0) - - semver = re.compile(r"^(\d+)\.(\d+)\.(\d+)(?:-alpha\.(\d+))?$") - def key(value): - parsed = semver.fullmatch(value) - if not parsed: - raise SystemExit(f"unsupported tap formula version: {value}") - major, minor, patch, alpha = parsed.groups() - stable_rank = 1 if alpha is None else 0 - alpha_number = -1 if alpha is None else int(alpha) - return (int(major), int(minor), int(patch), stable_rank, alpha_number) - - current = match.group(1) - if key(incoming) < key(current): - raise SystemExit(f"incoming Homebrew version {incoming} is older than current {current}") - print(f"tap version check passed: {current} -> {incoming}") - PY - - - name: Dispatch Homebrew update - if: env.DISPATCH_HOMEBREW == 'true' - env: - GH_TOKEN: ${{ secrets.HOMEBREW_BAML_DISPATCH_TOKEN }} - run: | - set -euo pipefail - python3 - <<'PY' - import json - from pathlib import Path - - payload = json.loads(Path("homebrew-payload.json").read_text()) - Path("repository-dispatch.json").write_text(json.dumps({ - "event_type": "update-baml", - "client_payload": payload, - }, indent=2, sort_keys=True) + "\n") - PY - gh api --method POST repos/BoundaryML/homebrew-baml/dispatches --input repository-dispatch.json - - # AUR pkgver can't contain hyphens, so we convert 0.11.0-alpha.602 to 0.11.0.alpha.602. - # When we exit alpha and use stable semver (e.g. 0.11.0), this conversion becomes a no-op. - publish-to-aur: - name: Publish to AUR - needs: - - prepare - - publish - if: needs.prepare.outputs.should_run == 'true' && needs.prepare.outputs.publish == 'true' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ needs.prepare.outputs.checkout_ref }} - - - name: Download release assets - env: - VERSION: ${{ needs.prepare.outputs.version }} - run: | - set -euo pipefail - mkdir -p artifacts - - curl -fSL -o artifacts/x86_64.tar.gz \ - "https://github.com/BoundaryML/baml/releases/download/baml-language-${VERSION}/baml-language-${VERSION}-x86_64-unknown-linux-gnu.tar.gz" - curl -fSL -o artifacts/aarch64.tar.gz \ - "https://github.com/BoundaryML/baml/releases/download/baml-language-${VERSION}/baml-language-${VERSION}-aarch64-unknown-linux-gnu.tar.gz" - curl -fSL -o artifacts/source.tar.gz \ - "https://github.com/BoundaryML/baml/archive/refs/tags/baml-language-${VERSION}.tar.gz" - - echo "SHA256_X86_64=$(sha256sum artifacts/x86_64.tar.gz | cut -d' ' -f1)" >> "$GITHUB_ENV" - echo "SHA256_AARCH64=$(sha256sum artifacts/aarch64.tar.gz | cut -d' ' -f1)" >> "$GITHUB_ENV" - echo "SHA256_SOURCE=$(sha256sum artifacts/source.tar.gz | cut -d' ' -f1)" >> "$GITHUB_ENV" - - - name: Install makepkg - run: sudo apt-get update && sudo apt-get install -y pacman-package-manager - - - name: Setup SSH - run: | - mkdir -p ~/.ssh - echo "${{ secrets.AUR_SSH_PRIVATE_KEY }}" > ~/.ssh/aur - chmod 600 ~/.ssh/aur - ssh-keyscan aur.archlinux.org >> ~/.ssh/known_hosts - cat >> ~/.ssh/config << 'EOF' - Host aur.archlinux.org - IdentityFile ~/.ssh/aur - User aur - EOF - - - name: Push baml-bin - env: - VERSION: ${{ needs.prepare.outputs.version }} - run: | - set -euo pipefail - VERSION_AUR="${VERSION//-/.}" # AUR pkgver can't have hyphens - - git clone ssh://aur@aur.archlinux.org/baml-bin.git aur-bin - cd aur-bin - - cp ../packaging/aur/baml-bin/PKGBUILD . - sed -i "s/__VERSION__/${VERSION_AUR}/g" PKGBUILD - sed -i "s/__VERSION_ORIG__/${VERSION}/g" PKGBUILD - sed -i "s/__SHA256_X86_64__/${SHA256_X86_64}/g" PKGBUILD - sed -i "s/__SHA256_AARCH64__/${SHA256_AARCH64}/g" PKGBUILD - - makepkg --printsrcinfo > .SRCINFO - - git config user.name "BoundaryML Bot" - git config user.email "aur@boundaryml.com" - git add PKGBUILD .SRCINFO - if ! git diff --cached --quiet; then - git commit -m "Update to ${VERSION_AUR}" - git push - else - echo "No changes to commit for baml-bin" - fi - - - name: Push baml - env: - VERSION: ${{ needs.prepare.outputs.version }} - run: | - set -euo pipefail - VERSION_AUR="${VERSION//-/.}" # AUR pkgver can't have hyphens - - git clone ssh://aur@aur.archlinux.org/baml.git aur-src - cd aur-src - - cp ../packaging/aur/baml/PKGBUILD . - sed -i "s/__VERSION__/${VERSION_AUR}/g" PKGBUILD - sed -i "s/__VERSION_ORIG__/${VERSION}/g" PKGBUILD - sed -i "s/__SHA256__/${SHA256_SOURCE}/g" PKGBUILD - - makepkg --printsrcinfo > .SRCINFO - - git config user.name "BoundaryML Bot" - git config user.email "aur@boundaryml.com" - git add PKGBUILD .SRCINFO - if ! git diff --cached --quiet; then - git commit -m "Update to ${VERSION_AUR}" - git push - else - echo "No changes to commit for baml" - fi diff --git a/.github/workflows/release-baml-language.yml b/.github/workflows/release-baml-language.yml new file mode 100644 index 00000000000..179e7dffdba --- /dev/null +++ b/.github/workflows/release-baml-language.yml @@ -0,0 +1,625 @@ +name: BAML Language Release + +# Channel-aware replacement for the alpha language release workflow. +# Release workflows run packaging smoke tests only; the full suite runs in CI. + +on: + workflow_run: + workflows: ["CI - BAML Language"] + types: [completed] + workflow_dispatch: + inputs: + channel: + description: "Release channel to plan" + type: choice + options: [nightly, canary] + default: nightly + required: true + dry_run: + description: "Build and publish only dry-run artifacts/manifests" + type: boolean + default: true + required: true + +permissions: + contents: write + actions: read + id-token: write + +concurrency: + group: ${{ inputs.dry_run == true && 'baml-language-release-dryrun' || 'baml-language-release-canary' }} + cancel-in-progress: false + +defaults: + run: + shell: bash + +env: + CARGO_INCREMENTAL: 0 + CARGO_NET_RETRY: 10 + CARGO_TERM_COLOR: always + RUSTUP_MAX_RETRIES: 10 + AWS_REGION: us-east-1 + AWS_ROLE_ARN: arn:aws:iam::277707123528:role/pkg-boundaryml-com-github-release + PKG_BUCKET: pkgboundarymlcomstack-pkgboundarymlcomsitebucket4f-ybhvygkqittp + +jobs: + plan: + name: Plan release + if: | + github.event_name == 'workflow_dispatch' || + (github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.event == 'push' && + github.event.workflow_run.head_branch == 'canary') + runs-on: ubuntu-latest + outputs: + channel: ${{ steps.plan.outputs.channel }} + version: ${{ steps.plan.outputs.version }} + pypi_version: ${{ steps.plan.outputs.pypi_version }} + git_tag: ${{ steps.plan.outputs.git_tag }} + release_plan_json: ${{ steps.plan.outputs.release_plan_json }} + dry_run: ${{ steps.plan.outputs.dry_run }} + wrapper_version: ${{ steps.wrapper.outputs.wrapper_version }} + wrapper_changed: ${{ steps.wrapper.outputs.wrapper_changed }} + dispatch_nightly_after_canary: ${{ steps.plan.outputs.dispatch_nightly_after_canary }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.sha }} + fetch-depth: 2 + + - name: Compute release plan + id: plan + env: + INPUT_CHANNEL: ${{ inputs.channel }} + INPUT_DRY_RUN: ${{ inputs.dry_run }} + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + channel="${INPUT_CHANNEL:-nightly}" + dry_run="${INPUT_DRY_RUN:-false}" + dispatch_nightly_after_canary="false" + if [[ "${{ github.event_name }}" == "workflow_run" ]]; then + channel="nightly" + if git rev-parse HEAD^ >/dev/null 2>&1 && git diff --quiet HEAD^ HEAD -- baml_language/release.toml; then + : + elif git rev-parse HEAD^ >/dev/null 2>&1; then + canary_version="$(scripts/baml-language-version compute --channel canary)" + if ! gh release view "baml-language-${canary_version}" >/dev/null 2>&1; then + channel="canary" + dispatch_nightly_after_canary="true" + fi + fi + fi + scripts/baml-language-version plan --channel "$channel" --out release-plan.json + version="$(jq -r .canonical_version release-plan.json)" + pypi_version="$(jq -r .pypi_version release-plan.json)" + git_tag="$(jq -r .git_tag release-plan.json)" + { + echo "channel=$channel" + echo "version=$version" + echo "pypi_version=$pypi_version" + echo "git_tag=$git_tag" + echo "release_plan_json<> "$GITHUB_OUTPUT" + + - name: Check wrapper version + id: wrapper + run: | + set -euo pipefail + wrapper_version="$(python3 - <<'PY' + import tomllib + from pathlib import Path + print(tomllib.loads(Path("baml_language/crates/baml/Cargo.toml").read_text())["package"]["version"]) + PY + )" + latest="" + if curl -fsSL https://pkg.boundaryml.com/manifest/v1/wrapper.json -o wrapper.json; then + latest="$(jq -r .version wrapper.json)" + fi + changed=$([[ "$wrapper_version" != "$latest" ]] && echo true || echo false) + { + echo "wrapper_version=$wrapper_version" + echo "wrapper_changed=$changed" + } >> "$GITHUB_OUTPUT" + + - uses: actions/upload-artifact@v4 + with: + name: release-plan + path: release-plan.json + + build-toolchain: + name: Build toolchain (${{ matrix.target }}) + needs: [plan, build-vsix] + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - target: aarch64-apple-darwin + os: macos-14 + - target: x86_64-apple-darwin + os: macos-15-intel + - target: x86_64-unknown-linux-gnu + os: ubuntu-latest + - target: aarch64-unknown-linux-gnu + os: ubuntu-24.04-arm + - target: x86_64-unknown-linux-musl + os: ubuntu-latest + - target: aarch64-unknown-linux-musl + os: ubuntu-24.04-arm + - target: x86_64-pc-windows-msvc + os: windows-latest + - target: aarch64-pc-windows-msvc + os: windows-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: release-plan + - uses: actions/download-artifact@v4 + with: + name: baml-vscode-vsix + path: vsix + - name: Stamp version + run: scripts/baml-language-version stamp --plan release-plan.json + - name: Build CLI and pack host + run: cargo build --manifest-path baml_language/Cargo.toml --release --bin baml-cli --bin baml-pack-host --target "${{ matrix.target }}" + - name: Archive layout smoke + run: | + set -euo pipefail + version="${{ needs.plan.outputs.version }}" + target="${{ matrix.target }}" + mkdir -p "staging/bin" "staging/assets" + suffix="" + if [[ "$target" == *windows-msvc ]]; then suffix=".exe"; fi + cp "baml_language/target/$target/release/baml-cli$suffix" "staging/bin/baml-cli$suffix" + cp "baml_language/target/$target/release/baml-pack-host$suffix" "staging/bin/baml-pack-host$suffix" + vsix="$(find vsix -maxdepth 1 -name '*.vsix' -print -quit)" + test -n "$vsix" + cp "$vsix" "staging/assets/baml-vscode.vsix" + if test -f "staging/bin/baml$suffix"; then + exit 1 + fi + test -f "staging/assets/baml-vscode.vsix" + echo "$version" > staging/VERSION + if [[ "$target" == *windows-msvc ]]; then + (cd staging && 7z a "../baml-language-$version-$target.zip" .) + else + tar -C staging -czf "baml-language-$version-$target.tar.gz" . + fi + - uses: actions/upload-artifact@v4 + with: + name: toolchain-${{ matrix.target }} + path: baml-language-${{ needs.plan.outputs.version }}-${{ matrix.target }}.* + + build-vsix: + name: Build platform-neutral VSIX + needs: [plan] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: release-plan + - run: scripts/baml-language-version stamp --plan release-plan.json + - run: pnpm install --frozen-lockfile + - run: pnpm --dir typescript2 run vscode:package + - name: Normalize VSIX asset name + run: | + set -euo pipefail + version="${{ needs.plan.outputs.version }}" + vsix="$(find typescript2/app-vscode-ext -maxdepth 1 -name '*.vsix' -print -quit)" + test -n "$vsix" + mv "$vsix" "typescript2/app-vscode-ext/baml-language-$version.vsix" + - name: Reject bundled native binaries + run: | + set -euo pipefail + vsix="typescript2/app-vscode-ext/baml-language-${{ needs.plan.outputs.version }}.vsix" + unzip -l "$vsix" | tee vsix-list.txt + ! grep -E 'dist/baml-cli|baml-cli|baml-pack-host' vsix-list.txt + - uses: actions/upload-artifact@v4 + with: + name: baml-vscode-vsix + path: typescript2/app-vscode-ext/*.vsix + + build-wrapper: + name: Build wrapper (${{ matrix.target }}) + needs: [plan] + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - target: aarch64-apple-darwin + os: macos-14 + - target: x86_64-apple-darwin + os: macos-15-intel + - target: x86_64-unknown-linux-gnu + os: ubuntu-latest + - target: aarch64-unknown-linux-gnu + os: ubuntu-24.04-arm + - target: x86_64-unknown-linux-musl + os: ubuntu-latest + - target: aarch64-unknown-linux-musl + os: ubuntu-24.04-arm + - target: x86_64-pc-windows-msvc + os: windows-latest + - target: aarch64-pc-windows-msvc + os: windows-latest + steps: + - uses: actions/checkout@v4 + - name: Build wrapper + run: cargo build --manifest-path baml_language/Cargo.toml --release --bin baml --target "${{ matrix.target }}" + - name: Archive wrapper + run: | + set -euo pipefail + version="${{ needs.plan.outputs.wrapper_version }}" + target="${{ matrix.target }}" + mkdir -p staging/bin + suffix="" + if [[ "$target" == *windows-msvc ]]; then suffix=".exe"; fi + cp "baml_language/target/$target/release/baml$suffix" "staging/bin/baml$suffix" + if test -f "staging/bin/baml-cli$suffix"; then + exit 1 + fi + if [[ "$target" == *windows-msvc ]]; then + (cd staging && 7z a "../baml-wrapper-$version-$target.zip" .) + else + tar -C staging -czf "baml-wrapper-$version-$target.tar.gz" . + fi + - uses: actions/upload-artifact@v4 + with: + name: wrapper-${{ matrix.target }} + path: baml-wrapper-${{ needs.plan.outputs.wrapper_version }}-${{ matrix.target }}.* + + all-builds: + name: All builds gate + runs-on: ubuntu-latest + needs: [plan, build-toolchain, build-vsix, build-wrapper] + steps: + - run: echo "all release build jobs completed" + + publish-pypi: + name: Publish Python PyPI + needs: [plan, all-builds] + if: ${{ needs.plan.outputs.dry_run == 'false' && github.ref == 'refs/heads/canary' }} + uses: ./.github/workflows/publish-python-pypi.yml + with: + publish: true + release_plan_json: ${{ needs.plan.outputs.release_plan_json }} + permissions: + id-token: write + contents: read + + publish-github-release: + name: Publish GitHub release assets + needs: [plan, all-builds] + if: ${{ needs.plan.outputs.dry_run == 'false' && github.ref == 'refs/heads/canary' }} + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/download-artifact@v4 + with: + pattern: toolchain-* + path: dist/toolchain + merge-multiple: true + - uses: actions/download-artifact@v4 + with: + pattern: wrapper-* + path: dist/wrapper + merge-multiple: true + - uses: actions/download-artifact@v4 + with: + name: baml-vscode-vsix + path: dist/vsix + - name: Generate checksum sidecars + run: | + set -euo pipefail + find dist -type f \( -name '*.tar.gz' -o -name '*.zip' -o -name '*.vsix' \) -print0 | + while IFS= read -r -d '' file; do + (cd "$(dirname "$file")" && sha256sum "$(basename "$file")" > "$(basename "$file").sha256") + done + - name: Publish toolchain release + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ needs.plan.outputs.git_tag }} + VERSION: ${{ needs.plan.outputs.version }} + run: | + set -euo pipefail + if gh release view "$TAG" >/dev/null 2>&1; then + gh release upload "$TAG" dist/toolchain/* dist/vsix/* --clobber + else + gh release create "$TAG" dist/toolchain/* dist/vsix/* \ + --title "BAML Language $VERSION" \ + --notes "BAML language toolchain $VERSION" + fi + - name: Publish wrapper release + if: ${{ needs.plan.outputs.wrapper_changed == 'true' }} + env: + GH_TOKEN: ${{ github.token }} + WRAPPER_VERSION: ${{ needs.plan.outputs.wrapper_version }} + run: | + set -euo pipefail + tag="baml-wrapper-$WRAPPER_VERSION" + if gh release view "$tag" >/dev/null 2>&1; then + gh release upload "$tag" dist/wrapper/* --clobber + else + gh release create "$tag" dist/wrapper/* \ + --title "BAML Wrapper $WRAPPER_VERSION" \ + --notes "BAML wrapper $WRAPPER_VERSION" + fi + + publish-pkg-boundaryml-com: + name: Publish pkg.boundaryml.com manifests + needs: [plan, all-builds, publish-pypi, publish-github-release, publish-homebrew, publish-aur] + if: ${{ always() && needs.plan.outputs.dry_run == 'false' && github.ref == 'refs/heads/canary' && needs.publish-pypi.result == 'success' && needs.publish-github-release.result == 'success' && (needs.plan.outputs.wrapper_changed != 'true' || (needs.publish-homebrew.result == 'success' && needs.publish-aur.result == 'success')) }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: release-plan + - uses: actions/download-artifact@v4 + with: + pattern: toolchain-* + path: dist/toolchain + merge-multiple: true + - uses: actions/download-artifact@v4 + with: + pattern: wrapper-* + path: dist/wrapper + merge-multiple: true + - uses: actions/download-artifact@v4 + with: + name: baml-vscode-vsix + path: dist/vsix + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.AWS_ROLE_ARN }} + aws-region: ${{ env.AWS_REGION }} + role-session-name: baml-language-release-${{ github.run_id }} + - name: Generate release manifests + run: | + set -euo pipefail + args=() + if [[ "${{ needs.plan.outputs.wrapper_changed }}" == "true" ]]; then + args+=(--wrapper-changed) + fi + scripts/baml-release-manifests \ + --toolchain-dir dist/toolchain \ + --wrapper-dir dist/wrapper \ + --vsix-dir dist/vsix \ + --out manifests \ + --channel "${{ needs.plan.outputs.channel }}" \ + --version "${{ needs.plan.outputs.version }}" \ + --pypi-version "${{ needs.plan.outputs.pypi_version }}" \ + --wrapper-version "${{ needs.plan.outputs.wrapper_version }}" \ + "${args[@]}" + - name: Upload install scripts, index, and manifests + run: | + set -euo pipefail + upload_immutable() { + local source="$1" + local dest="$2" + local tmp + tmp="$(mktemp)" + if aws s3 cp "s3://$PKG_BUCKET/$dest" "$tmp" >/dev/null 2>&1; then + if cmp -s "$source" "$tmp"; then + echo "immutable object already exists and matches: $dest" + else + echo "::error::immutable object exists with different content: $dest" + exit 1 + fi + else + aws s3 cp "$source" "s3://$PKG_BUCKET/$dest" \ + --content-type application/json \ + --cache-control "public, max-age=86400, immutable" + fi + rm -f "$tmp" + } + + version="${{ needs.plan.outputs.version }}" + channel="${{ needs.plan.outputs.channel }}" + upload_immutable "manifests/version/$version.json" "manifest/v1/version/$version.json" + aws s3 cp "manifests/$channel.json" "s3://$PKG_BUCKET/manifest/v1/$channel.json" \ + --content-type application/json \ + --cache-control "public, max-age=60, must-revalidate" + if [[ -f manifests/wrapper.json ]]; then + aws s3 cp manifests/wrapper.json "s3://$PKG_BUCKET/manifest/v1/wrapper.json" \ + --content-type application/json \ + --cache-control "public, max-age=60, must-revalidate" + fi + aws s3 cp scripts/install.sh "s3://$PKG_BUCKET/install.sh" --content-type text/x-shellscript --cache-control "public, max-age=60, must-revalidate" + aws s3 cp scripts/install.ps1 "s3://$PKG_BUCKET/install.ps1" --content-type text/plain --cache-control "public, max-age=60, must-revalidate" + aws s3 cp tools/pkg_boundaryml_com/data/index.html "s3://$PKG_BUCKET/index.html" --content-type text/html --cache-control "public, max-age=300" + + dispatch-nightly-after-canary: + name: Queue nightly after canary publish + needs: [plan, publish-pkg-boundaryml-com] + if: ${{ needs.plan.outputs.dispatch_nightly_after_canary == 'true' && needs.plan.outputs.dry_run == 'false' && github.ref == 'refs/heads/canary' }} + runs-on: ubuntu-latest + permissions: + actions: write + contents: read + steps: + - name: Dispatch nightly release run + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + gh workflow run release-baml-language.yml \ + --ref canary \ + -f channel=nightly \ + -f dry_run=false + + publish-homebrew: + name: Publish Homebrew wrapper formula + needs: [plan, all-builds, publish-github-release] + if: ${{ needs.plan.outputs.wrapper_changed == 'true' && needs.plan.outputs.dry_run == 'false' && github.ref == 'refs/heads/canary' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + pattern: wrapper-* + path: dist/wrapper + merge-multiple: true + - name: Generate Homebrew formula + run: | + set -euo pipefail + scripts/baml-package-manager-artifacts \ + --wrapper-dir dist/wrapper \ + --version "${{ needs.plan.outputs.wrapper_version }}" \ + --out package-manager + - name: Push formula to BoundaryML/homebrew-tap + env: + TOKEN: ${{ secrets.HOMEBREW_BAML_DISPATCH_TOKEN }} + WRAPPER_VERSION: ${{ needs.plan.outputs.wrapper_version }} + run: | + set -euo pipefail + test -n "$TOKEN" + git clone "https://x-access-token:${TOKEN}@github.com/BoundaryML/homebrew-tap.git" tap + cp package-manager/homebrew/Formula/baml.rb tap/Formula/baml.rb + cd tap + git config user.name "boundaryml-release-bot" + git config user.email "boundaryml-release-bot@users.noreply.github.com" + git add Formula/baml.rb + git commit -m "Update baml wrapper to ${WRAPPER_VERSION}" || exit 0 + git push origin HEAD + + publish-aur: + name: Publish AUR wrapper packages + needs: [plan, all-builds, publish-github-release] + if: ${{ needs.plan.outputs.wrapper_changed == 'true' && needs.plan.outputs.dry_run == 'false' && github.ref == 'refs/heads/canary' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + pattern: wrapper-* + path: dist/wrapper + merge-multiple: true + - name: Configure AUR SSH + env: + AUR_SSH_PRIVATE_KEY: ${{ secrets.AUR_SSH_PRIVATE_KEY }} + run: | + set -euo pipefail + test -n "$AUR_SSH_PRIVATE_KEY" + mkdir -p ~/.ssh + printf '%s\n' "$AUR_SSH_PRIVATE_KEY" > ~/.ssh/aur + chmod 600 ~/.ssh/aur + ssh-keyscan aur.archlinux.org >> ~/.ssh/known_hosts + printf 'Host aur.archlinux.org\n IdentityFile ~/.ssh/aur\n User aur\n' >> ~/.ssh/config + - name: Generate AUR package files + run: | + set -euo pipefail + source_sha="$(curl -fsSL "https://github.com/BoundaryML/baml/archive/refs/tags/baml-wrapper-${{ needs.plan.outputs.wrapper_version }}.tar.gz" | sha256sum | awk '{print $1}')" + scripts/baml-package-manager-artifacts \ + --wrapper-dir dist/wrapper \ + --version "${{ needs.plan.outputs.wrapper_version }}" \ + --source-sha256 "$source_sha" \ + --out package-manager + - name: Push AUR repos + env: + WRAPPER_VERSION: ${{ needs.plan.outputs.wrapper_version }} + run: | + set -euo pipefail + for pkg in baml baml-bin; do + git clone "ssh://aur@aur.archlinux.org/${pkg}.git" "aur-${pkg}" + cp "package-manager/aur/${pkg}/PKGBUILD" "aur-${pkg}/PKGBUILD" + cp "package-manager/aur/${pkg}/.SRCINFO" "aur-${pkg}/.SRCINFO" + cd "aur-${pkg}" + git config user.name "boundaryml-release-bot" + git config user.email "boundaryml-release-bot@users.noreply.github.com" + git add PKGBUILD .SRCINFO + git commit -m "Update baml wrapper to ${WRAPPER_VERSION}" || { cd ..; continue; } + git push origin HEAD + cd .. + done + + dry-run-artifacts: + name: Generate dry-run manifests and package files + needs: [plan, all-builds] + if: ${{ needs.plan.outputs.dry_run == 'true' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + pattern: toolchain-* + path: dist/toolchain + merge-multiple: true + - uses: actions/download-artifact@v4 + with: + pattern: wrapper-* + path: dist/wrapper + merge-multiple: true + - uses: actions/download-artifact@v4 + with: + name: baml-vscode-vsix + path: dist/vsix + - name: Generate dry-run files + run: | + set -euo pipefail + args=() + if [[ "${{ needs.plan.outputs.wrapper_changed }}" == "true" ]]; then + args+=(--wrapper-changed) + fi + scripts/baml-release-manifests \ + --toolchain-dir dist/toolchain \ + --wrapper-dir dist/wrapper \ + --vsix-dir dist/vsix \ + --out dry-run/manifest/v1 \ + --channel "${{ needs.plan.outputs.channel }}" \ + --version "${{ needs.plan.outputs.version }}" \ + --pypi-version "${{ needs.plan.outputs.pypi_version }}" \ + --wrapper-version "${{ needs.plan.outputs.wrapper_version }}" \ + "${args[@]}" + scripts/baml-package-manager-artifacts \ + --wrapper-dir dist/wrapper \ + --version "${{ needs.plan.outputs.wrapper_version }}" \ + --out dry-run/package-manager + - uses: actions/upload-artifact@v4 + with: + name: dry-run-release-files + path: dry-run + - uses: aws-actions/configure-aws-credentials@v4 + if: ${{ github.ref == 'refs/heads/canary' }} + with: + role-to-assume: ${{ env.AWS_ROLE_ARN }} + aws-region: ${{ env.AWS_REGION }} + role-session-name: baml-language-dry-run-${{ github.run_id }} + - name: Upload public dry-run manifests + if: ${{ github.ref == 'refs/heads/canary' }} + run: | + set -euo pipefail + aws s3 sync dry-run/manifest/v1 "s3://$PKG_BUCKET/dryrun/${{ github.run_id }}/manifest/v1" \ + --content-type application/json \ + --cache-control "public, max-age=300" + + dry-run-summary: + name: Dry-run summary + needs: [plan, all-builds, dry-run-artifacts] + if: ${{ needs.plan.outputs.dry_run == 'true' }} + runs-on: ubuntu-latest + steps: + - run: | + base="https://pkg.boundaryml.com/dryrun/${{ github.run_id }}/manifest/v1" + { + if [[ "${{ github.ref }}" == "refs/heads/canary" ]]; then + echo "Dry-run manifest base: $base" + else + echo "Dry-run files were uploaded as the dry-run-release-files workflow artifact." + fi + echo + echo 'Validation example:' + echo '```bash' + echo "BAML_HOME=\"\$(mktemp -d)\" BAML_MANIFEST_BASE_URL=\"$base\" baml toolchain use ${{ needs.plan.outputs.channel }}" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/release-cli.yaml b/.github/workflows/release-cli.yaml deleted file mode 100644 index 86197b1c07f..00000000000 --- a/.github/workflows/release-cli.yaml +++ /dev/null @@ -1,9 +0,0 @@ -name: Release BAML CLI - -on: - release: - types: [published, unpublished, created, edited, prereleased] - -jobs: - build-cli: - uses: ./.github/workflows/build-cli-release-reusable.yaml \ No newline at end of file diff --git a/.github/workflows/release-pkg-boundaryml-com.yml b/.github/workflows/release-pkg-boundaryml-com.yml deleted file mode 100644 index 2ffe8e17768..00000000000 --- a/.github/workflows/release-pkg-boundaryml-com.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: release-pkg-boundaryml-com - -# TODO: tie this into the release process so it runs automatically on publish. -on: - workflow_dispatch: - -concurrency: - group: release-pkg-boundaryml-com - cancel-in-progress: false - -permissions: - id-token: write - contents: read - -jobs: - publish: - runs-on: ubuntu-latest - env: - AWS_REGION: us-east-1 - AWS_ROLE_ARN: arn:aws:iam::277707123528:role/pkg-boundaryml-com-github-release - BUCKET: pkgboundarymlcomstack-pkgboundarymlcomsitebucket4f-ybhvygkqittp - SOURCE: tools/pkg_boundaryml_com/data/index.html - steps: - - uses: actions/checkout@v4 - - - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.AWS_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION }} - role-session-name: pkg-boundaryml-com-release-${{ github.run_id }} - - - name: Upload index.html - run: | - aws s3 cp "$SOURCE" "s3://$BUCKET/index.html" \ - --content-type text/html \ - --cache-control "public, max-age=300" diff --git a/TASK/AMENDMENTS.md b/TASK/AMENDMENTS.md new file mode 100644 index 00000000000..b8817c7f669 --- /dev/null +++ b/TASK/AMENDMENTS.md @@ -0,0 +1,110 @@ +# Release Plan Amendments + +These amendments capture the latest team feedback around commands, IDE install, VSIX behavior, and channel-cache behavior. They have been folded into `TASK/TASK.md` and `TASK/USER-PATHS.md`; this file remains as the review-comment rationale. + +## 1. Command Surface + +Use `TASK/COMMANDS.md` as the source of truth for the proposed command additions. + +The key correction is: + +- do not add `baml setup-ide`; +- do not make top-level `baml install` the toolchain/channel command; +- keep toolchain management under `baml toolchain ...`; +- add only one IDE command to the selected CLI payload: `baml-cli ide install`; +- allow users to run that IDE command as `baml ide install` through wrapper pass-through. + +This keeps the wrapper clean while preserving room for future package-manager verbs such as `baml install ` or `baml add `. + +## 2. VSIX And LSP Compatibility + +The VSIX should not force users to install a new extension every time they switch BAML toolchain versions. + +Current pre-wrapper implementation reality: + +- The old VSIX path started the internal CLI's LSP directly and read the LSP `initializeResult.serverInfo.version` for display. +- The native playground WebSocket sends a simple `ready` message when the playground server is connected. +- The browser/WASM playground worker also sends `ready` plus build-time metadata. +- This is readiness/version plumbing, not an explicit compatibility handshake. + +Engine prior art: + +- Engine's old VSIX listened for `baml_src_generator_version`, downloaded/resolved a matching `baml-cli`, and restarted the LSP when the project generator version differed. +- Engine also checked generator/runtime/client versions before generation. +- This is useful prior art for version alignment, but it is not enough for the new model because it fixes mismatches by switching/restarting the LSP. The new VSIX should tolerate users moving between projects/toolchains without requiring extension reinstalls or routine IDE restarts. + +Desired behavior: + +- The VSIX remains a thin IDE client that launches `baml lsp` through the configured `baml` path or `baml` from `PATH`; it should not invoke the internal toolchain binary directly. +- The wrapper resolves the project-specific toolchain before launching the selected toolchain's LSP. +- Two projects open in the same IDE can use different BAML toolchains without requiring the user to reinstall the VSIX or restart the IDE for each switch. +- Add explicit compatibility metadata between the IDE client/playground and the LSP: BAML toolchain version for display/debugging, plus integer protocol ranges for LSP and playground compatibility. +- Compatibility is not BAML semver equality. A new BAML release should not imply a new VSIX install unless the BAML-specific LSP or playground protocol has a breaking change. +- If the LSP and IDE playground are incompatible, the IDE shows a clear diagnostic with the exact action needed. This should be an exceptional extension/toolchain compatibility issue, not the normal per-release path. + +Implementation shape: + +- VSIX/LSP compatibility is piggybacked on the existing LSP `initialize` request/result. Do not add an extra process spawn, extra post-initialize request, or network check just to verify compatibility. +- VSIX sends `initializationOptions.bamlClient` with extension version, supported LSP protocol range, supported playground protocol range, and optional capability flags. +- LSP returns normal `serverInfo.version` plus `capabilities.experimental.baml` metadata with the selected toolchain version, current LSP protocol, current playground protocol, minimum supported client protocol, and capability flags. +- Playground WebSocket compatibility is checked only when the playground opens. In v1, the server sends a `hello` message with playground protocol metadata and then sends the existing `ready` message without waiting for a client round trip; the webview validates `hello` locally and suppresses playground behavior if incompatible. A client that only understands `ready` is treated as legacy protocol 0. `clientHello` is optional/future-facing and must not be required on the v1 startup path. +- The wrapper is not responsible for VSIX compatibility. It only resolves/installs/selects the toolchain and execs `baml-cli lsp`. + +Performance constraints: + +- No compatibility check may run `baml --version`, `baml toolchain list`, `baml toolchain update`, or any network command from VSIX activation. +- No compatibility check may block normal editor startup on playground-only compatibility. +- Compute compatibility once per LSP session and once per playground WebSocket session; do not validate on every message. + +We may still generate a VSIX artifact for every toolchain release for reproducibility, smoke testing, and manual installs. That does not mean users are expected to install that VSIX for every toolchain version. Marketplace or normal extension updates should happen only when the IDE client/protocol actually needs to change. + +## 3. IDE Install Timing + +Do not automatically install the IDE extension as a side effect of installing the wrapper or installing/updating a toolchain. + +Instead: + +- `brew install baml`, curl install, and `baml toolchain use/install ...` should get the wrapper/toolchain into a working CLI state. +- When a user runs any `baml` command from a supported IDE terminal, BAML may detect that context and check whether the IDE extension appears to be installed. +- If the extension is missing, print a short one-time recommendation: + +```text +BAML IDE extension is not installed for this editor. +Run: baml ide install +``` + +- The recommendation must not block the command being run. +- Non-interactive contexts should not prompt. +- The prompt should be rate-limited or recorded so repeated terminal commands do not spam the user. + +The install command itself remains owned by `baml-cli ide install`; `baml ide install` is just wrapper resolution plus pass-through. + +## 4. Channel And Manifest Cache + +Normal pass-through commands must not perform a blocking network check just because the user is on `nightly`. + +The wrapper should store enough local state to run offline: + +- active selector, such as `canary`, `nightly`, or a concrete version; +- resolved concrete version currently in use; +- installed toolchain directories; +- cached channel manifests under `~/.baml/manifest-cache/`. + +Initial cache policy: + +- Channel metadata cache TTL is 24 hours. +- Only allowlisted commands make network requests: + - `baml toolchain update`: always checks the network and advances the active channel when possible. + - `baml toolchain install`: always resolves/downloads/verifies the requested selector or version. + - `baml toolchain use`: checks the network only when the relevant channel cache is missing or expired. + - `baml toolchain list`: local-only by default; a remote/latest mode may check the network when the relevant cache is missing or expired. +- All other commands, including `baml generate`, `baml run`, `baml describe`, `baml pack`, and `baml lsp`, use local state only and never hit the network. + +Network fetch behavior: + +- Treat the old cache entry as stale once a command decides to make a network request. +- Fetch to a temporary file, validate schema and checksums, then atomically replace the cache entry. +- If validation fails, delete the fetched temporary file and do not change the active concrete toolchain. +- If the network fails, keep running the already-installed concrete toolchain and report that the latest remote version could not be checked. + +This gives nightly users predictable local execution while still making `baml toolchain update` and remote status commands honest about freshness. diff --git a/TASK/COMMANDS.md b/TASK/COMMANDS.md new file mode 100644 index 00000000000..6e51ee11670 --- /dev/null +++ b/TASK/COMMANDS.md @@ -0,0 +1,54 @@ +# BAML Command Additions + +This is the short command-surface summary for the release-pipeline plan. + +The important split: + +- `baml` is the wrapper users run. +- `baml-cli` is the selected language toolchain binary. +- Normal commands like `baml generate`, `baml run`, `baml describe`, `baml pack`, and `baml lsp` are not new wrapper commands. The wrapper resolves a toolchain and forwards them to `baml-cli`. + +## Added To `baml` + +| Command | Purpose | +|---|---| +| `baml toolchain install ` | Download and verify a toolchain without necessarily making it active. Channel inputs resolve to a concrete version at install time. | +| `baml toolchain use ` | Select the active default toolchain. Installs the resolved toolchain if missing. | +| `baml toolchain update` | Refresh the active channel and advance to the latest canary/nightly toolchain. Concrete pinned versions do not move. | +| `baml toolchain list` | Show installed toolchains and the active selector/version. May have a remote/list mode later. | +| `baml toolchain uninstall ` | Remove an installed concrete toolchain. | +| `baml self-update` | Update only the wrapper for curl-installed wrappers. Refuse for package-manager-managed wrappers with the right package-manager command. | + +## Added To `baml-cli` + +| Command | Purpose | +|---|---| +| `baml-cli ide install` | Install or update the IDE extension/assets for the selected toolchain. The wrapper exposes this as `baml ide install` by resolving the toolchain and forwarding the command. Common editors should have non-interactive flags such as `--cursor`, and `--code`. | + +## Possible `baml-cli` Work Because Of The Wrapper + +These are not approved new commands yet. They are implementation pressure points to keep visible. + +| Possible Surface | Why It Might Be Needed | +|---|---| +| `baml-cli ide status` or `baml-cli ide doctor` | Only if `baml ide install` needs a clean way to detect installed editor extensions, explain mismatches, or debug user setup. Prefer avoiding this unless the implementation needs it. | +| `baml-cli lsp` compatibility metadata | Not a new command. The LSP reports BAML-specific protocol metadata through the existing LSP `initialize` result, and the playground reports playground protocol metadata when the WebSocket connects. The VSIX uses integer protocol ranges and capability flags so multiple toolchain versions can work without a new extension install per release. This must not require extra VSIX startup commands or network checks. | + +## Not Added + +| Command | Reason | +|---|---| +| `baml setup-ide` | Removed from the plan. IDE installation belongs to `baml-cli ide install`; users may invoke it through `baml ide install`. | +| `baml install nightly` / `baml install canary` | Avoid spending the top-level `install` verb on toolchains if BAML later grows project packages. Use `baml toolchain ...`. | +| `baml update` as the primary toolchain command | Avoid ambiguity with future package/dependency updates. Use `baml toolchain update`. | + +## Reserved For Future Package Management + +These are intentionally not part of the toolchain release plan: + +| Future Command | Likely Meaning | +|---|---| +| `baml install ` | Install a BAML project package/dependency. | +| `baml add ` | Add a BAML project package/dependency. | +| `baml remove ` | Remove a BAML project package/dependency. | +| `baml update` | Update project dependencies/packages, if that becomes the chosen package-manager shape. | diff --git a/TASK/FAILURE-MODES.md b/TASK/FAILURE-MODES.md new file mode 100644 index 00000000000..31b47a321d9 --- /dev/null +++ b/TASK/FAILURE-MODES.md @@ -0,0 +1,58 @@ +# BAML Failure Modes + +This document lists the user-facing failures we expect from the wrapper/toolchain/runtime split. + +## Compatibility Policy + +| Surface | Compatibility Rule | +|---|---| +| `baml-wrapper` | Must understand the manifest schema and project `[toolchain]` schema. It may be older than the toolchain if schemas are still supported. | +| Internal toolchain binary / LSP | Must match the project-selected concrete toolchain version. The wrapper should prevent accidental skew. User-facing docs and diagnostics should say `baml`, not `baml-cli`, except when describing internal implementation. | +| Canary SDK runtimes, such as Python `baml_core` | Compatible across the same major/minor line. Patch skew is allowed unless a release explicitly marks a patch as breaking. | +| Nightly SDK runtimes | Prefer exact canonical version match unless we explicitly define nightly ranges later. | +| VSIX / IDE client | Compatible by explicit LSP/playground protocol range and capability flags, not by exact BAML toolchain version. Protocol metadata is exchanged through existing LSP initialize and playground WebSocket startup paths, without network checks. | + +## Expected User Errors + +| Case | Tool That Fails | Expected Error | +|---|---|---| +| `baml.toml` pins `0.3.0`, but the wrapper is too old to understand the project or manifest schema | `baml` wrapper | `This project/manifest requires wrapper schema support newer than this baml wrapper. Update the wrapper with: baml self-update` or the package-manager command, such as `brew upgrade baml`. | +| `baml.toml` pins `0.3.0`, but only toolchain `0.4.0` is installed | `baml` wrapper | `Project pins BAML 0.3.0, but that toolchain is not installed. Installed toolchains: 0.4.0. Run: baml toolchain install 0.3.0`. The wrapper must not fall forward to `0.4.0`. | +| `baml.toml` selects `nightly`, local nightly is installed, network is unavailable | `baml` wrapper | Normal commands keep using the installed concrete nightly. `baml toolchain update` reports that the latest nightly could not be checked. | +| `baml.toml` selects `nightly`, but no concrete nightly is installed and network is unavailable | `baml` wrapper | `No installed nightly toolchain is available, and the latest nightly could not be fetched. Connect to the network or install a concrete version.` | +| User runs the internal toolchain binary directly inside a project pinned to another version | Internal toolchain binary | Print a warning to stdout before continuing or failing: `Using the internal BAML toolchain binary directly is not recommended. Use baml instead.` If it detects a version mismatch: `This project pins BAML 0.3.0, but this toolchain is 0.4.0. Run: baml toolchain install 0.3.0`. | +| Installed toolchain directory exists but is missing `baml-cli` or checksum metadata | `baml` wrapper | `Installed toolchain 0.3.0 is corrupt. Reinstall with: baml toolchain install 0.3.0 --force`. | +| Manifest points to an archive whose checksum does not match | `baml` wrapper | `Checksum verification failed for BAML 0.3.0. The toolchain was not installed.` | +| Project/generated Python code expects `0.3.x`, but `baml_core` is `0.2.0` | Python runtime / generated client | `BAML runtime mismatch: generated with 0.3.x, but baml_core is 0.2.0. Update the Python runtime with: uv add 'baml_core~=0.3' or pip install 'baml_core~=0.3'. Then regenerate with: baml generate.` | +| Project/generated Python code expects `0.3.0`, but `baml_core` is `0.3.2` | Python runtime / generated client | Allowed for canary releases. No error if major/minor match and the generated compatibility range permits it. | +| Project/generated Python code expects `0.3.x`, but `baml_core` is `0.4.0` | Python runtime / generated client | `BAML runtime mismatch: generated for 0.3.x, but baml_core is 0.4.0. Downgrade the runtime with: uv add 'baml_core~=0.3' or pip install 'baml_core~=0.3'. Or update the toolchain with: baml toolchain update, then run: baml generate.` | +| Nightly generated code expects `0.3.1-nightly.20260529.a`, but `baml_core` is another nightly | Python runtime / generated client | Fail exact-match by default: `Nightly runtime mismatch. Install the exact baml_core version with your Python package manager, or regenerate with: baml generate.` | +| Node/TypeScript runtime package does not match generated-code compatibility range | Node/TypeScript runtime / generated client | Same policy as Python: canary patch skew allowed within range; major/minor mismatch fails; nightly exact-match unless changed later. | +| VSIX launches an LSP whose BAML LSP protocol range is unsupported | VSIX / IDE client | Show an IDE diagnostic and avoid BAML custom requests: `This BAML extension is not compatible with the selected BAML toolchain's LSP protocol. Update the extension or switch toolchains.` | +| Playground webview connects to an LSP playground server whose playground protocol range is unsupported | VSIX / IDE client | Keep LSP/editor language features running, but disable or fail only the playground: `This BAML extension is not compatible with the selected BAML toolchain's playground protocol. Update the extension or switch toolchains.` | +| VSIX cannot find `baml` on `PATH` and no configured path exists | VSIX / IDE client | `BAML command not found. Install BAML or set the BAML executable path in extension settings.` | +| User invokes `baml self-update` from a Homebrew/AUR-managed wrapper | `baml` wrapper | Refuse: `This wrapper is managed by Homebrew/AUR. Update with: brew upgrade baml` or the detected manager's command. | +| User invokes future package-style `baml install ` before package management exists | `baml` wrapper | `BAML package installation is not implemented yet. For toolchains, use baml toolchain install .` | + +## Error Ownership + +| Tool | Owns These Failures | +|---|---| +| `baml` wrapper | Toolchain resolution, install/update, manifest/schema/checksum, corrupt cache, managed self-update refusal, missing local toolchain. | +| Internal toolchain binary | Direct-binary warning/version mismatch, language commands, generation, `pack`, `lsp`, and `ide install`. User-facing docs should describe these as `baml ` because the wrapper is what users run. | +| SDK runtimes (`baml_core`, Node runtime, future runtimes) | Generated-code/runtime compatibility checks at import/init/call time. They never self-update. | +| VSIX / IDE client | Finding `baml`, launching LSP, constant-based LSP/playground protocol compatibility diagnostics, extension install/update UX. It must not perform network checks or spawn extra `baml` commands merely to determine compatibility. | +| Package managers | Wrapper package installation and wrapper package upgrades. | + +## Release Coupling Decision + +We decouple the wrapper from the language toolchain. + +| Product | Release Cadence | Why | +|---|---|---| +| `baml-wrapper` | Rare, independent | It only resolves, installs, updates, and forwards. Shipping it through package managers on every language release would create unnecessary churn. | +| `baml-toolchain` | Every canary/nightly language release | It contains the versioned language behavior: `baml-cli`, LSP, pack host, SDK runtimes, generated-code compatibility metadata, and VSIX artifact. | + +We should not decouple `baml-cli`, Python `baml_core`, Node runtime/codegen surfaces, and VSIX artifacts inside the language release. They should be produced from one release plan and one canonical BAML version. + +Reason: decoupling those surfaces creates a cross-product compatibility matrix for users and support. The plan should only have one compatibility question for a project: "Which BAML language version does this project use?" diff --git a/TASK/OPEN-QUESTIONS.md b/TASK/OPEN-QUESTIONS.md new file mode 100644 index 00000000000..9d5400a1ae6 --- /dev/null +++ b/TASK/OPEN-QUESTIONS.md @@ -0,0 +1,962 @@ +**Highest Priority** +1. **VSIX multi-project/toolchain model is underspecified** + [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:603), [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:1000) + The doc says the VSIX launches `baml lsp` and should support projects/toolchains without reinstalling the VSIX, but it does not define whether the extension starts one LSP per workspace folder, one per `baml.toml` project root, or one global LSP. Since the wrapper resolves toolchain at process start from cwd, a single LSP launched from “first workspace folder” cannot safely support multiple open projects with different `[toolchain]` pins. This needs a concrete process model. + + **Answer:** Start one lazy VS Code `LanguageClient` / LSP process per BAML project root, not one global LSP and not one process per file. A BAML project root is the nearest ancestor of the active/open `.baml` file containing `baml.toml`; if none exists, fall back to the containing VS Code workspace folder; if the file is outside any workspace folder, use the file's directory as an ad-hoc root or run in limited-support mode. + + The VSIX maintains a `Map`. When a `.baml` file opens or becomes active, it finds that file's BAML project root, starts a client if one does not already exist, and launches it as `baml lsp` with `cwd = `. This makes wrapper toolchain resolution match the CLI behavior for that project: `$BAML_VERSION`, nearest `baml.toml [toolchain]`, user default, then canary fallback. + + Project-root-per-client is required for monorepos where two sibling BAML projects may pin different toolchains. Workspace-folder-per-client is not sufficient because one workspace can contain multiple `baml.toml` roots. Nested roots are routed to the nearest BAML project root. To guard against VS Code document-selector leakage, each LSP receives its intended `projectRoot` in `initializationOptions.baml.projectRoot` and ignores documents whose nearest `baml.toml` root differs from that declared root. + + Playground ownership is per LSP process. Each project-root LSP owns its own playground server/port, and `baml.openPlayground` uses the client associated with the active editor's project root. Restart only the affected project-root client when that project's `[toolchain]` changes, the configured `baml` executable path changes, compatibility metadata changes after a wrapper/toolchain update, or that LSP crashes. Restart all clients only for truly global setting changes. + +2. **Package-manager bootstrap may be impossible or inconsistent as written** + [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:96), [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:936) + It says package-manager artifacts install only the wrapper, but first-time `brew install baml` should leave a working canary toolchain. The doc does not say whether Homebrew post-install runs `baml toolchain install canary`, whether that is acceptable for Homebrew tap policy, and what AUR does since package scripts run as root and cannot safely write a user’s `~/.baml`. This needs per-package-manager mechanics. + + **Answer:** Package-manager installs must remain wrapper-only in v1. `brew install baml`, AUR `baml`, and AUR `baml-bin` install only the `baml` wrapper and must not download, install, select, or update a BAML language toolchain during package install, reinstall, or upgrade. + + The previous goal that `brew install baml` should leave the user with a working canary toolchain is removed. The replacement promise is: package managers put a working `baml` wrapper on `PATH` and print the exact user-scoped follow-up command to install/select the canary toolchain. + + Package-manager caveats/post-install messages should say: + + ```text + BAML wrapper installed. + + To install and select the current canary language toolchain: + baml toolchain use canary + + To use nightly: + baml toolchain use nightly + + IDE extension setup is explicit: + baml ide install --cursor + ``` + + Rationale: + + - AUR/package install scripts are system package lifecycle hooks and must not write user-scoped `~/.baml` state. + - Homebrew supports post-install work, but using it to download and select a mutable language toolchain would blur the wrapper/toolchain boundary and make `brew install baml` depend on channel state at install time. + - `brew upgrade baml` and `paru -Syu baml-bin` must update only the wrapper. They must not silently change the user's active toolchain/channel/project behavior. + - Curl installers remain different: because they are explicitly user-scoped BAML installers, they may still install the wrapper and bootstrap the requested toolchain unless `--wrapper-only` is passed. + + This decision depends on point 3: `baml toolchain use ` should be the one-command user setup path. If the selected concrete toolchain is missing, `use` should install it and then mark it active. `install` should download a toolchain without making it active. + +3. **`baml toolchain use` vs `install` semantics are still ambiguous** + [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:274), [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:322) + The first-run error suggests `baml toolchain use canary`, but package-manager bootstrap uses both `install` and `use`. Network policy says `use` may fetch channel metadata, but does not clearly say whether `use` also installs the selected toolchain if missing. This is a core UX behavior and should be exact. + + **Answer:** `baml toolchain use ` is the one-command setup path. It resolves the selector, ensures the selected concrete toolchain is installed, and then records that selector as the user's default. If the concrete toolchain is missing, `use` downloads/verifies/installs it before marking it active. + + `baml toolchain install ` downloads/verifies/installs the selected concrete toolchain but does not change the user's active default selector. It is for prefetching, CI image setup, and installing alternate versions side by side. + + `baml toolchain update` refreshes the currently active default selector when it is a channel (`canary` or `nightly`) and advances it to the latest concrete version. If the active default is an exact version, `update` should report that exact versions do not advance automatically and suggest `baml toolchain use canary` or `baml toolchain use nightly`. + + `baml toolchain list` is local-only by default. A separate explicit remote mode may be added, but plain `list` must not hit the network. + + Network behavior: + + - `use ` may fetch channel metadata when the channel cache is missing or expired, then install the selected concrete toolchain if missing. + - `use ` fetches the immutable per-version manifest only if that version is not installed locally or its cached manifest is missing. + - `install ` always resolves the latest channel pointer and installs that concrete version, but does not make it active. + - `install ` fetches that immutable version manifest only when needed. + - Normal pass-through commands never install or update toolchains. + + First-run package-manager caveat therefore uses one command: + + ```text + baml toolchain use canary + ``` + + First-run no-toolchain error should also recommend `baml toolchain use canary` as the primary fix. This command both installs and selects the canary toolchain. + +4. **Wrapper state schema is not concrete enough** + [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:322), [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:344) + The text refers to “locally recorded concrete version” for channels, but `config.toml` only shows `[default] channel = "canary"`. Where is `canary -> 0.11.0` stored? In manifest cache? In config? In a separate channel state file? This matters for offline behavior and atomic updates. + + **Answer:** Use separate files for user intent, wrapper-owned machine state, remote metadata cache, and installed toolchain inventory. Do not make the manifest cache authoritative for normal command resolution. + + Final layout: + + ```text + ~/.baml/ + config.toml # user-authored/user-editable intent + state.toml # wrapper-owned active channel resolutions + manifest-cache/ # fetched remote metadata, cache only + canary.json + nightly.json + version/ + 0.11.0.json + toolchains/ + 0.11.0/ + VERSION + install.json + bin/baml-cli + bin/baml-pack-host + assets/baml-vscode.vsix + ``` + + `config.toml` stores what the user wants: + + ```toml + [default] + selector = "canary" # "canary" | "nightly" | "" + + [update] + auto_check = false + ``` + + `state.toml` stores the last successfully installed/activated concrete version for each channel: + + ```toml + [channels.canary] + active_version = "0.11.0" + resolved_at = "2026-06-02T12:00:00Z" + manifest_path = "manifest-cache/version/0.11.0.json" + + [channels.nightly] + active_version = "0.11.1-nightly.20260602.a" + resolved_at = "2026-06-02T12:30:00Z" + manifest_path = "manifest-cache/version/0.11.1-nightly.20260602.a.json" + ``` + + `manifest-cache/` stores remote JSON plus fetch metadata where useful (`fetched_at`, `etag`, etc.). It is not the active-version authority. A remote/list operation may refresh `manifest-cache/canary.json` and discover a newer version, but normal commands must continue using `state.toml` until a toolchain-management command successfully installs and activates the newer concrete version. + + `toolchains//VERSION` contains the exact canonical version and is read as a tamper/sanity check on every wrapper invocation. `toolchains//install.json` should record install metadata such as source manifest URL, archive URL, archive sha256, installed_at, and target triple. + + Resolution invariants: + + - Normal pass-through commands never hit the network and never resolve directly from `manifest-cache`. + - Exact-version selectors resolve directly to `toolchains//` and require that directory to exist and pass the `VERSION` sanity check. + - Channel selectors (`canary`, `nightly`) resolve through `state.toml` to an installed concrete version. + - If `config.toml` selects a channel but `state.toml` has no active version for that channel, normal commands fail with the primary fix `baml toolchain use `. + - If `state.toml` points to a missing or corrupt toolchain directory, normal commands fail with a corrupt/missing state diagnostic and suggest `baml toolchain use ` or `baml toolchain install --force`. + + Mutation rules: + + - `baml toolchain use ` fetches channel metadata if needed, installs the selected concrete version if missing, then atomically updates `state.toml` and records `[default].selector = ""` in `config.toml`. + - `baml toolchain use ` installs the version if missing, then records `[default].selector = ""` in `config.toml`. It does not update any channel entry in `state.toml`. + - `baml toolchain install ` fetches and installs the latest concrete version for the channel, but does not update `[default].selector` and does not update that channel's `active_version` in `state.toml`. + - `baml toolchain install ` installs only that version and does not mutate `config.toml` or `state.toml`. + - `baml toolchain update` only advances `state.toml` when the active default selector is a channel. It installs the latest concrete version for that channel, then atomically swaps the channel's `active_version`. If the active selector is an exact version, it reports that exact versions do not advance automatically. + + Atomicity rule: write `state.toml` and `config.toml` through temp files in the same directory, validate the serialized TOML, fsync where practical, then rename. Only update state after the toolchain archive is downloaded, verified, extracted, `VERSION`-checked, and fully materialized under `toolchains//`. + + This gives the wrapper a clean invariant: + + ```text + user intent (config.toml) + + active local channel state (state.toml) + + installed toolchain metadata (toolchains//VERSION) + = normal command resolution + ``` + + Remote manifests are only inputs to explicit toolchain-management commands. + +5. **Dry-run release testing lacks a wrapper override mechanism** + [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:1156), [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:1172) + The doc says dry-run may write to a `dryrun/` prefix and merge criteria include `baml toolchain install ` against a dry-run manifest. But the wrapper only knows `https://pkg.boundaryml.com/manifest/v1/...`. We need an explicit env var or flag like `BAML_MANIFEST_BASE_URL` for tests and mirrors. + + **Answer:** A dry run means running the release graph as realistically as possible without publishing to production release paths consumed by users. Production users read: + + ```text + https://pkg.boundaryml.com/manifest/v1/canary.json + https://pkg.boundaryml.com/manifest/v1/nightly.json + https://pkg.boundaryml.com/manifest/v1/version/.json + ``` + + Dry-run releases publish to an isolated namespace instead, for example: + + ```text + https://pkg.boundaryml.com/dryrun//manifest/v1/nightly.json + https://pkg.boundaryml.com/dryrun//manifest/v1/version/.json + https://pkg.boundaryml.com/dryrun//artifacts/ + ``` + + The wrapper therefore needs a manifest-base override so validation can point at the dry-run namespace: + + ```text + BAML_MANIFEST_BASE_URL=https://pkg.boundaryml.com/dryrun//manifest/v1 + ``` + + Default remains: + + ```text + https://pkg.boundaryml.com/manifest/v1 + ``` + + Maintainer workflow: + + - A maintainer runs `.github/workflows/release-baml-language.yml` manually with `workflow_dispatch`. + - Inputs include at least `channel` and `dry_run`. + + ```text + channel: nightly | canary + dry_run: true + ``` + + - Equivalent `gh` command: + + ```bash + gh workflow run release-baml-language.yml \ + -f channel=nightly \ + -f dry_run=true + ``` + + - The workflow builds, packages, and smoke-tests exactly like a production release. + - In dry-run mode, publish jobs do not write production channel pointers and do not create production user-visible release state. + - Dry-run manifests and artifacts are uploaded under `pkg.boundaryml.com/dryrun//...`, or kept as workflow artifacts when public HTTP access is not required. If wrapper install validation is required, the dry-run manifest must point at HTTP-accessible dry-run artifact URLs. + - The workflow summary prints the dry-run manifest base URL and example validation command. + + Validation command shape: + + ```bash + BAML_HOME="$(mktemp -d)" \ + BAML_MANIFEST_BASE_URL="https://pkg.boundaryml.com/dryrun//manifest/v1" \ + baml toolchain use nightly + ``` + + Use a temporary `BAML_HOME` for dry-run validation so production user/developer state is not polluted. + + Wrapper override rules: + + - `BAML_MANIFEST_BASE_URL` is required for dry-run wrapper validation and useful for mirrors/internal test fixtures. + - An optional explicit CLI flag may also be added for toolchain-management commands: + + ```text + baml toolchain install --manifest-base-url + baml toolchain use --manifest-base-url + baml toolchain update --manifest-base-url + ``` + + - Precedence is: `--manifest-base-url` -> `BAML_MANIFEST_BASE_URL` -> production default. + - The override applies only to wrapper/toolchain manifest reads. Normal pass-through commands still do not hit the network. + - The override is never persisted into `config.toml`. + - Cache entries are namespaced by manifest base URL so dry-run manifests cannot poison the production cache. Production may use `manifest-cache/prod/`; overrides use `manifest-cache/override//`. + - Channel state written under an override records the manifest base URL/hash in `state.toml`. A later command using a different manifest base must not silently treat that channel state as valid; it should either use a temporary `BAML_HOME` in CI or print a clear diagnostic telling the user to run `baml toolchain use ` under the current source. + +6. **Release concurrency is not specified** + [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:178), [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:529) + Nightly suffix selection reads existing GitHub releases and picks the next letter. Without a workflow concurrency group or locking rule, two successful `canary` runs close together can compute the same letter. The PyPI collision check catches some fallout, but the release graph should prevent the race. + + **Answer:** GitHub merge queue is the first line of defense: all normal writes to `canary` should go through the queue, and the release workflow should publish only from actual `push` events / successful CI on `refs/heads/canary`, not from PR or `merge_group` branches. However, merge queue is not the release lock. Manual dispatches, retries, admin bypasses, and slow publish jobs can still overlap. + + Add a non-cancelling GitHub Actions concurrency group to the release graph entrypoint: + + ```yaml + concurrency: + group: baml-language-release-canary + cancel-in-progress: false + ``` + + This serializes all production BAML language release runs. Do not include the version, channel, run id, or commit SHA in this group; those would allow overlapping runs and defeat the lock. `cancel-in-progress: false` is required so GitHub queues later releases instead of cancelling a run that may already be halfway through publishing. + + Manual `workflow_dispatch` production publishes use the same concurrency group. Dry-run releases may either use the same group for maximum simplicity or a separate dry-run group if we want dry-run validation not to block production: + + ```yaml + concurrency: + group: ${{ inputs.dry_run == 'true' && 'baml-language-release-dryrun' || 'baml-language-release-canary' }} + cancel-in-progress: false + ``` + + Production publish jobs must still perform idempotency/uniqueness checks after acquiring the concurrency slot: + + - Recompute/read the frozen `release-plan.json`. + - Check whether `baml-language-` already exists. + - If the tag/release exists and all expected artifacts/manifests match, treat rerun as success/idempotent repair. + - If the tag/release exists but content differs, hard-fail. + - If publishing nightly, choose the nightly letter inside the serialized release run, after the concurrency slot is acquired. + + With this rule, merge queue serializes branch state, and the release workflow serializes publishing state. + +7. **Rollback is too optimistic** + [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:1190) + “Rollback is a code rollback” restores workflow files, but it does not address already-published mutable pointers like `canary.json`, `nightly.json`, `wrapper.json`, install scripts, or a bad GitHub release. The rollback playbook needs explicit behavior for channel pointers and whether to advance to a fixed version, repoint, or leave immutable versions alone. + + **Answer:** Rollback is code revert plus mutable pointer repair. `git revert ` restores workflow files, but it does not undo external release state. The rollback playbook must distinguish immutable release artifacts from mutable pointers. + + Immutable artifacts are never overwritten or deleted during normal rollback: + + ```text + manifest/v1/version/.json + GitHub release baml-language- + GitHub release baml-wrapper- + PyPI baml_core + ``` + + If a bad version was published, leave the immutable record intact and move the channel pointer away from it. This preserves reproducibility and avoids users seeing disappearing versions. + + Mutable pointers may be repaired: + + ```text + manifest/v1/canary.json + manifest/v1/nightly.json + manifest/v1/wrapper.json + install.sh + install.ps1 + index.html + Homebrew formula / AUR metadata, for wrapper releases + ``` + + Channel pointer repair flow: + + 1. Choose the last known good concrete version. + 2. Download `manifest/v1/version/.json`. + 3. Validate schema and target completeness. + 4. Upload that JSON as `manifest/v1/canary.json` or `manifest/v1/nightly.json`. + 5. Use mutable cache headers: + + ```text + Cache-Control: public, max-age=60, must-revalidate + ``` + + Wrapper rollback is separate. If a bad wrapper release was published, repair `manifest/v1/wrapper.json` and any package-manager definitions that point to it. Do not delete the bad `baml-wrapper-` GitHub release. Prefer publishing a newer fixed wrapper when package-manager version ordering makes repointing awkward. + + PyPI rollback is fix-forward. Do not try to delete/reuse a published `baml_core` version. Move channel pointers away from the bad release when possible and publish the next nightly/canary with a fix. + + Add an explicit rollback script or manual workflow, for example: + + ```bash + scripts/baml-release-rollback point-channel \ + --channel nightly \ + --version 0.11.1-nightly.20260601.b + + scripts/baml-release-rollback point-channel \ + --channel canary \ + --version 0.11.0 + + scripts/baml-release-rollback point-wrapper \ + --version 0.1.0 + ``` + + The rollback script must not publish new immutable version artifacts. It only validates existing immutable manifests/releases and repairs mutable pointers. + + If the release graph itself is broken: + + 1. Stop or avoid further production publishes. + 2. Repair user-facing mutable pointers if users are affected. + 3. Revert the merge commit. + 4. Confirm old workflow files are restored. + 5. Run one controlled release or dry run if needed. + + Maintainers have these rollback tools available: + + - `git revert ` for repo/workflow rollback. + - `scripts/baml-release-rollback point-channel` for `canary.json` / `nightly.json`. + - `scripts/baml-release-rollback point-wrapper` for `wrapper.json`. + - Fix-forward release for immutable package mistakes. + - Short cache TTL on mutable pointers so pointer repair reaches wrappers quickly. + +**Medium Priority** +8. **Wrapper release version source is vague** + [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:83), [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:963) + It mentions wrapper crate version / `BAML_WRAPPER_VERSION` and a wrapper-version-changed check, but not the authoritative file or comparison rule. Is it `baml_language/crates/baml/Cargo.toml`, a constant, or workflow metadata? + + **Answer:** The authoritative wrapper version is the literal package version in `baml_language/crates/baml/Cargo.toml`, once the wrapper crate exists. Remove `BAML_WRAPPER_VERSION` from the plan; do not introduce an environment-variable version authority for the wrapper. + + Future wrapper crate shape: + + ```toml + # baml_language/crates/baml/Cargo.toml + [package] + name = "baml" + version = "0.1.0" + publish = false + ``` + + The wrapper crate must not inherit the workspace version: + + ```toml + # Do not do this for the wrapper: + version.workspace = true + ``` + + Reason: `baml_language/Cargo.toml` workspace `version = "0.0.0-beta"` is intentionally not the public BAML language/toolchain version and is also not the wrapper version. The wrapper is the one Rust package in this plan whose `CARGO_PKG_VERSION` is intentionally meaningful as a public product version. + + `baml --version` may use `env!("CARGO_PKG_VERSION")` because the wrapper's Cargo package version is the wrapper product version. This is different from `baml-cli`, LSP, SDK runtimes, and generated-code surfaces, which must use the stamped BAML language version instead. + + Release graph rule: + + ```text + wrapper_version = parse baml_language/crates/baml/Cargo.toml [package].version + latest_wrapper_version = fetch manifest/v1/wrapper.json version if it exists + wrapper_changed = wrapper_version != latest_wrapper_version + ``` + + If `wrapper_changed` is true, publish the wrapper release: + + ```text + GitHub release baml-wrapper- + wrapper archives + manifest/v1/wrapper.json + Homebrew formula + AUR packages + install.sh / install.ps1 only if changed + ``` + + If `wrapper_changed` is false, build and smoke-test the wrapper in the release graph, but do not publish wrapper artifacts or package-manager updates. + + Required checks: + + - Fail if `baml_language/crates/baml/Cargo.toml` uses `version.workspace = true`. + - Fail if the wrapper version is not valid SemVer. + - Fail if the wrapper version is lower than the latest published `manifest/v1/wrapper.json` version. + - Fail if any workflow step tries to set or consume `BAML_WRAPPER_VERSION`. + - Fail if `baml --version` does not print exactly `baml ` in smoke tests. + - Keep `BAML_RELEASE_VERSION` removed for the language/toolchain path; do not replace it with a wrapper-specific env var. + + Bump UX: + + - V1 may document a manual edit to `baml_language/crates/baml/Cargo.toml [package].version`. + - Prefer adding a helper so maintainers do not hand-edit SemVer incorrectly: + + ```bash + scripts/baml-wrapper-version bump --patch + scripts/baml-wrapper-version bump --minor + scripts/baml-wrapper-version check + ``` + + - The helper edits only `baml_language/crates/baml/Cargo.toml` and runs the wrapper-version checks above. It does not touch `baml_language/release.toml`, `baml_version`, `baml_core`, VSIX version, SDK versions, or any BAML language/toolchain version surface. + + Final invariant: + + ```text + Wrapper version source: baml_language/crates/baml/Cargo.toml [package].version + Language/toolchain version source: baml_language/release.toml + release-plan.json stamping + ``` + +9. **Direct `baml-cli` warning detection needs a mechanism** + [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:228) + It says `baml-cli` should warn when invoked outside the wrapper. How does it know? The wrapper likely needs to set an env var such as `BAML_WRAPPER_EXEC=1`, and direct invocation warns when absent. + + **Answer:** The wrapper sets an advisory environment variable before execing the selected internal toolchain binary: + + ```text + BAML_WRAPPER_EXEC=1 + ``` + + Wrapper pass-through behavior: + + ```text + baml generate + ``` + + becomes approximately: + + ```text + BAML_WRAPPER_EXEC=1 \ + BAML_WRAPPER_VERSION= \ + BAML_WRAPPER_RESOLVED_TOOLCHAIN= \ + ~/.baml/toolchains//bin/baml-cli generate + ``` + + Only `BAML_WRAPPER_EXEC=1` is required for warning suppression. `BAML_WRAPPER_VERSION` and `BAML_WRAPPER_RESOLVED_TOOLCHAIN` are optional diagnostic/logging aids and must not become version authorities. + + `baml-cli` startup behavior: + + - If `BAML_WRAPPER_EXEC=1`, do not warn. + - If `BAML_CLI_ALLOW_DIRECT=1`, do not warn. This escape hatch is for tests, packaging smoke tests, and maintainers debugging the internal binary directly. + - Otherwise, print a once-per-process warning to stderr: + + ```text + warning: using the internal BAML toolchain binary directly is not recommended. Use `baml` instead. + ``` + + Use stderr, not stdout. stdout may be machine-readable for commands like `--version`, `describe`, future JSON output, or scripts. With stderr, `baml-cli --version` stdout remains exact. + + This is advisory, not security. A user can set `BAML_WRAPPER_EXEC=1`; that is acceptable because the goal is to guide users toward the wrapper, not enforce a trust boundary. + + Direct project/toolchain mismatch detection is separate and best-effort. If `baml-cli` is invoked directly inside a project whose nearest `baml.toml [toolchain]` selects a different concrete version/channel than the binary's own `baml_version::CANONICAL_VERSION`, it may print an additional local warning: + + ```text + warning: this project selects BAML 0.11.0, but this internal toolchain is 0.11.1-nightly.20260602.a. Run commands through `baml` so the wrapper can select the right toolchain. + ``` + + That mismatch check must not hit the network, read wrapper `state.toml`, install toolchains, or attempt to repair anything. It only reads local project metadata and compares against the current binary's stamped toolchain version. + + Implementation locations: + + - Wrapper: in the pass-through exec path in the new `baml_language/crates/baml/` crate, immediately before replacing the process with the selected `baml-cli`. + - CLI: early in the `baml-cli` entrypoint, before command dispatch, likely in `baml_language/crates/baml_cli/src/main.rs` or the equivalent top-level command bootstrap. + - Version comparison: use the stamped BAML language version (`baml_version::CANONICAL_VERSION`), not `CARGO_PKG_VERSION`. + + Tests: + + - Direct `baml-cli --version` without `BAML_WRAPPER_EXEC` prints the warning to stderr and exact version output to stdout. + - Direct `baml-cli --version` with `BAML_WRAPPER_EXEC=1` prints no warning. + - Direct `baml-cli --version` with `BAML_CLI_ALLOW_DIRECT=1` prints no warning. + - Wrapper pass-through command sets `BAML_WRAPPER_EXEC=1`, so invoking `baml --version` or `baml generate` through the wrapper does not show the direct-binary warning. + - Machine-readable stdout tests assert the warning never appears on stdout. + - Optional mismatch test: create a temporary project with `baml.toml [toolchain]` selecting a different version and invoke `baml-cli` directly; assert the mismatch warning appears without any network access. + +10. **Installer path/profile behavior is too light** + [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:491) + Flags are listed, but not which shell profiles are edited, how PATH modification is skipped/detected, whether writes are atomic, how Windows PATH mutation works, or what exit codes mean. This is less architectural, but installers are sharp edges. + + **Answer:** Curl/PowerShell installers are user-scoped BAML installers. They may install/update the wrapper under `BAML_HOME`, optionally bootstrap a requested toolchain, and optionally add `BAML_HOME/bin` to the user's PATH. They must not use `sudo`, write system directories, mutate Homebrew/AUR/package-manager paths, install IDE extensions automatically, or write outside `BAML_HOME` except for user profile/PATH configuration. + + Defaults: + + ```text + Unix BAML_HOME: $HOME/.baml + Windows BAML_HOME: %USERPROFILE%\.baml + Unix wrapper: $BAML_HOME/bin/baml + Windows wrapper: %USERPROFILE%\.baml\bin\baml.exe + ``` + + Required flags: + + ```text + --channel # default canary + --version # exact version; wins over --channel + --wrapper-only # install/update wrapper but skip toolchain bootstrap + --no-modify-path # do not edit shell profile / user PATH + --yes # disable prompts / accept defaults; explicit consent for profile/PATH edits in piped/non-interactive installs + --help + ``` + + Flag rules: + + - `--version` wins over `--channel`. + - `--wrapper-only` skips toolchain bootstrap. + - `--no-modify-path` disables profile/PATH edits even when `--yes` is supplied. + - `--yes` disables prompts, accepts defaults, and is the explicit consent signal for profile/PATH edits in piped or otherwise non-interactive installs. + - Do not add `--modify-path` in v1. If PATH modification is the default in interactive installs, the only necessary opposite flag is `--no-modify-path`. + - Piped installs such as `curl ... | sh -s` are treated as non-prompting because stdin is the script stream. They install wrapper + default canary toolchain, do not edit shell profiles by default, and print PATH instructions. + + Unix `install.sh` flow: + + 1. Resolve `BAML_HOME`. + 2. Detect platform/target triple. + 3. Fetch `manifest/v1/wrapper.json`. + 4. Download wrapper archive to a temp directory. + 5. Verify sha256 before extraction/execution. + 6. Extract into a temp directory. + 7. Validate archive layout contains the expected wrapper executable at `bin/baml` and does not contain `baml-cli`, `baml-pack-host`, VSIX, absolute paths, `..` paths, or unsafe symlinks. + 8. `chmod +x` the wrapper. + 9. Atomically replace `$BAML_HOME/bin/baml` (write/extract to temp path in same directory, fsync where practical, rename into place). + 10. Optionally update PATH. + 11. Unless `--wrapper-only`, bootstrap the requested toolchain with: + + ```bash + "$BAML_HOME/bin/baml" toolchain use + ``` + + Do not run both `toolchain install` and `toolchain use`; point 3 defines `use` as "install if missing, then select". + + PATH behavior on Unix: + + - Installer writes a generated env file at: + + ```text + $BAML_HOME/env + ``` + + - File contents: + + ```sh + export BAML_HOME="$HOME/.baml" + case ":$PATH:" in + *":$BAML_HOME/bin:"*) ;; + *) export PATH="$BAML_HOME/bin:$PATH" ;; + esac + ``` + + - Shell profile files only source that env file: + + ```sh + . "$HOME/.baml/env" + ``` + + - Profile edits are idempotent: if the exact source line already exists, do not add another. + - If `BAML_HOME/bin` is already on PATH, do not modify profile files unless `--yes` is explicitly supplied and the source line is missing. + - Interactive install: update PATH by default unless `--no-modify-path`. + - Piped/non-interactive install without `--yes`: do not edit profile files; print manual PATH instructions. + - Piped/non-interactive install with `--yes`: edit profile files by default unless `--no-modify-path` is also supplied. + - CI/Docker examples should use `--no-modify-path` and set PATH explicitly. + + Explicit piped install behavior: + + ```bash + curl -fsSL https://pkg.boundaryml.com/install.sh | sh -s + ``` + + This installs/updates the wrapper, bootstraps the default canary toolchain with `baml toolchain use canary`, does not edit shell profiles, and prints: + + ```text + BAML installed at ~/.baml/bin/baml + + Add BAML to your PATH by adding this to your shell profile: + + . "$HOME/.baml/env" + + Or run for this shell session: + + export PATH="$HOME/.baml/bin:$PATH" + ``` + + ```bash + curl -fsSL https://pkg.boundaryml.com/install.sh | sh -s -- --yes + ``` + + This installs/updates the wrapper, bootstraps the default canary toolchain, and edits shell profile/PATH configuration by default. + + ```bash + curl -fsSL https://pkg.boundaryml.com/install.sh | sh -s -- --yes --no-modify-path + ``` + + This accepts defaults but still skips profile/PATH edits. + + Unix shell targets: + + - `zsh`: add source line to `~/.zshrc`; on macOS, also add to `~/.zprofile` if that file already exists. + - `bash`: add source line to `~/.bashrc`; on macOS, also add to `~/.bash_profile` if that file already exists. + - `fish`: write `~/.config/fish/conf.d/baml.fish` instead of editing generic shell files. The fish file should set `BAML_HOME` and prepend `$BAML_HOME/bin` only if missing. + - Unknown shell: do not edit profiles; print manual PATH instructions. + + Windows `install.ps1` flow: + + 1. Resolve `BAML_HOME` from env or default `%USERPROFILE%\.baml`. + 2. Detect architecture. + 3. Fetch `manifest/v1/wrapper.json`. + 4. Download wrapper archive to a temp directory. + 5. Verify sha256 before extraction/execution. + 6. Extract and validate archive layout contains `bin/baml.exe` only for the wrapper payload. + 7. Replace `%USERPROFILE%\.baml\bin\baml.exe` safely. If the executable is in use, use a safe replace/deferred replace path and print the action taken. + 8. Optionally update the user's PATH. + 9. Unless `--WrapperOnly`, bootstrap with: + + ```powershell + & "$env:BAML_HOME\bin\baml.exe" toolchain use + ``` + + Windows PATH behavior: + + - Update user PATH only, never machine PATH. + - Do not require Administrator. + - Use `[Environment]::SetEnvironmentVariable("Path", ..., "User")`. + - Avoid duplicate entries. + - If PATH changes, tell the user to restart the terminal. + - `--NoModifyPath` skips PATH mutation. + - Non-interactive mode does not mutate user PATH unless `--Yes` is passed. + + Stable exit codes: + + ```text + 0 success + 1 general failure + 2 unsupported platform + 3 download/network failure + 4 checksum verification failure + 5 archive validation/extraction failure + 6 PATH/profile update failure + 7 toolchain bootstrap failure + ``` + + Failure behavior: + + - If wrapper install succeeds but PATH update fails, leave the wrapper installed, print manual PATH instructions, and exit `6`. + - If wrapper install succeeds but toolchain bootstrap fails, leave the wrapper installed, print the exact `baml toolchain use ` command to retry, and exit `7`. + - If checksum or archive validation fails, do not replace the existing wrapper. + - Re-running the installer is idempotent: it refreshes the wrapper from `wrapper.json`, repairs PATH/env file entries if requested, and only bootstraps the requested toolchain when `--wrapper-only` is not passed. + +11. **`baml ide install` fallback is risky as stated** + [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:970) + “Fall back to dropping into the editor’s extensions directory and unzipping” needs exact per-editor directories, extension IDs, version replacement behavior, and whether that is actually supported. I’d either specify it tightly or remove fallback in v1. + + **Answer:** Remove the manual extension-directory unzip fallback from v1. `baml ide install` installs only through supported editor CLIs. If no supported editor CLI is available, print the VSIX path and the exact manual command the user can run. + + V1 supports only Cursor and VS Code out of the box: + + ```text + baml ide install + baml ide install --cursor + baml ide install --code + ``` + + No v1 flags for `--windsurf`, `--all`, `--editor`, `--force`, or `--dry-run`. + + Behavior: + + - Resolve the active selected toolchain. + - Find the VSIX at: + + ```text + /assets/baml-vscode.vsix + ``` + + - `--cursor`: run `cursor --install-extension `. + - `--code`: run `code --install-extension `. + - No flag: + - if only `cursor` is found on PATH, install to Cursor; + - if only `code` is found on PATH, install to VS Code; + - if both are found and the terminal is interactive, prompt the user to choose; + - if both are found and the terminal is non-interactive, error and require `--cursor` or `--code`; + - if neither is found, error with manual install commands. + + Implementation may internally pass editor-specific flags needed for update/reinstall, such as `--force` if the editor CLI supports it, but those are not user-facing BAML flags in v1. + + Failure message shape: + + ```text + No supported editor CLI found. + + Install manually: + code --install-extension /assets/baml-vscode.vsix + + Or choose an editor if its CLI is installed: + baml ide install --cursor + baml ide install --code + ``` + + Implementation locations: + + - Add the command variant in `baml_language/crates/baml_cli/src/commands.rs`. + - Implement behavior in `baml_language/crates/baml_cli/src/ide_command.rs`. + - The wrapper does not own this command. `baml ide install --cursor` is wrapper pass-through to the selected `baml-cli ide install --cursor`. + + Tests: + + - Finds VSIX relative to active toolchain root. + - `--cursor` invokes `cursor --install-extension `. + - `--code` invokes `code --install-extension `. + - No flag + only Cursor detected installs to Cursor. + - No flag + only VS Code detected installs to VS Code. + - No flag + both detected + non-interactive errors and requires a flag. + - No supported CLI detected prints manual install commands and VSIX path. + - No test or implementation path manually unzips into editor extension directories. + +12. **Homebrew/AUR publishing details are abstract** + [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:168), [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:936) + The desired package shape is clear, but credentials, target repos, dispatch vs direct commit, AUR update mechanism, and source/archive URL conventions are not as concrete as the GitHub/S3/PyPI parts. + + **Answer:** Homebrew and AUR publishing are wrapper-release-only jobs. They never run for `baml-toolchain` nightly/canary releases, never track BAML language versions, never bootstrap toolchains during package install/upgrade, and never write user-scoped `~/.baml` state. + + Homebrew: + + - Repository: `BoundaryML/homebrew-tap`. + - Formula path: `Formula/baml.rb`. + - Package contents: installs only `bin/baml`. + - Formula version: wrapper version from `baml_language/crates/baml/Cargo.toml [package].version`. + - Formula source/archive URL: + + ```text + https://github.com/BoundaryML/baml/releases/download/baml-wrapper-/baml-wrapper--.tar.gz + ``` + + - Formula must not install, symlink, or reference `baml-cli`, `baml-pack-host`, or the VSIX. + - Formula must not run `baml toolchain install`, `baml toolchain use`, or any toolchain bootstrap in `post_install`. + - Publish method: release graph commits directly to `BoundaryML/homebrew-tap` using the existing `HOMEBREW_BAML_DISPATCH_TOKEN`, kept for compatibility with the current secret naming. The token needs contents write access to `BoundaryML/homebrew-tap`. + - Caveats should print: + + ```text + BAML wrapper installed. + + To install and select the current canary language toolchain: + baml toolchain use canary + + To use nightly: + baml toolchain use nightly + + IDE extension setup is explicit: + baml ide install --cursor + ``` + + AUR: + + - Packages: `baml` and `baml-bin`. + - `baml-bin` downloads the prebuilt `baml-wrapper--.tar.gz` wrapper archive from GitHub Releases and installs only `bin/baml`. + - `baml` source-builds the wrapper from the `baml-wrapper-` source archive/tag, builds only `--bin baml`, and installs only `/usr/bin/baml`. + - Both AUR package versions track the wrapper version only. + - No AUR nightly/canary package stream in v1. + - No AUR install hook writes `~/.baml` or runs toolchain install/use. + - AUR install messages should point users to `baml toolchain use canary`. + - Publish method: update the AUR package repositories over SSH, regenerate `.SRCINFO`, commit, and push. + - Required maintainer-configured AUR remotes: + + ```text + ssh://aur@aur.archlinux.org/baml.git + ssh://aur@aur.archlinux.org/baml-bin.git + ``` + + - Required CI secret: + + ```text + AUR_SSH_PRIVATE_KEY + ``` + + Publish guards for both Homebrew and AUR: + + ```text + wrapper_changed == true + dry_run == false + github.ref == refs/heads/canary + ``` + + A `baml-toolchain` release must not dispatch or run Homebrew/AUR publishing. A wrapper release may occur in the same release graph codebase, but it is a separate publish decision keyed only off wrapper version changes. + + Dry-run behavior: + + - Generate the Homebrew formula and AUR `PKGBUILD` / `.SRCINFO` files. + - Upload generated package files as workflow artifacts. + - Do not commit to `BoundaryML/homebrew-tap`. + - Do not push to AUR. + + Required checks before publishing: + + - Homebrew formula points at `baml-wrapper-`, not `baml-language-`. + - Homebrew formula version equals wrapper version. + - Homebrew formula installs only `bin/baml`. + - Homebrew formula has no `post_install` toolchain bootstrap. + - AUR `pkgver` equals wrapper version. + - AUR package sources point at wrapper release/source. + - AUR `PKGBUILD` installs only `baml`. + - AUR install hooks do not run `baml toolchain use` or `baml toolchain install`. + - Caveats/install messages include `baml toolchain use canary`. + +13. **Manifest target completeness has a subtle policy gap** + [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:358) + It defines required and best-effort targets, but not whether the publish workflow may publish a manifest missing a Tier 1 target due to a failed matrix job. I think Tier 1 failures should block publish, while best-effort failures can be omitted only with an explicit gate. + + **Answer:** Do not support optional/best-effort targets in v1. If a target is in the supported release matrix, it is required. Any target build, archive-layout, checksum, or smoke failure blocks the entire release. Maintainers can rerun the failed matrix job or rerun the workflow to recover. + + Release policy: + + - The release matrix is the required target set. + - A `baml-toolchain` release publishes only when every target in the matrix succeeds. + - A `baml-wrapper` release publishes only when every wrapper target in the matrix succeeds. + - There is no `optional_targets`, `best_effort_targets`, or `omitted_targets` concept in v1. + - If a platform is too flaky to block releases, remove it from the supported release matrix rather than marking it optional. + + Manifest publish validation: + + - The manifest's `artifacts` target set must exactly equal the release matrix target set for that product. + - Every artifact entry must include HTTPS `url` and lowercase 64-character hex `sha256`. + - Every included archive must have passed archive-layout validation. + - No publish job runs if any target is missing, failed, or lacks a checksum. + - `manifest/v1/version/.json` is not uploaded until the full target set is complete. + - `canary.json` / `nightly.json` pointers are not updated until the immutable per-version manifest is complete. + + Wrapper behavior for a missing target is defensive only. It should be treated as an incomplete/corrupt release manifest, not an expected optional-target case: + + ```text + error: BAML 0.11.0 manifest is missing artifact for target aarch64-pc-windows-msvc. + This release manifest is incomplete. Try `baml toolchain update`, or report this release. + ``` + + This keeps the release model simple: if a BAML version exists, all supported targets for that release exist. + +**Lower Priority** +14. **Docs phase is intentionally broad** + [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:1106) + “New page: install matrix...” is fine for a milestone, but not enough if this is meant to hand directly to a docs implementer. It needs page paths, audience, and required examples. + + **Answer:** These docs refer to new docs that must be created as part of the feature branch. There is no established public `baml_language` documentation home yet, so v1 should create in-repo implementation/user-flow docs under `TASK/docs/`. Public product-docs migration is a later product/docs decision and should not block the release-infra implementation. + + Create: + + ```text + TASK/docs/install.md + TASK/docs/release-maintainer.md + TASK/docs/toolchain-system.md + ``` + + `TASK/docs/install.md` + + - Audience: users, support, and people testing the new installer. + - Required contents: + - `brew install boundaryml/tap/baml` installs the wrapper only. + - Package-manager installs do not bootstrap toolchains. + - `curl ... | sh -s` installs the wrapper and default canary toolchain. + - Docker/CI install example using `--no-modify-path`. + - PATH behavior and `~/.baml/env`. + - `baml toolchain use canary`. + - `baml toolchain use nightly`. + - `baml toolchain install `. + - `baml toolchain update`. + - `baml self-update`. + - Package-manager wrapper upgrade behavior (`brew upgrade baml`, AUR upgrade). + - IDE setup: + + ```bash + baml ide install --cursor + baml ide install --code + ``` + + `TASK/docs/release-maintainer.md` + + - Audience: maintainers and implementation agents. + - Required contents: + - Release graph overview. + - Canary vs nightly model. + - How to bump canary. + - How nightly suffixes are chosen. + - Release workflow concurrency rule. + - Dry-run release workflow and `BAML_MANIFEST_BASE_URL`. + - Production publish guards. + - PyPI trusted-publisher portal pause. + - Homebrew/AUR wrapper release flow. + - Rerun/idempotency behavior. + - Rollback / mutable pointer repair flow. + + `TASK/docs/toolchain-system.md` + + - Audience: engineers. + - Required contents: + - Wrapper vs toolchain product boundary. + - `~/.baml` layout. + - `config.toml` user intent. + - `state.toml` active channel resolutions. + - manifest cache role. + - manifest schema. + - network/cache policy. + - `toolchain use` vs `toolchain install` semantics. + - VSIX/LSP/playground compatibility protocol. + - one-LSP-per-BAML-project-root model. + - direct `baml-cli` warning behavior. + - `baml pack` fetcher unification. + + These docs are part of the handoff and release-infra branch. They can be moved or rewritten for the public docs site later. + +15. **Marketplace/deprecation story is deferred but product-impacting** + [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:1136) + It is correctly marked as product decision, but if implementation depends on extension ID, Marketplace publisher, or coexistence behavior, that decision may need to move earlier. + + **Answer:** Marketplace publishing and old-extension deprecation remain deferred to Phase 4, but the new VSIX identity is not deferred. V1 ships a stable, distinct, toolchain-bundled VSIX installed explicitly by `baml ide install`; it is not published to the VS Code Marketplace in v1. + + Stable new VSIX identity: + + ```json + { + "publisher": "Boundary", + "name": "baml-language", + "displayName": "BAML", + "description": "Language support and playground for BAML." + } + ``` + + Stable extension ID: + + ```text + Boundary.baml-language + ``` + + The user-visible display name is simply `BAML`. The internal `name` is `baml-language` to avoid ambiguity with the existing Marketplace extension and to keep a distinct stable ID for the toolchain-bundled extension. + + V1 coexistence behavior: + + - The old Marketplace extension remains untouched. + - The new extension has a distinct extension ID. + - `baml ide install --cursor` and `baml ide install --code` install the new toolchain-bundled VSIX only. + - No old-extension detection. + - No warning if the old extension is also installed. + - No automatic disable/uninstall/migration. + - If a user has both extensions installed and sees duplicate behavior, docs/support guidance is to disable one manually. + + Deferred to Phase 4: + + - Publishing the new extension to the Marketplace. + - Deprecating or replacing the old Marketplace extension. + - Marketplace migration messaging. + - Any old-extension detection, warning, disable, uninstall, or migration behavior. diff --git a/TASK/PRIOR-ART.md b/TASK/PRIOR-ART.md new file mode 100644 index 00000000000..262acbaa0f6 --- /dev/null +++ b/TASK/PRIOR-ART.md @@ -0,0 +1,552 @@ +# Prior Art For BAML Release And Toolchain UX + +Research date: 2026-05-29. + +## Executive Summary + +The current architecture is directionally sound: a small `baml` wrapper, versioned toolchains under `~/.baml`, project pins in `baml.toml`, normal commands forwarded to the concrete CLI, and Python runtime packages controlled by Python package managers. + +The main correction is command surface. Prior art strongly suggests that top-level `install` and `update` should not mean "select a language toolchain" if BAML expects to grow a package/dependency manager. Rust avoids this by splitting `rustup toolchain install` from `cargo install` and `cargo update`. Python/uv separates `uv self update`, `uv python install`, `uv tool install`, `uv add`, and `uv sync`. Go makes toolchain updates a module/toolchain concern, while `go install pkg@version` installs executables. Bun and Deno show why single-binary tools must reserve words carefully: `bun install` and `bun update` are package manager verbs, while `bun upgrade` updates Bun itself; `deno install` installs a script/app while `deno upgrade` updates the runtime. + +Recommended BAML shape: + +```bash +baml toolchain install canary +baml toolchain install nightly +baml toolchain install 0.11.0 +baml toolchain use canary +baml toolchain use nightly +baml toolchain use 0.11.0 +baml toolchain update +baml toolchain list +baml toolchain uninstall 0.11.0 +baml self-update +baml ide install --cursor +``` + +Top-level `baml install` is reserved for future BAML project packages/dependencies. Top-level `baml update` is also reserved until package-management semantics exist. The documented toolchain update command is `baml toolchain update`. + +Normal commands such as `baml generate`, `baml run`, `baml describe`, `baml pack`, and `baml lsp` should remain pure pass-throughs and should not block on network after a suitable toolchain is installed. + +## Proposed BAML Command Surface + +| User intent | Recommended command | Semantics | +|---|---|---| +| Install/select canary toolchain globally | `baml toolchain use canary` | Resolves current canary, installs if missing, records active channel as `canary`. | +| Subscribe to nightly globally | `baml toolchain use nightly` | Resolves current nightly, installs if missing, records active channel as `nightly`. | +| Install but do not activate a version/channel | `baml toolchain install 0.11.0` / `canary` / `nightly` | Downloads and verifies a concrete toolchain. For a channel, resolves to the current concrete version. | +| Pin a project | `baml toolchain pin 0.11.0` or edit `baml.toml` | Writes `[toolchain] version = "0.11.0"` to the nearest project. Optional `pin nightly` writes channel if the team wants project-level channel following. | +| Update active channel | `baml toolchain update` | If active selector is `canary` or `nightly`, refresh channel manifest and advance to current head. If active selector is a concrete version, say it is pinned and suggest `baml toolchain use canary/nightly`. | +| Update wrapper itself | `baml self-update` | Replaces only the wrapper for curl/script installs. Refuses for Homebrew/AUR/uv/pipx-managed installs with manager-specific instructions. | +| Install IDE assets | `baml ide install` | User-facing command for IDE setup/update. It is forwarded by the wrapper to the selected toolchain payload. Common editors should support flags such as `--cursor`, `--code`, `--windsurf`, and `--all` to avoid an interactive prompt. | +| Future project package install | `baml install ` | Reserved for BAML packages/dependencies; do not spend this verb on toolchains. | +| Future project package update | `baml update` or `baml package update` | Reserve until package manager design is concrete. If ambiguous, prefer `baml package update`. | + +Why `toolchain` over `version`: `version` reads like information display or project-version bumping in several ecosystems (`uv version` reads/updates the project version). `toolchain` is explicit, matches Rust and Go vocabulary, and makes future package-manager verbs unambiguous. + +Compatibility option: + +```bash +baml version use nightly +baml version update +``` + +This is acceptable only as an alias if the team strongly prefers "version" wording in docs. The primary help text should use `toolchain` to avoid confusing toolchain selection with package or project version changes. + +## Prior-Art Matrix + +| Ecosystem | Tool self-update | Runtime/toolchain update | Project dependency update | Project pin | Channel model | Offline/cache behavior | IDE/LSP pattern | BAML implication | +|---|---|---|---|---|---|---|---|---| +| Rust | `rustup self update`; `rustup update` also checks self-update in rustup-managed installs | `rustup toolchain install`, `rustup update`, `rustup default` | `cargo update` updates `Cargo.lock`; `cargo install` installs executables | `rust-toolchain.toml` plus command/env/directory overrides | `stable`, `beta`, `nightly`, dated nightlies | Cargo has `--offline`; rustup installs/updates explicitly sync manifests | `rust-analyzer` is separate from compiler/cargo but usually finds project toolchain | Strongest model: namespace toolchains away from package manager verbs. | +| Bun | `bun upgrade`; Homebrew users are told to use `brew upgrade bun` | Single binary, so upgrade replaces runtime/PM/test runner together | `bun install`, `bun add`, `bun update` | `package.json` + `bun.lock` | `bun upgrade --canary`, `--stable` | Global cache, frozen lockfile, dependency lifecycle restrictions | No separate ecosystem LSP model central to Bun | Do not use `install`/`update` for BAML toolchains if future packages exist. | +| mise/asdf | `mise self-update` unavailable when package-manager-installed; asdf says update with original install method | `mise install`, `mise use`; `asdf install`, `asdf set` | Delegated to backend tools | `mise.toml`, `.tool-versions`, global config | Uses exact, fuzzy, `latest`, backend-specific channels | Installs into local data dir; normal shim execution resolves local installs | Shims are common for IDEs/CI | `install` is safe only because mise is explicitly a tool manager, not BAML's language/package CLI. | +| Node/npm/Corepack/Volta | npm can update npm; Volta has its own installer; Corepack manages package-manager shims | nvm/Volta install Node versions; Corepack prepares package managers | `npm install`, `npm update`, `pnpm update`, `yarn add` | `package.json` `packageManager`, Volta section, lockfiles | Node has LTS/current; package managers have dist-tags | npm/npx cache; prompts before ephemeral installs | Language service delegated to editor/extensions | Node shows user confusion when runtime managers and package managers share vocabulary. | +| Python/uv/pipx | `uv self update`; pipx itself updated by pipx/pip/brew/etc. | `uv python install/upgrade`; pyenv for Python versions | `uv add`, `uv sync`, `uv lock`, `pip install`, `pip install -U` | `pyproject.toml`, lockfiles, `.python-version` | Pre/dev releases opt-in; exact versions common | uv aggressively caches; `uvx` env is disposable cache, `uv tool install` env is persistent | Python extension bundles/delegates servers; tools exposed through console scripts | PyPI `baml` command can be a launcher, but `import baml_core` must remain the installed wheel. No wheel self-mutation. | +| Go | Install Go by package/installer; no in-command self-update norm | Go 1.21+ `go` can auto-switch/download toolchain based on `go.mod`/`go.work`; `go get go@latest` updates toolchain dependency | `go get` changes module requirements; `go mod tidy`; `go install pkg@version` installs executables | `go` and `toolchain` lines in `go.mod`/`go.work` | Stable/release-candidate toolchain names | Module cache + `go.sum` verification; vendor mode avoids network | `gopls` is official language server installed as a Go tool | Toolchain mismatch should be explicit and actionable; automatic downloads are acceptable only in commands that opted into mutation. | +| Deno | `deno upgrade` updates executable; supports version/channel/nightly and checksum | Single runtime binary; `upgrade` is runtime-level | `deno add`, `deno update`; `deno install` installs script/app binaries | `deno.json`, lockfile | `deno upgrade nightly`, alpha/beta/rc | Cached downloads, lockfile integrity, checksum flag | `deno lsp` is built into the runtime; VS Code extension delegates to Deno | Use `upgrade`/`self-update` for wrapper, not `update`; keep checksums first-class. | +| Homebrew/AUR | Package manager owns wrapper updates | Formula/pkg upgrades should not silently mutate user state beyond package files | Package manager is external | Formula/pkgbuild version and checksum | Stable formula versions; AUR can offer `-bin` or VCS variants | Bottles/source archives have checksums; post-install caveats are normal | Caveats can tell users how to run IDE setup | Managed wrappers must refuse `self-update` and point at manager commands. | +| VS Code extensions | Marketplace owns extension updates | Extension may bundle, download, or launch local server | N/A | User/workspace settings | Extension version channel via marketplace/prerelease | Extension host should avoid unexpected heavy work on activation | LSP runs as a separate process; VSIX can be installed from file | Prefer VSIX as a thin client launching local `baml`/toolchain, with clear mismatch diagnostics. | +| GitHub Releases | N/A | Hosts versioned assets and release notes | N/A | Tags/releases | Canary and nightly/prerelease tags possible | Assets are durable URLs; not a directory/index by themselves | N/A | Use GitHub Releases as immutable storage/changelog, while `pkg.boundaryml.com` is the machine-facing directory. | + +## Detailed Findings + +### Rust + +Rust has the cleanest split for BAML's desired architecture: + +- `rustup` manages Rust toolchains and channels. +- `cargo` manages project dependencies, lockfiles, package publishing, executable installs, and builds. +- `rust-toolchain.toml` pins a project toolchain in source control. +- `cargo +nightly test` is an explicit one-command override. + +`rustup` resolves toolchains by a documented precedence: command shorthand, environment variable, directory override, `rust-toolchain.toml`, then default. That maps directly onto BAML's proposed precedence of `BAML_VERSION`, nearest `baml.toml`, user config, then canary. + +Rust's important naming lesson is that `install` is not globally overloaded. The full commands are `rustup toolchain install nightly` and `cargo install ripgrep`. Users learn which product owns which surface. + +BAML decision: + +- Use a namespaced toolchain command, ideally `baml toolchain install/use/update`. +- Keep `baml install` free for packages. +- Make `baml.toml [toolchain]` source-control-friendly and explicit, like `rust-toolchain.toml`. +- Add a command override if needed later, e.g. `BAML_VERSION=nightly baml generate` before inventing `baml +nightly generate`. + +### Bun + +Bun is a counterexample because it intentionally combines runtime, package manager, test runner, bundler, and installer in one binary. Its command surface is correspondingly careful: + +- `bun install` installs project dependencies and writes `bun.lock`. +- `bun update` updates dependencies. +- `bun add` changes dependencies. +- `bun upgrade` updates the Bun binary itself. +- `bunx` runs package executables, checking local packages before falling back to package install/cache. +- For Homebrew/Scoop installs, Bun tells users to use the package manager instead of self-upgrading. + +Bun also has nightly via `bun upgrade --nightly`, but that changes the single Bun binary. BAML has two products, so copying Bun literally would be wrong. + +BAML decision: + +- Do not use `baml install nightly` if `baml install ` is likely later. +- Do not use `baml update` for toolchains without a namespace unless the team intentionally accepts future ambiguity. +- Use `baml self-update` only for the wrapper, and refuse under managed installs. + +### mise / asdf + +mise and asdf are useful because they are pure tool managers: + +- `mise install node@20` installs a tool version but does not necessarily activate it. +- `mise use node@20` installs and records activation in `mise.toml` or global config. +- `mise self-update` updates mise itself and is unavailable when installed by a package manager. +- asdf separates `asdf install ` from `asdf set `, with `.tool-versions` as the project pin. + +This supports a BAML distinction between "download a concrete toolchain" and "select a default channel/version." + +BAML decision: + +- `baml toolchain install ` should install but not necessarily select. +- `baml toolchain use ` should install if missing and record the selected selector. +- For channels, store both the selector (`nightly`) and the resolved version (`0.11.1-nightly.20260529.a`) so offline commands know what to run and status commands can explain drift. + +### Node + +Node's ecosystem splits responsibility in practice, but users often experience it as one large surface: + +- nvm/Volta install and select Node versions. +- npm/pnpm/yarn install and update packages. +- npx/npm exec and pnpm dlx run package binaries from local dependencies or cache. +- Corepack provides package-manager shims keyed by project metadata. +- Volta pins runtime and package-manager versions in `package.json`. + +The warning sign is user confusion around "install node", "npm install", "npm update", "npx", package manager versions, and project lockfiles. BAML can avoid that by not making the same verb mean toolchain today and packages tomorrow. + +BAML decision: + +- Avoid `baml version` as the primary namespace because `version` can also mean "project/package version" in tools like uv and npm. +- Prefer `baml toolchain ...` for the language payload and reserve `baml package ...` or top-level package verbs for dependency management. + +### Python / uv / pipx + +Python is directly relevant to `uvx baml generate`, `uv run baml generate`, and `import baml_core`. + +Important norms: + +- Installed distributions expose commands through `console_scripts`; installers create wrappers in the environment's script directory. +- `uvx` is `uv tool run`: it runs a Python package command in an isolated cached environment. `uv tool install` creates a persistent isolated environment. `uv run` runs inside a project environment and may sync/update that environment. +- `uvx` may use cached tool environments, but those environments are disposable and should not be manually mutated. +- `pipx` installs Python app packages into isolated virtual environments and exposes their apps on PATH; `pipx run` uses temporary environments. +- PEP 440 developmental releases are `.devN`, are treated as pre-releases, and SemVer hyphen prereleases cannot be used as public PyPI versions without translation. + +BAML decision: + +- Publish separate conceptual packages even if distribution names need compatibility: + - `baml_core`: importable runtime library. It must never self-update or install toolchains as an import side effect. + - `baml`: console-script launcher package. It may provide `baml = baml_launcher:main`, resolve BAML toolchains, and run CLI commands. +- If packaging constraints require one PyPI project for both, still treat imports and CLI as separate surfaces: importing `baml_core` only imports the installed wheel; invoking `baml` can launch/download CLI toolchains only for explicit CLI commands. +- `uvx baml generate` should run the latest PyPI `baml` launcher chosen by uv unless pinned (`uvx baml==0.10.0 generate`). The launcher may download a BAML toolchain into `~/.baml`, but it must not mutate the Python environment or upgrade `baml_core`. +- `uv run baml generate` should use the project environment's installed `baml` console script. If the project does not include it, uv's normal rules apply; BAML should not bypass uv's environment semantics. +- If `pip install baml==0.10.0` but `baml.toml` expects `0.9.x`, the launcher should use the project/toolchain selector and warn only if the launcher protocol is too old to understand the manifest. It should not rewrite the Python package. +- Generated code should embed the generator/toolchain version and the expected `baml_core` compatibility range. On import or first runtime call, if installed `baml_core` is incompatible, raise a loud diagnostic with exact installed, generated, and expected versions plus package-manager commands such as `uv add baml_core==...`, `pip install baml_core==...`, or `uv sync`. + +This is the main fix for the "current version conflict class": do not silently synchronize Python packages and CLI toolchains. Detect and explain. + +### Go + +Go 1.21+ treats the Go toolchain like a dependency of the module/workspace: + +- `go` and `toolchain` lines in `go.mod`/`go.work` guide toolchain selection. +- The `go` command may find a suitable toolchain on PATH or download/cache one. +- If automatic switching is disabled, Go refuses to run when the project requires a newer version. +- `go get go@latest` updates the module's toolchain requirement. +- `go install pkg@version` installs executable packages without modifying the current module. +- Module downloads are verified through `go.sum` and the checksum database for public modules. + +BAML decision: + +- Project pinning belongs in `baml.toml`, but commands that mutate that pin should be explicit (`baml toolchain pin`, not `baml generate`). +- A normal `baml generate` may auto-install a missing exact pinned toolchain only if the team accepts Go-like auto-download. If implemented, it must print that it is downloading because the project requires that version. If the team wants stricter behavior, fail and suggest `baml toolchain install`. +- If a concrete pinned version is unavailable offline, fail with a local-cache message; do not silently fall forward to canary/nightly. + +### Deno + +Deno's single-binary model uses: + +- `deno upgrade` for updating the runtime executable, including specific versions and channels such as nightly. +- `deno install` for installing scripts/app commands. +- `deno update` for dependencies. +- cached runtime downloads and checksum verification. +- a built-in `deno lsp` rather than a separate package-managed server. + +BAML decision: + +- `upgrade`/`self-update` should mean the installed wrapper, not the active language toolchain. +- `install` should be left available for future installable BAML artifacts/packages, matching Deno's use of `deno install` for user apps, not runtime channels. +- Per-version toolchain archives should have SHA-256 in manifests from day one. + +### Homebrew / AUR + +Package-manager norms: + +- Homebrew formulae install into a managed Cellar and symlink into the prefix. Formulae are versioned package definitions with checksums. +- Homebrew caveats are a standard way to show post-install instructions. +- Homebrew package upgrades should be controlled by `brew upgrade`; tools installed by Homebrew should avoid replacing their own binaries. +- Homebrew generally avoids repackaging language-specific dependencies when those ecosystems already manage them directly. +- Arch/AUR packages are built from PKGBUILD metadata and should install files owned by the package manager, not surprise-mutate user state during every upgrade. +- PKGBUILD sources and checksums are part of the package recipe; VCS-style packages have separate naming/versioning conventions. That supports separate `baml` and `baml-bin` package templates if BAML wants source-built and binary-wrapper variants. + +BAML decision: + +- Homebrew/AUR should package only `baml-wrapper`. +- First install may bootstrap canary toolchain if implemented as an explicit, idempotent wrapper-owned step, but package upgrades must not reset channel/config or reinstall user-scoped toolchains. +- Prefer caveats that say: + +```text +BAML installed the wrapper. To install or update the language toolchain: + baml toolchain use canary + baml toolchain update + +To install editor support: + baml ide install --cursor +``` + +- If managed wrapper is too old: + +```text +This baml wrapper is too old to read manifest schema v2. +It was installed by Homebrew. Update it with: + brew upgrade baml +Then rerun: + baml toolchain update +``` + +Use equivalent messages for AUR, uv tool, pipx, and curl installs. + +### VS Code Extensions And LSP + +VS Code's LSP model is a client extension launching a language server in a separate process. The language server can be implemented in any language. Extensions commonly choose among: + +- bundle a server, +- download a server, +- locate a server on PATH, +- use a user/workspace setting for the server path. + +Rust Analyzer's current extension model is relevant even without copying its exact implementation: the extension is a client, the server is versioned independently, and project Rust version is still governed by rustup/toolchain selection. Go's `gopls` is an official language server installed as a tool. Deno exposes `deno lsp` from the runtime. + +BAML decision: + +- Public command should be `baml ide install`, forwarded by the wrapper to the selected toolchain payload. Do not add `baml setup-ide`. +- Toolchain install/update may recommend IDE setup/update, but must not silently install editor extensions. +- The VSIX should be platform-neutral and launch the configured `baml` wrapper from PATH or a user setting. It should not bundle platform-specific `baml-cli`. +- The extension should show mismatches in the editor when the project-pinned toolchain, active wrapper-selected toolchain, and VSIX protocol expectations disagree. The compatibility gate is explicit LSP/playground protocol ranges and capability flags, not exact BAML semver equality. +- LSP compatibility metadata should ride on the existing LSP `initialize` result. Playground compatibility should be checked only when the playground WebSocket connects. Do not add activation-time network checks, extra `baml` process spawns, or VSIX-owned toolchain downloads. + +### GitHub Releases + +GitHub Releases are a good asset store and maintainer-facing changelog: + +- Releases are associated with tags. +- Release assets are easy for humans and automation to inspect. +- Release notes can be generated and edited. + +They are not a complete channel directory. Mutable "latest canary" and "latest nightly" pointers are better represented by small manifest JSON files on `pkg.boundaryml.com`. + +BAML decision: + +- Create GitHub Releases for every canary toolchain. +- Create GitHub prereleases for nightly only if maintainers need the audit trail and GitHub asset UI. This is reasonable, but the user-facing channel should remain `pkg.boundaryml.com/manifest/v1/nightly.json`. +- `pkg.boundaryml.com` should be the directory/API: wrapper manifest, channel pointers, per-version metadata, install scripts. +- GitHub Releases should be storage/changelog infrastructure, not the normal end-user install surface. + +## Direct Implications For BAML + +### 1. What `baml install` Means + +Reserve it for future project packages/dependencies. Toolchain/channel selection should move under `baml toolchain`. + +Using `baml install nightly` for toolchains would force the future package manager into awkward alternatives such as `baml package install`, `baml add`, or `baml deps add`. The plan avoids that now. + +### 2. How `baml update` Is Reserved + +Do not define top-level `baml update` as the primary toolchain command. Use: + +```bash +baml toolchain update # active channel/toolchain +baml self-update # wrapper binary +baml package update # future package/dependency update +``` + +If a temporary top-level alias is ever added for v1 compatibility, document it as: + +```text +Alias for: baml toolchain update +Updates the active BAML language toolchain. Does not update the wrapper or project packages. +``` + +This alias should print exactly what it updated: + +```text +Updated nightly: 0.11.1-nightly.20260528.b -> 0.11.1-nightly.20260529.a +Wrapper unchanged. To update the wrapper, run baml self-update. +``` + +### 3. How should nightly channel resolution work? + +Installing/using a channel should do both: + +- record the selector (`nightly`) as the user's active channel; +- resolve and install the current concrete nightly version at that moment. + +Normal commands should not do blocking network lookups. They should use the recorded/resolved local toolchain. Channel pointers refresh only on explicit commands: + +```bash +baml toolchain update +baml toolchain use nightly +baml toolchain install nightly +``` + +Offline behavior: + +- If the required local toolchain exists, run it. +- If the active selector is `nightly` but the network is unavailable during `toolchain update`, keep the current installed nightly and report that update could not check the remote. +- If a project pins an exact missing version and the network is unavailable, fail with the exact missing version and suggest retrying `baml toolchain install ` when online. + +Status wording: + +```text +Active channel: nightly +Installed version: 0.11.1-nightly.20260528.b +Latest known remote: 0.11.1-nightly.20260529.a +Run `baml toolchain update` to advance. +``` + +### 4. What should happen for `uvx baml generate` and `uv run baml generate`? + +`uvx baml generate` runs a Python-distributed console script in an isolated uv tool environment. That console script should behave as a BAML launcher for CLI commands. It may install/download BAML toolchains in `~/.baml` only as part of explicit CLI execution. It must not mutate the uv tool environment or upgrade `baml_core`. + +`uv run baml generate` runs inside a project Python environment. It should respect whatever `baml` package version the project resolved. If the project environment contains `baml_core`, it remains governed by the project lockfile/package manager. + +`import baml_core` is never a launcher and never a self-updater. + +Mismatch policy: + +- CLI/toolchain versus project pin mismatch: wrapper selects project pin; if unsupported by wrapper, fail with wrapper update instructions. +- Generated code versus `baml_core` mismatch: runtime error with installed/generated/expected versions and package-manager command suggestions. +- PyPI launcher version versus toolchain version mismatch: allowed if launcher protocol supports the manifest; otherwise fail with actionable update command. + +### 5. How should IDE install/update work? + +Use `baml ide install` as the user-facing command. The command is implemented by the selected toolchain payload and exposed through wrapper pass-through. Support common editor flags such as `--cursor`, `--code`, `--windsurf`, and `--all`. + +Toolchain install/update should run the same setup flow only when interactive, or print the follow-up command when not interactive. The flow must be idempotent. + +The VSIX should launch local `baml lsp` through the wrapper. This makes IDE behavior match the CLI's project-pin resolution. The VSIX should not bundle platform-specific CLIs. + +VSIX/toolchain compatibility is by protocol range, not by exact BAML version. The VSIX passes supported LSP/playground protocol ranges during LSP initialize, the LSP returns its current protocol metadata, and the playground validates its own protocol lazily over the WebSocket startup path. This avoids the old extension-owned "download exact CLI and restart" loop. + +### 6. How should package-manager installs behave? + +Package managers should install the wrapper only. First install may offer or run canary bootstrap if it is: + +- idempotent; +- skipped when `~/.baml/config.toml` or installed toolchains already exist; +- non-resetting on wrapper upgrades; +- easy to disable in non-interactive installs. + +Wrapper `self-update` must be refused for managed installs. The refusal should name the detected manager and command. + +### 7. How should GitHub Releases be used? + +Use GitHub Releases for immutable assets, checksums, provenance attachments when available, and maintainer-facing notes. Use `pkg.boundaryml.com` manifests for user-facing channel resolution and install/update directory lookups. + +Canary and nightly can both have GitHub release pages, but nightly releases should be marked prerelease and should not be the primary user documentation path. Release notes should live in both generated GitHub releases and a human-readable changelog page for canary releases. + +### 8. What security and reproducibility expectations are standard? + +Minimum v1 expectations: + +- immutable per-version archives; +- SHA-256 for every archive in the per-version manifest; +- manifest schema version; +- wrapper min-version requirement in manifests; +- atomic install into temp dir followed by rename; +- executable bit and expected binary names verified before activation; +- archive path traversal checks before extraction; +- idempotent reruns; +- mutable channel manifests that point to immutable per-version manifests; +- local cache that never runs a partially extracted toolchain. + +Strongly recommended v1.1 or v1 if feasible: + +- Sigstore/GitHub artifact attestation or SLSA provenance for release artifacts; +- manifest signatures or signed checksums; +- transparent provenance documentation for package-manager formulas. + +`pkg.boundaryml.com` as directory plus GitHub Releases as storage is a reasonable split. Do not execute any downloaded toolchain until checksum verification, archive extraction validation, binary presence validation, and basic `baml-cli --version` sanity check pass. + +## Specific Risks In The Current Plan + +1. Top-level `baml install nightly` spends the most obvious future package-manager verb on toolchain selection. + +2. Top-level `baml update` will be ambiguous once BAML has packages: users will not know if it updates wrapper, toolchain, dependencies, generated code, or indexes. + +3. `baml version ...` is better than `install`, but still ambiguous because many CLIs use `version` for project/package versions. `toolchain` is more explicit. + +4. Python launcher and Python runtime are easy to conflate. If the PyPI `baml` package both exposes `baml` and contains `baml_core`, users may reasonably expect `pip install baml==X` to control imports and CLI behavior together. The plan must document the boundary. + +5. IDE setup must stay discoverable even though it lives in the selected toolchain payload. Use `baml ide install` in user docs and add common editor flags so users can avoid prompts. + +6. Package-manager post-install bootstrap can become surprising if it runs on upgrades or changes existing channel state. The plan needs stricter idempotency language. + +7. Automatic nightly following can accidentally become "network on every command." The plan should explicitly say normal pass-through commands are local-only and channel pointers refresh on explicit update/install/use. + +8. GitHub Releases can become the accidental public API if manifest URLs point users there directly. Keep `pkg.boundaryml.com` as the documented API. + +## Recommended Changes To `TASK.md` + +These edits have been applied to the current plan and user-path docs. + +1. Wrapper-owned commands are: + +```text +baml toolchain install > +baml toolchain use > +baml toolchain uninstall +baml toolchain list +baml toolchain update +baml self-update +``` + +2. `baml install`, `baml add`, and top-level `baml update` are reserved for future package/dependency management until that design is settled. + +3. Nightly examples use: + +```bash +baml toolchain use nightly +baml toolchain update +``` + +4. `baml update` is not the primary toolchain command. If a temporary alias is added later, it must clearly print that it is aliasing `baml toolchain update` and have a deprecation plan before package management ships. + +5. Keep `baml ide install` as the primary user command. Describe it as wrapper pass-through to the selected toolchain payload, and include non-interactive editor flags. + +6. Python packaging boundary: + +```text +The Python runtime import (`baml_core`) is governed only by the Python environment. The Python `baml` console script is a launcher for CLI commands. It may install BAML toolchains under `~/.baml`, but it must never modify the installed Python wheel or silently upgrade `baml_core`. +``` + +7. Generated-code compatibility requirement: generated code embeds generator version and expected `baml_core` range; runtime mismatch produces a loud diagnostic with package-manager commands and `baml generate` when regeneration is the fix. + +8. Nightly status semantics: active selector, installed concrete version, latest known remote, and explicit update command. + +9. Package-manager bootstrap: first install only, idempotent, non-resetting, skipped for existing config/toolchains, non-interactive-safe, and no automatic editor extension install. + +10. V1 security checks: SHA-256 verification, path traversal protection, temp-dir extraction, atomic rename, manifest schema/min wrapper version, and post-extract version sanity check. + +## Recommended Changes To `USER-PATHS.md` + +1. New Canary User: + +`brew install baml` installs the wrapper and either bootstraps canary idempotently or prints `baml toolchain use canary`; editor setup is explicit through `baml ide install`. + +2. Existing Canary User: + +Use `baml toolchain update`. + +3. Nightly User: + +Use: + +```bash +baml toolchain use nightly +baml toolchain update +``` + +4. IDE User: + +Keep `baml ide install` as the primary story, show editor flags such as `baml ide install --cursor`, and note that the VSIX launches local `baml lsp`. + +5. Python User: + +Add explicit stories for: + +```bash +uvx baml generate +uv run baml generate +python -c "import baml_core" +``` + +The first two are launcher/CLI paths; the last is runtime import and must report version mismatches rather than mutating the environment. + +## Resolved Decisions From Review + +1. `baml toolchain use ` writes the active default selector in wrapper-owned user config. Project-local selection is done by editing `baml.toml [toolchain]`; do not make `use` prompt for global versus project-local in v1. + +2. Normal pass-through commands do not make network requests. A missing exact pinned toolchain fails with installed-toolchain context and an explicit `baml toolchain install ` command. `baml toolchain use` and `baml toolchain install` may install missing toolchains because those commands are allowlisted for network access. + +3. Top-level `baml update` is not the primary toolchain command. The documented command is `baml toolchain update`; top-level `baml update` remains reserved for future package/dependency semantics unless a temporary alias is intentionally added with a deprecation plan. + +4. `baml ide install` installs the release-built VSIX/assets from the selected toolchain. It should support editor flags such as `--cursor`, `--code`, `--windsurf`, and `--all`; Marketplace publishing is deferred. + +5. Keep the conceptual split between the Python `baml` launcher surface and `baml_core` runtime surface. The implementation may use one or two PyPI distributions depending on migration constraints, but imports must never self-update or mutate the Python environment. + +6. Use `pkg.boundaryml.com` manifests as the user-facing channel directory. GitHub Releases may exist for nightly auditability and immutable asset storage, but they are not the normal user install/update surface. + +7. Signing/provenance is not a v1 blocker. The minimum v1 bar is SHA-256, safe extraction, schema validation, and atomic install. Sigstore/SLSA/provenance should be tracked as a follow-up if it does not fit v1 release engineering scope. + +## Sources + +- Rust rustup overrides: https://rust-lang.github.io/rustup/overrides.html +- Rust rustup channels: https://rust-lang.github.io/rustup/concepts/channels.html +- Cargo update: https://doc.rust-lang.org/cargo/commands/cargo-update.html +- Cargo install: https://doc.rust-lang.org/cargo/commands/cargo-install.html +- Bun install: https://bun.sh/docs/pm/cli/install +- Bun update: https://bun.sh/docs/pm/cli/update +- Bun upgrade/nightly/Homebrew guidance: https://bun.com/docs/installation +- Bunx: https://bun.sh/docs/pm/bunx +- mise getting started: https://mise.jdx.dev/getting-started.html +- mise install: https://mise.jdx.dev/cli/install.html +- mise use: https://mise.jdx.dev/cli/use.html +- mise self-update: https://mise.jdx.dev/cli/self-update.html +- asdf versions and shims: https://asdf-vm.com/manage/versions.html +- asdf core update guidance: https://asdf-vm.com/manage/core.html +- npm npx: https://docs.npmjs.com/cli/commands/npx/ +- Volta tool installation and pinning: https://www.voltajs.com/guide/installing.html +- uv tools: https://docs.astral.sh/uv/concepts/tools/ +- uv CLI reference: https://docs.astral.sh/uv/reference/cli/ +- uv cache: https://docs.astral.sh/uv/concepts/cache/ +- pipx docs: https://pipx.pypa.io/canary/docs/ +- Python entry points specification: https://packaging.python.org/en/latest/specifications/entry-points/ +- Python version specifiers / PEP 440: https://packaging.python.org/en/latest/specifications/version-specifiers/ +- Go toolchains: https://go.dev/doc/toolchain +- Go modules reference (`go install`, module cache, checksums): https://go.dev/ref/mod +- gopls package docs: https://pkg.go.dev/golang.org/x/tools/gopls +- Deno upgrade: https://docs.deno.com/runtime/reference/cli/upgrade/ +- Deno install: https://docs.deno.com/runtime/reference/cli/install/ +- Deno modules/cache/lockfiles: https://docs.deno.com/runtime/fundamentals/modules/ +- Homebrew formula cookbook: https://docs.brew.sh/Formula-Cookbook +- Homebrew manpage: https://docs.brew.sh/Manpage +- Arch PKGBUILD: https://wiki.archlinux.org/title/PKGBUILD +- Arch VCS package guidelines: https://wiki.archlinux.org/title/VCS_package_guidelines +- Arch User Repository: https://wiki.archlinux.org/title/Arch_User_Repository +- VS Code language server extension guide: https://code.visualstudio.com/api/language-extensions/language-server-extension-guide +- VS Code install from VSIX: https://code.visualstudio.com/docs/configure/extensions/extension-marketplace +- GitHub Releases: https://docs.github.com/en/repositories/releasing-projects-on-github/about-releases diff --git a/TASK/TASK.md b/TASK/TASK.md new file mode 100644 index 00000000000..d360cdd15ff --- /dev/null +++ b/TASK/TASK.md @@ -0,0 +1,1638 @@ +# BAML Language Release Architecture Overhaul + +## Scope + +In scope: everything under `baml_language/` plus its release workflow, the new Homebrew tap, the new install/wrapper toolchain, and the `boundaryml-tap` rename. + +Out of scope (kept separate, no changes): the `engine/` release pipeline (`.github/workflows/release.yml`, `tools/bump-version`, `tools/versions/*.cfg`). Old `baml-py` / `baml_client` users continue on that track. + +## Architecture (target state) + +```mermaid +flowchart TB + subgraph dev [Developer machine] + BamlBin["baml (wrapper, thin pass-through)"] + Toolchains["~/.baml/toolchains/version/
baml-cli + baml-pack-host + assets/baml-vscode.vsix"] + BamlToml["baml.toml in project
(adds [toolchain] table)"] + end + + subgraph dist [Distribution] + BrewTap["BoundaryML/homebrew-tap
(Formula/baml.rb, wrapper only + setup caveats)"] + PkgDomain["pkg.boundaryml.com
install.sh + manifest/v1/*.json
(reuses existing CDK + S3 + OIDC)"] + GhReleases["GitHub Releases
(binary storage: toolchain + wrapper archives)"] + end + + subgraph ci [GitHub Actions] + CanaryCi["canary CI green
(workflow_run)
publishes nightly"] + ReleaseToml["release.toml changed
(canary intent)"] + Plan["plan job
(version + channel)"] + BuildMatrix["build matrix
cli + pack-host + vsix + baml_core wheels + wrapper"] + Gate["all-builds gate"] + Publish["publish-* jobs
(GH toolchain, PyPI, manifest; wrapper packages only when wrapper changes)"] + end + + CanaryCi --> Plan + ReleaseToml -.-> Plan + Plan --> BuildMatrix + BuildMatrix --> Gate + Gate --> Publish + Publish --> GhReleases + Publish --> BrewTap + Publish --> PkgDomain + + BrewTap --> BamlBin + PkgDomain --> BamlBin + BamlBin -->|"reads (directory)"| PkgDomain + BamlBin -->|"downloads (storage)"| GhReleases + BamlBin -->|"installs into"| Toolchains + BamlToml -.->|"pins"| BamlBin +``` + +Note on the two-tier distribution: `pkg.boundaryml.com` is the **directory** (channel pointers + per-version metadata + install scripts, mutable on the channel side); GitHub Releases is the **storage** (large binary artifacts + sha256 sidecars, immutable per tag). End users never browse the GitHub releases tab in the normal flow; the wrapper fetches everything via the manifest. Maintainers still see GitHub releases (auto-generated changelogs, asset uploads, release-notes UI) as the source of truth for "what was published when". + +## Decided constraints (from prior conversation) + +- Version format: `0.11.0-nightly.YYYYMMDD.` (no leading zeros in date segments to stay SemVer-compliant). `nightly` matches the channel name 1:1; valid SemVer. +- Two channels: `canary` + `nightly`. Nightly publishes automatically from every successful `canary` branch CI run. Canary publishes when the human-edited `baml_language/release.toml` `canary_version` advances. There is no `stable` channel in v1. GitHub tags/releases are outputs created by the workflow, not inputs that trigger BAML language releases. +- The thing distros install is the wrapper, not the CLI. `baml-cli`, `baml-pack-host`, and the VSIX live inside per-toolchain directories under `~/.baml/`. +- There are two release products: `baml-wrapper` and `baml-toolchain`. Package managers publish only `baml-wrapper`; language releases publish `baml-toolchain` archives and manifests. A toolchain nightly/canary release must never require a Homebrew/AUR package update. +- The wrapper is a thin pure pass-through. Only toolchain-management commands (`toolchain install`, `toolchain use`, `toolchain uninstall`, `toolchain list`, `toolchain update`, `self-update`) live in the wrapper itself. Everything else (including `ide install`) is owned by the selected toolchain payload and forwarded by the wrapper. +- VSIX is platform-neutral, built once, and bundled into every per-target toolchain archive. Users should not need to install a new VSIX for every toolchain release; compatibility is by explicit LSP/playground protocol ranges and capability flags. +- Tap rename: `homebrew-baml` -> `homebrew-tap`, one-shot with GitHub redirect + deprecation banner. +- Hosting: reuse the **existing** `pkg.boundaryml.com` CDK stack (S3 + OIDC release-publish IAM role, already deployed via [tools/pkg_boundaryml_com/lib/pkg-boundaryml-com-stack.ts](tools/pkg_boundaryml_com/lib/pkg-boundaryml-com-stack.ts)). Cloudflare, configured outside this CDK stack, is the HTTPS front door for the domain. Install script and channel manifests live there. No new DNS, no new buckets, no new hosting stack. +- Engine repo release path is untouched. +- The selected toolchain binary and `baml_core` (Python) carry the same canonical version. PyPI is the only ecosystem that requires a one-way version translation (PEP 440 doesn't accept the canonical form). +- Project pin lives in the existing `baml.toml` (under a new `[toolchain]` table). No new project-level config file. +- Old workflow shape (fan-out -> fan-in gate -> separate publish jobs) is the target shape. + +### Why the wrapper is a separate binary (not built into `baml-cli`) + +Tested alternative: collapse the wrapper into `baml-cli` itself, with toolchain-management subcommands (`baml-cli install`, `baml-cli use`, etc.). Rejected because: + +- **Bootstrap problem.** To install `baml-cli` you'd need `baml-cli`. The wrapper is the artifact that ships via `brew install` / `curl ... | sh`; the CLI is the artifact it downloads. They have different release cadences (wrapper rarely changes; CLI changes weekly). +- **Size and update cost.** `baml-cli` is multi-MB (Rust + engine). Forcing every user to redownload it for a wrapper bug fix is wrong. Splitting keeps the brew-installable thing small. +- **CEO constraint** (recorded earlier): wrapper is pure pass-through. No business logic in the wrapper means there's no need to ever release a wrapper update for a CLI bug, and vice-versa. +- **Industry pattern.** `rustup`+`rustc`, `nvm`+`node`, `pyenv`+`python`, `volta`+`node` — all separate-binary version managers. Same shape we're adopting. + +### Release products: `baml-wrapper` vs `baml-toolchain` + +This is the product boundary that prevents the new release path from becoming two coupled systems: + +| Product | Version source | Contains | Published to | Cadence | +|---|---|---|---|---| +| `baml-wrapper` | `baml_language/crates/baml/Cargo.toml` `[package].version` | `bin/baml` only | GitHub release assets under `baml-wrapper-`, `manifest/v1/wrapper.json`, Homebrew `baml`, AUR `baml` / `baml-bin`, `install.sh` / `install.ps1` | Rare. Only when wrapper/install behavior changes. | +| `baml-toolchain` | `baml_language/release.toml` canary intent + CI-derived channel version | `bin/baml-cli`, `bin/baml-pack-host`, `assets/baml-vscode.vsix` | GitHub release assets under `baml-language-`, `manifest/v1/version/.json`, `manifest/v1/canary.json`, `manifest/v1/nightly.json`, PyPI `baml_core` | Every successful `canary` branch commit publishes nightly; canary publishes when `canary_version` advances. | + +Hard rules: + +- Homebrew and AUR package the wrapper only. They do not vendor, symlink, or version the language toolchain inside the package artifact. +- The wrapper archive never contains `baml-cli`, `baml-pack-host`, or the VSIX. +- The toolchain archive never contains `baml`. +- A `baml-toolchain` release does not dispatch Homebrew/AUR updates. Users subscribed to nightly receive new toolchains through `manifest/v1/nightly.json`, not through package-manager churn. +- A `baml-wrapper` release may happen inside the same release graph codebase, but it is a separate publish decision keyed off wrapper-version changes. It updates `manifest/v1/wrapper.json` and package-manager formulas/templates; it does not create a new language toolchain version. +- The authoritative wrapper version is the literal `[package].version` in `baml_language/crates/baml/Cargo.toml`. The wrapper crate must use its own package version and must not inherit `version.workspace = true`. +- Do not introduce or consume `BAML_WRAPPER_VERSION` as a release/workflow version authority. Wrapper release decisions parse `baml_language/crates/baml/Cargo.toml`, validate it as SemVer, compare against the latest `manifest/v1/wrapper.json` version, and set `wrapper_changed = wrapper_version != latest_wrapper_version`. A runtime diagnostic env var with that name is optional, but it must not drive build, publish, or version-selection behavior. +- `baml --version` may use `env!("CARGO_PKG_VERSION")` because the wrapper's Cargo package version is the wrapper product version. This exception does not apply to `baml-cli`, LSP, SDK runtimes, or generated-code surfaces, which all use the stamped BAML language/toolchain version. +- `baml toolchain update` updates the active language toolchain only. It never updates the wrapper binary. +- `baml self-update` updates the wrapper only. It never installs or changes the active language toolchain. +- Normal wrapper commands never silently self-update. If the wrapper is too old for a manifest/schema, it fails with an actionable command (`baml self-update` for curl installs, package-manager upgrade command for managed installs). +- Package-manager installs are wrapper-only in v1. Homebrew and AUR install only `bin/baml`; they must not download, install, select, or update a BAML language toolchain during install, reinstall, or upgrade. +- Package-manager caveats/install messages print the user-scoped follow-up command `baml toolchain use canary` (and optionally `baml toolchain use nightly`). They may also recommend `baml ide install --cursor`, but IDE extension installation is always explicit. +- Curl/PowerShell installers are different because they are user-scoped BAML installers: unless `--wrapper-only` is passed, they may install/update the wrapper and bootstrap the requested toolchain with `baml toolchain use `. + +## Dead code and duplication to remove + +This plan replaces, not adds. Concrete deletions, each gated by a corresponding new construct: + +| File / path | Action | Reason | +|---|---|---| +| [.github/workflows/release-baml-language-alpha.yml](.github/workflows/release-baml-language-alpha.yml) | DELETE | Replaced by the new release graph orchestrator, with `release-baml-language.yml` as the entrypoint (Phase 2.1). | +| [.github/workflows/release-sdk.yaml](.github/workflows/release-sdk.yaml) | RENAME / INTEGRATE | Rename to `.github/workflows/publish-python-pypi.yml` and invoke it from the BAML language release graph. This is the current Python `baml_core` release workflow, and the new name scales to future SDK publish workflows such as `publish-typescript-npm.yml`, `publish-ruby-rubygems.yml`, and `publish-java-maven.yml`. Update or validate the PyPI trusted-publisher binding for project `baml-core` after the rename. Keep OIDC trusted publishing; do not add username/password or API-token secrets. | +| [.github/workflows/release-cli.yaml](.github/workflows/release-cli.yaml) | DELETE | Broken (`uses:` references nonexistent `build-cli-release-reusable.yaml`; actual file is `build-cli-release.reusable.yaml`). Has been silently failing on every `release` event including nightly alpha prereleases. Engine's [release.yml](.github/workflows/release.yml) L80-85 already invokes the build directly. This file is dead. | +| [.github/workflows/release-pkg-boundaryml-com.yml](.github/workflows/release-pkg-boundaryml-com.yml) | FOLD into release graph | Currently `workflow_dispatch`-only and its own TODO says "tie this into the release process". Becomes the `publish-pkg-boundaryml-com` job in the new release graph, reusing the existing `pkg-boundaryml-com-github-release` IAM role. | +| `release.toml` single-table form | REPLACE | One human-edited `[release] canary_version = "X.Y.Z"` table. Nightly versions are derived by CI from that canary version, so no one manually edits nightly release state. The old single `[channel]` table only ever had one reader (`scripts/baml-language-version`), so no compat shim needed. | +| `BAML_RELEASE_VERSION` env override in [.github/workflows/release-baml-language-alpha.yml](.github/workflows/release-baml-language-alpha.yml) / [baml_cli/src/commands.rs](baml_language/crates/baml_cli/src/commands.rs) | REMOVE | Deprecated alpha-era version injection. New design uses `baml_language/release.toml` as the only human source of truth, a per-run `release-plan.json` as the frozen CI derivation, and stamped version files consumed by Python, VSIX/TypeScript, Node, Rust, and future SDK surfaces. | +| [packaging/aur/baml/PKGBUILD](packaging/aur/baml/PKGBUILD), [packaging/aur/baml-bin/PKGBUILD](packaging/aur/baml-bin/PKGBUILD) | REDRAFT | Current templates package `baml-cli` and symlink `baml`. New templates package only the wrapper and track wrapper versions, not BAML language versions. | +| VSIX-bundled `baml-cli` in [typescript2/package.json](typescript2/package.json) / [typescript2/app-vscode-ext/src/extension.ts](typescript2/app-vscode-ext/src/extension.ts) | REMOVE | The current VSIX copies a platform-specific `baml-cli` into `dist/baml-cli`. The new VSIX must be platform-neutral and launch the installed `baml` wrapper instead. | +| Pack's three free functions for URL resolution in [baml_language/crates/baml_cli/src/pack_command.rs](baml_language/crates/baml_cli/src/pack_command.rs) L481-621 | MOVE to new `baml_release` crate | Currently inlined in pack; the wrapper would otherwise duplicate them. Extraction is pure refactor in PR 3 with zero behavior change (Phase 2.7). | + +What the cleanup achieves: at the end of the plan there is **one coherent BAML language release graph**, **one version script** (`scripts/baml-language-version`), **one fetcher crate** (`baml_release`), **one hosting domain** (`pkg.boundaryml.com`), and **one manifest system** with explicit `baml-toolchain` and `baml-wrapper` schemas. The graph may use multiple focused workflows/reusable workflows for Python, TypeScript/VSIX, Rust binaries, and publishing, but they fan out/fan in through one release plan and do not publish independently with conflicting version logic. + +## Explicitly out of scope (no-touch list) + +Things that look related but are intentionally untouched. Calling these out so PRs don't drift: + +| File / system | Why untouched | +|---|---| +| [.github/workflows/release.yml](.github/workflows/release.yml) | Engine SDK release. Separate product (`baml-py`, `baml-extension`, engine CLI). User decision: keep engine releases going as-is. | +| [.github/workflows/build-cli-release.reusable.yaml](.github/workflows/build-cli-release.reusable.yaml), [.github/workflows/build-python-release.reusable.yaml](.github/workflows/build-python-release.reusable.yaml), [.github/workflows/build-vscode-release.reusable.yaml](.github/workflows/build-vscode-release.reusable.yaml), [.github/workflows/build-typescript-release.reusable.yaml](.github/workflows/build-typescript-release.reusable.yaml), [.github/workflows/build-ruby-release.reusable.yaml](.github/workflows/build-ruby-release.reusable.yaml), [.github/workflows/build-jetbrains-release.reusable.yaml](.github/workflows/build-jetbrains-release.reusable.yaml), [.github/workflows/publish-zed-release.reusable.yaml](.github/workflows/publish-zed-release.reusable.yaml) | Engine-side reusable workflows, called from `release.yml`. | +| [tools/bump-version](tools/bump-version), [tools/versions/*.cfg](tools/versions/) | Engine SDK version management. Separate cadence; users of `bump-version` are engine maintainers. | +| [typescript/apps/vscode-ext/](typescript/apps/vscode-ext/) | OLD VSCode extension (`baml-extension`, version `0.222.0`, currently on Marketplace). Engine-side. The new VSIX (`typescript2/app-vscode-ext/`) is a separate, smaller LSP-only extension that ships bundled in toolchain archives. Coexistence story is a product decision deferred to Phase 4. | +| [baml_language/crates/tools_stow/Cargo.toml](baml_language/crates/tools_stow/Cargo.toml) (`cargo-stow`) | Independent tool, `publish = true`, hardcoded `version = "0.1.0"`, decoupled from the BAML language product version. Has its own lifecycle. | +| [engine/](engine/) entire directory | Old runtime / engine codebase. Not in `baml_language/`. | +| [baml_language/Cargo.toml](baml_language/Cargo.toml) workspace `version = "0.0.0-beta"` | Intentionally decoupled. All crates are `publish = false` except `tools_stow`. Public product versions come from the stamped `baml_version` module, not Cargo package metadata. Plan does NOT modify the workspace version field (see "Workspace version footgun" risk). | + +## Cross-language version compatibility + +BAML targets six host languages. The canonical SemVer string `0.11.0-nightly.20260522.a` must round-trip cleanly into each language's package ecosystem. Only one ecosystem (PyPI) requires translation; everything else accepts the canonical form as-is. + +| Ecosystem | Accepts canonical form? | Notes | +|---|---|---| +| Rust / Cargo | Yes | SemVer-native. Used directly in `Cargo.toml` `version = "..."`. | +| TypeScript / npm | Yes | SemVer-native. `npm publish --tag nightly` is the dist-tag for nightly builds; the version string itself doesn't need to change. | +| Go modules | Yes (with `v` prefix on tag) | Git tag is `v0.11.0-nightly.20260522.a`; module version inside `go.mod` is `v0.11.0-nightly.20260522.a`. For v2+ modules the module path needs `/v2` suffix (separate concern; surfaces only at v2.0.0). | +| C | N/A | No package manager; version embedded in headers / pkg-config files we control. | +| C++ | N/A | Same as C. If we ever ship via vcpkg / Conan, those accept arbitrary version strings. | +| **PyPI (PEP 440)** | **No** | Must translate. See below. | + +PEP 440 translation (PyPI only): + +| Canonical SemVer | PEP 440 | +|---|---| +| `0.11.0` | `0.11.0` | +| `0.11.0-nightly.20260522.a` | `0.11.0.dev2026052200` | +| `0.11.0-nightly.20260522.b` | `0.11.0.dev2026052201` | +| `0.11.0-nightly.20260522.z` | `0.11.0.dev2026052225` | +| `0.11.0-nightly.20260523.a` | `0.11.0.dev2026052300` | + +Encoding: `0.11.0.dev` where `LL` is the zero-padded 2-digit letter index (`a`=00, `b`=01, ..., `z`=25). This: + +- Sorts monotonically under both PEP 440 and lexical byte comparison. +- Uses PEP 440 `.devN` (dev-release) rather than `aN` (alpha) because nightly IS a dev release semantically, and `pip install baml_core` excludes dev releases by default — exactly the behavior we want. Users opt in with `pip install --pre baml_core` or by pinning an exact version. +- Survives the letter doubling (`aa`..`zz` -> indices 26..701) with a 4-digit zero-padded scheme if we ever exceed `z` (deferred; today we hard-fail at `z`). + +`baml_core.__version__` always returns the canonical SemVer form (`0.11.0-nightly.20260522.a`), set explicitly in [baml_language/sdks/python/src/baml_core/__init__.py](baml_language/sdks/python/src/baml_core/__init__.py) at build time. Python users never have to see the PEP 440 form unless they're pinning in `requirements.txt`. + +Go tag-prefix handling: when a Go SDK ships, the git tag for an SDK release is `sdk/go/v0.11.0-nightly.20260522.a` (separate from the language tag `baml-language-0.11.0-nightly.20260522.a`); this lets Go module proxies fetch directly without `replace` directives. Tracked in Phase 4. + +## Phase 0 - Standalone prep (low-risk, ship first) + +Goal: settle conventions before the bigger work lands. + +### 0.1 Brew tap rename + +- Rename GitHub repo `BoundaryML/homebrew-baml` -> `BoundaryML/homebrew-tap` (no formula file moves; `Formula/baml.rb` stays where it is). +- Update CI dispatch target in [.github/workflows/release-baml-language-alpha.yml](.github/workflows/release-baml-language-alpha.yml): + - L480: `gh api repos/BoundaryML/homebrew-baml/contents/Formula/baml.rb` -> `homebrew-tap` + - L527: `gh api --method POST repos/BoundaryML/homebrew-baml/dispatches` -> `homebrew-tap` +- Add a one-time deprecation `caveats` block to `Formula/baml.rb` in the tap so anyone still on the old name sees an upgrade hint after `brew update`. +- Update docs / READMEs to `brew install boundaryml/tap/baml`. +- Keep secret name `HOMEBREW_BAML_DISPATCH_TOKEN` (renaming is cosmetic; deferred). + +### 0.2 Version format + `release.toml` canary intent + +Replace [baml_language/release.toml](baml_language/release.toml) with a single human-edited canary release intent: + +```toml +[release] +canary_version = "0.11.0" +``` + +Rewrite [scripts/baml-language-version](scripts/baml-language-version) to: + +- Parse the new `[release] canary_version` table (drop the single `[channel]` table; the script is the only reader of this file). +- Add `compute --channel {canary|nightly}`: + - canary -> exactly `` + - nightly -> `-nightly.YYYYMMDD.`, where `` is computed by reading existing GitHub release tags for today's date and picking the next single-letter suffix (`a`, `b`, ..., `z`). Error if `z` is exceeded (forces noticing and bumping to 2-letter scheme). +- Add `--pypi` flag to `compute` that applies the PEP 440 translation from the Cross-language compatibility section (`0.11.0-nightly.20260522.a` -> `0.11.0.dev2026052200`). +- Add `check`: + - `[release].canary_version` must not decrease vs `origin/canary` merge-base. + - Nightly base is derived and must be greater than canary (`next_patch(canary_version) > canary_version`). + - If `canary_version` changed in the source commit and that canary version has not already been published, the workflow publishes both canary and nightly for that commit. Otherwise, every successful `canary` branch commit publishes nightly only. +- Bump commands update canary intent only: `bump --minor`, `bump --patch`, etc. + +Acceptance: with `canary_version = "0.11.0"`, `scripts/baml-language-version compute --channel canary` prints `0.11.0`; `scripts/baml-language-version compute --channel nightly` prints `0.11.1-nightly.20260522.a` (or next letter) given today's date and existing tags; same nightly command with `--pypi` prints `0.11.1.dev2026052200`. + +First unified version decision: the first canary release under this system should be `0.11.0`. Today the language CLI and Python `baml_core` do not share a committed source of truth: the CLI falls back to the workspace Cargo version (`0.0.0-beta`) unless the alpha workflow injects `BAML_RELEASE_VERSION`, while `baml_core` currently has independent Python package metadata. The new system deliberately starts the unified product version at the existing BAML language release intent (`0.11.0`) and stamps that same canonical version into `baml-cli`, `baml_core`, LSP, VSIX metadata, Node/TypeScript surfaces, and future SDKs. That means Python `baml_core` will intentionally jump to `0.11.0` so it matches the BAML language/toolchain version; this is preferable to carrying the old independent `baml_core` package version forward. The first nightly derived from that canary is `0.11.1-nightly.YYYYMMDD.a`. + +## Phase 1 - The `baml` wrapper, manifest, and bash installer + +Goal: introduce the thin shim that every user actually installs, and the infrastructure that feeds it. + +### 1.1 New crate `baml_language/crates/baml/` + +A small Rust binary, name `baml`, built in the existing workspace. The wrapper is a **pure pass-through** by design (constraint from CEO): everything except toolchain management is forwarded to the selected toolchain payload. Responsibilities: + +Wrapper crate version shape: + +```toml +[package] +name = "baml" +version = "0.1.0" +publish = false +``` + +The wrapper crate must not use `version.workspace = true`. V1 may document a manual edit to bump this version; prefer adding `scripts/baml-wrapper-version bump --patch|--minor` and `scripts/baml-wrapper-version check` so maintainers do not hand-edit SemVer incorrectly. That helper edits only `baml_language/crates/baml/Cargo.toml` and does not touch `baml_language/release.toml`, `baml_version`, `baml_core`, VSIX version, SDK versions, or any BAML language/toolchain version surface. + +- Resolve active toolchain (precedence: `$BAML_VERSION` env -> nearest `baml.toml` `[toolchain]` table -> `~/.baml/config.toml` default selector -> `canary`). +- Exec the matching `baml-cli` from `~/.baml/toolchains//bin/`. +- Own these subcommands (everything else, including `run`, `generate`, `test`, `fmt`, `ide`, `pack`, passes through unchanged to the selected toolchain payload): + - `baml toolchain install >` (download/verify/install a concrete toolchain, but do not change the active default selector) + - `baml toolchain use >` (resolve, install if missing, and select as the user default) + - `baml toolchain uninstall ` + - `baml toolchain list` (local-only by default; no network) + - `baml toolchain update` (refresh the active default channel to head; exact-version defaults do not advance; does not update the wrapper) + - `baml self-update` (replace the wrapper itself; refused for managed installs; does not install toolchains) +- Self-detect install path. If the wrapper lives under a path owned by Homebrew (`brew --prefix` / `/opt/homebrew/...` / formula marker file), `baml self-update` refuses with a pointer to `brew upgrade baml`. + +Pass-through examples (these all just exec the active selected toolchain with identical argv): + +``` +baml run -f main # forwards to selected toolchain +baml generate --target python # forwards to selected toolchain +baml ide install --cursor # forwards to selected toolchain +baml pack # forwards to selected toolchain +``` + +The internal `baml-cli` binary is not a documented user command. The wrapper sets `BAML_WRAPPER_EXEC=1` before execing the selected internal toolchain binary; `baml-cli` suppresses the direct-invocation warning only when `BAML_WRAPPER_EXEC=1` or `BAML_CLI_ALLOW_DIRECT=1` is set. If `baml-cli` is invoked directly outside the wrapper, it should print a once-per-process warning to stderr before continuing or failing: + +```text +warning: using the internal BAML toolchain binary directly is not recommended. Use `baml` instead. +``` + +The warning must never go to stdout because commands such as `--version`, `describe`, and future JSON outputs may be machine-readable. Optional diagnostic env vars such as `BAML_WRAPPER_VERSION` and `BAML_WRAPPER_RESOLVED_TOOLCHAIN` may be set by the wrapper for logging, but they must not become version authorities. Direct `baml-cli` may also perform a local-only best-effort project mismatch warning by comparing nearest `baml.toml [toolchain]` metadata against `baml_version::CANONICAL_VERSION`; it must not hit the network, read wrapper state, install toolchains, or attempt repair. + +Tests: + +- Direct `baml-cli --version` without `BAML_WRAPPER_EXEC` prints the warning to stderr and exact version output to stdout. +- Direct `baml-cli --version` with `BAML_WRAPPER_EXEC=1` prints no warning. +- Direct `baml-cli --version` with `BAML_CLI_ALLOW_DIRECT=1` prints no warning. +- Wrapper pass-through sets `BAML_WRAPPER_EXEC=1`, so invoking `baml --version` or `baml generate` through the wrapper does not show the direct-binary warning. +- Machine-readable stdout tests assert the warning never appears on stdout. +- Optional mismatch test: create a temporary project with `baml.toml [toolchain]` selecting a different version and invoke `baml-cli` directly; assert the mismatch warning appears without any network access. + +User-facing docs, onboarding copy, and diagnostics should prefer `baml ` unless they are describing archive contents or implementation internals. + +Layout the wrapper manages: + +``` +~/.baml/ + config.toml # user-authored/user-editable intent + state.toml # wrapper-owned active channel resolutions + bin/baml # the wrapper itself (when bash-installed) + toolchains/ + 0.11.0/ + VERSION + install.json + bin/baml-cli + bin/baml-pack-host + assets/baml-vscode.vsix + 0.11.0-nightly.20260522.c/ + ... + manifest-cache/ # remote metadata cache only, TTL-controlled for channel pointers +``` + +Project pin extends the existing `baml.toml` (already created by `baml init`, see [baml_language/crates/baml_cli/src/init_command.rs](baml_language/crates/baml_cli/src/init_command.rs) L113-L116, L165-L173). Adds a new `[toolchain]` table: + +```toml +[package] +name = "my-project" + +[toolchain] +channel = "nightly" +# or: +# version = "0.11.0-nightly.20260522.c" +``` + +Schema notes: + +- `[toolchain]` is optional. Absent -> wrapper falls through to the `~/.baml/config.toml` default selector, then `canary`. +- The wrapper only reads `[toolchain]`; the rest of `baml.toml` is the selected toolchain's responsibility. No coupling between the two. +- `baml init` is **not** modified to write `[toolchain]` by default; the table is opt-in for projects that need to pin (CI, libraries, etc.). + +#### Toolchain resolution algorithm (deterministic, no ambiguity) + +The wrapper resolves the active toolchain on **every invocation** using this exact precedence (first match wins): + +```text +1. $BAML_VERSION env var (highest priority) + - If set to "canary" or "nightly", resolve to the locally recorded concrete version for that channel. + - If set to an exact version "X.Y.Z[-...]", that's the active toolchain. + +2. baml.toml [toolchain] in the nearest parent directory of $PWD + - Walk up from $PWD until a directory contains baml.toml. + - Stop at $HOME and at any filesystem boundary. + - If the file has [toolchain].version, use that. + - Else if [toolchain].channel, resolve to the locally recorded concrete version for that channel. + - Else (no [toolchain] table), continue. + +3. ~/.baml/config.toml [default] table + - selector = "canary" | "nightly" | "" + - Resolve same way as step 2. + +4. Hardcoded fallback: channel "canary". +``` + +Resolution outcomes: + +- **Resolved to an installed version**: wrapper execs that toolchain with the original argv. +- **Resolved to a version not installed locally**: error with actionable fix and the locally installed versions: + + ``` + error: baml.toml [toolchain] pins version 0.11.0-nightly.20260522.c, but it isn't installed. + Installed toolchains: 0.11.0, 0.11.1-nightly.20260521.a + Run: baml toolchain install 0.11.0-nightly.20260522.c + ``` + + Wrapper does **not** auto-install during normal pass-through commands. Network access during `baml run`, `baml generate`, `baml pack`, or `baml lsp` is surprising; user must opt in with a toolchain command. + + For the common package-manager first-run case, where the wrapper is installed but no toolchain exists yet, the error should be direct: + + ```text + error: no BAML toolchain is installed. + Run: baml toolchain use canary + Or: baml toolchain use nightly + ``` + +- **Resolved to a channel with no active concrete version in `state.toml`**: normal commands do not fetch. They fail with a local-state message and suggest `baml toolchain use ` as the primary fix. + +- **No internal CLI binary in the resolved toolchain dir**: corrupt install error pointing at `baml toolchain install --force`. + +#### Network and cache policy + +The wrapper stores enough local state to run normal commands offline: + +- user intent in `config.toml`, such as `canary`, `nightly`, or a concrete version; +- active channel resolutions in `state.toml`; +- installed toolchain directories with `VERSION` sanity files and `install.json` metadata; +- cached remote manifests under `~/.baml/manifest-cache/`, which are cache only and never authoritative for normal command resolution. + +Channel metadata cache TTL is 24 hours. + +Only allowlisted commands make network requests: + +- `baml toolchain use ` may fetch channel metadata when the channel cache is missing or expired, then installs the selected concrete toolchain if missing, atomically updates `state.toml`, and records `[default].selector = ""` in `config.toml`. +- `baml toolchain use ` fetches the immutable per-version manifest only if that version is not installed locally or its cached manifest is missing, installs if needed, then records `[default].selector = ""` in `config.toml`. It does not update any channel entry in `state.toml`. +- `baml toolchain install ` always resolves the latest channel pointer and installs that concrete version, but does not change `[default].selector` and does not update that channel's `active_version` in `state.toml`. +- `baml toolchain install ` fetches that immutable version manifest only when needed and does not mutate `config.toml` or `state.toml`. +- `baml toolchain update` only advances `state.toml` when the active default selector is a channel. It installs the latest concrete version for that channel, then atomically swaps the channel's `active_version`. If the active selector is an exact version, it reports that exact versions do not advance automatically and suggests `baml toolchain use canary` or `baml toolchain use nightly`. +- `baml toolchain list` is local-only by default. A separate explicit remote mode may be added later, but plain `list` must not hit the network. + +All other commands, including `baml generate`, `baml run`, `baml describe`, `baml pack`, and `baml lsp`, use local state only and never hit the network. + +When a command decides to fetch remote metadata, it treats the old cache entry as stale for that request, writes the fetched JSON to a temporary file, validates schema and checksums, then atomically replaces the cache entry. If validation fails, delete the temporary file and do not change the active concrete toolchain. If the network fails, keep the already-installed concrete toolchain and report that the latest remote version could not be checked. + +Manifest source override for dry runs and mirrors: + +- Production default: `https://pkg.boundaryml.com/manifest/v1`. +- Env override: `BAML_MANIFEST_BASE_URL=`. +- Optional command override for toolchain-management commands: `--manifest-base-url `. +- Precedence: `--manifest-base-url` -> `BAML_MANIFEST_BASE_URL` -> production default. +- The override applies only to wrapper/toolchain manifest reads. Normal pass-through commands still do not hit the network. +- The override is never persisted into `config.toml`. +- Cache entries are namespaced by manifest base URL so dry-run manifests cannot poison production cache. Production may use `manifest-cache/prod/`; overrides use `manifest-cache/override//`. +- Channel state written under an override records the manifest base URL/hash in `state.toml`. A later command using a different manifest base must not silently treat that channel state as valid; it should either use a temporary `BAML_HOME` in CI or print a clear diagnostic telling the user to run `baml toolchain use ` under the current source. + +#### Wrapper config file + +Wrapper state is split so user intent, wrapper-owned active state, remote metadata cache, and installed inventory do not blur together. + +`~/.baml/config.toml` stores what the user wants; hand-edits are supported: + +```toml +[default] +selector = "canary" # "canary" | "nightly" | "" + +[update] +auto_check = false # never silently nag (CI-friendly default) +``` + +`~/.baml/state.toml` stores the last successfully installed/activated concrete version for each channel: + +```toml +[channels.canary] +active_version = "0.11.0" +resolved_at = "2026-06-02T12:00:00Z" +manifest_path = "manifest-cache/prod/version/0.11.0.json" + +[channels.nightly] +active_version = "0.11.1-nightly.20260602.a" +resolved_at = "2026-06-02T12:30:00Z" +manifest_path = "manifest-cache/prod/version/0.11.1-nightly.20260602.a.json" +``` + +`manifest-cache/` stores remote JSON plus fetch metadata where useful (`fetched_at`, `etag`, etc.). It is not the active-version authority. A remote/list operation may refresh `manifest-cache/canary.json` and discover a newer version, but normal commands continue using `state.toml` until a toolchain-management command successfully installs and activates the newer concrete version. + +`toolchains//VERSION` contains the exact canonical version and is read as a tamper/sanity check on every wrapper invocation. `toolchains//install.json` records install metadata such as source manifest URL, archive URL, archive sha256, installed_at, and target triple. + +Atomicity rule: write `state.toml` and `config.toml` through temp files in the same directory, validate the serialized TOML, fsync where practical, then rename. Only update state after the toolchain archive is downloaded, verified, extracted, `VERSION`-checked, and fully materialized under `toolchains//`. + +Resolution invariant: + +```text +user intent (config.toml) + + active local channel state (state.toml) + + installed toolchain metadata (toolchains//VERSION) += normal command resolution +``` + +### 1.2 Manifest schemas (formal) + +#### Toolchain manifest + +Each `baml-toolchain` release publishes a JSON manifest. Schema lives in `baml_language/crates/baml_release/src/manifest.rs` (in the shared crate so both wrapper and `baml-cli` can validate without duplication). Envelope is versioned via `schema` integer field: + +```json +{ + "schema": 1, + "version": "0.11.0-nightly.20260522.c", + "channel": "nightly", + "released_at": "2026-05-22T19:00:00Z", + "artifacts": { + "aarch64-apple-darwin": { + "url": "https://github.com/BoundaryML/baml/releases/download/baml-language-0.11.0-nightly.20260522.c/baml-language-0.11.0-nightly.20260522.c-aarch64-apple-darwin.tar.gz", + "sha256": "abc123..." + }, + "x86_64-unknown-linux-gnu": { "url": "...", "sha256": "..." }, + "x86_64-unknown-linux-musl": { "url": "...", "sha256": "..." }, + "aarch64-unknown-linux-gnu": { "url": "...", "sha256": "..." }, + "aarch64-unknown-linux-musl":{ "url": "...", "sha256": "..." }, + "x86_64-apple-darwin": { "url": "...", "sha256": "..." }, + "x86_64-pc-windows-msvc": { "url": "...", "sha256": "..." }, + "aarch64-pc-windows-msvc": { "url": "...", "sha256": "..." } + }, + "vsix": { + "url": "https://github.com/BoundaryML/baml/releases/download/baml-language-0.11.0-nightly.20260522.c/baml-language-0.11.0-nightly.20260522.c.vsix", + "sha256": "..." + }, + "baml_core_pypi": { + "version": "0.11.0.dev2026052202" + } +} +``` + +Field requirements (schema=1): + +- **Required**: `schema`, `version`, `channel`, `released_at`, `artifacts` (with at least one entry). +- **Required keys inside `artifacts`**: `url`, `sha256`. URL must be HTTPS. SHA-256 is lowercase hex, 64 chars. +- **Optional**: `vsix`, `baml_core_pypi`. Wrapper treats missing `vsix` as "no IDE install available for this version" and missing `baml_core_pypi` as "no PyPI wheel". +- **Required target set**: exactly the supported release matrix for that product. No target in the matrix is optional/best-effort in v1. + - A `baml-toolchain` release publishes only when every target in the toolchain matrix succeeds. + - A `baml-wrapper` release publishes only when every target in the wrapper matrix succeeds. + - The manifest's `artifacts` target set must exactly equal the release matrix target set for that product. If a platform is too flaky to block releases, remove it from the supported release matrix rather than marking it optional. + - Wrapper behavior for a missing target is defensive only and should treat the manifest as incomplete/corrupt, not expected optional-target behavior. + + ```text + error: BAML 0.11.0 manifest is missing artifact for target aarch64-pc-windows-msvc. + This release manifest is incomplete. Try `baml toolchain update`, or report this release. + ``` + +Schema versioning policy: + +- **Minor additions** (new optional fields) keep `schema = 1`. Old wrappers ignore unknown keys (`serde(default)`). +- **Breaking changes** bump `schema` to `2`. Old wrappers refuse-with-explanation: "manifest schema 2 is newer than this wrapper; run `baml self-update`". +- Wrappers older than 6 months may see a schema bump; the `wrapper.json` pointer is updated synchronously to encourage upgrades. There is no field-by-field migration story — we just require a fresh wrapper. + +Bad release handling in v1: publish a newer fixed version and let the channel pointer advance to it. Per-version manifests and GitHub release assets remain immutable for reproducibility. + +Manifest endpoints (all on the existing `pkg.boundaryml.com` bucket): + +- `https://pkg.boundaryml.com/manifest/v1/canary.json` -> latest canary (mutable; overwritten on every canary release) +- `https://pkg.boundaryml.com/manifest/v1/nightly.json` -> latest nightly (mutable; overwritten on every nightly release) +- `https://pkg.boundaryml.com/manifest/v1/version/.json` -> immutable per-version snapshot (write-once; never overwritten) +- `https://pkg.boundaryml.com/manifest/v1/wrapper.json` -> latest wrapper version (separate from toolchain channels; wrapper is intentionally slow-rolling) + +#### Wrapper manifest + +`manifest/v1/wrapper.json` is not a channel pointer for the language. It is a separate `baml-wrapper` manifest consumed by `install.sh`, `install.ps1`, and `baml self-update`. + +```json +{ + "schema": 1, + "version": "0.1.0", + "released_at": "2026-05-22T19:00:00Z", + "artifacts": { + "aarch64-apple-darwin": { + "url": "https://github.com/BoundaryML/baml/releases/download/baml-wrapper-0.1.0/baml-wrapper-0.1.0-aarch64-apple-darwin.tar.gz", + "sha256": "abc123..." + }, + "x86_64-unknown-linux-gnu": { "url": "...", "sha256": "..." }, + "x86_64-unknown-linux-musl": { "url": "...", "sha256": "..." }, + "aarch64-unknown-linux-gnu": { "url": "...", "sha256": "..." }, + "aarch64-unknown-linux-musl":{ "url": "...", "sha256": "..." }, + "x86_64-apple-darwin": { "url": "...", "sha256": "..." }, + "x86_64-pc-windows-msvc": { "url": "...", "sha256": "..." }, + "aarch64-pc-windows-msvc": { "url": "...", "sha256": "..." } + } +} +``` + +Wrapper manifest requirements: + +- **Required**: `schema`, `version`, `released_at`, `artifacts`. +- **Required keys inside `artifacts`**: `url`, `sha256`. URL must be HTTPS. SHA-256 is lowercase hex, 64 chars. +- **Required archive layout**: exactly one executable at `bin/baml` (`bin/baml.exe` on Windows), plus optional license/readme files. No `baml-cli`, no `baml-pack-host`, no VSIX. +- **Versioning**: wrapper versions are normal SemVer and intentionally independent from BAML language versions. `0.1.0` of the wrapper can install `0.11.1-nightly.20260522.a` of the toolchain. +- **Managed installs**: wrappers installed by Homebrew/AUR refuse `baml self-update`; those users update the wrapper via their package manager. Curl-installed wrappers may self-update by reading `wrapper.json`. +- **Obsolete-wrapper behavior**: if a wrapper sees a manifest schema it cannot read, it must not guess or auto-upgrade. It prints the correct wrapper-update command for its install source and exits. + +### 1.3 Hosting — reuse `pkg.boundaryml.com` (no new infrastructure) + +`pkg.boundaryml.com` already has the exact shape we need: + +- S3 bucket with static website hosting + public read ([tools/pkg_boundaryml_com/lib/pkg-boundaryml-com-stack.ts](tools/pkg_boundaryml_com/lib/pkg-boundaryml-com-stack.ts) L17-24) +- Cloudflare-proxied DNS / HTTPS for `pkg.boundaryml.com`, managed outside this repo. The release plan depends on the domain remaining proxied, but does **not** require Cloudflare Cache Rules for v1. +- IAM role `pkg-boundaryml-com-github-release` assumable via GitHub OIDC. It grants CI/CD write access to the `pkg.boundaryml.com` S3 bucket for release-owned files. v1 narrows the trust policy to the `canary` branch only because both nightly and canary BAML language releases are decided from successful `canary` branch CI. GitHub tags/releases are created as records, not used as publish inputs. +- CDK stack with `cdk deploy` lifecycle + +Already done in this branch / prod: + +- [pnpm-workspace.yaml](pnpm-workspace.yaml) includes `tools/pkg_boundaryml_com`, so the infra package can be addressed with `pnpm --filter @baml/pkg-boundaryml-com-infra ...`. +- [tools/pkg_boundaryml_com/lib/pkg-boundaryml-com-stack.ts](tools/pkg_boundaryml_com/lib/pkg-boundaryml-com-stack.ts) narrows the GitHub OIDC trust policy for `pkg-boundaryml-com-github-release` to this exact subject: + + ```text + repo:BoundaryML/baml:ref:refs/heads/canary + ``` + +- The prod stack has been deployed with that trust-policy change using `boundaryml-prod`. Operational command shape: + + ```bash + AWS_PROFILE=boundaryml-prod CDK_DEFAULT_ACCOUNT=277707123528 CDK_DEFAULT_REGION=us-east-1 pnpm --filter @baml/pkg-boundaryml-com-infra run diff + AWS_PROFILE=boundaryml-prod CDK_DEFAULT_ACCOUNT=277707123528 CDK_DEFAULT_REGION=us-east-1 pnpm --filter @baml/pkg-boundaryml-com-infra run deploy + ``` + +Future maintainers: if the repo later introduces a long-lived canary branch that should publish release manifests directly, replace the trust policy with a narrow allowlist containing `repo:BoundaryML/baml:ref:refs/heads/canary` and `repo:BoundaryML/baml:ref:refs/heads/`. Do not allow `refs/tags/*` unless tags become release inputs again by design. + +Remaining concrete changes: + +- No additional S3 write policy is required for `manifest/v1/`, `install.sh`, or `install.ps1`. The bucket policy already grants `s3:PutObject` to the role; the new workflow just writes those paths. +- Optionally output the bucket name as a CloudFormation export so the new release workflow can read it. Simpler v1 path: hardcode the bucket name in the workflow — it's already in the existing `release-pkg-boundaryml-com.yml` L21 as `pkgboundarymlcomstack-pkgboundarymlcomsitebucket4f-ybhvygkqittp`. +- Cache-control is set per uploaded object by the release job, not by Cloudflare rules: + - Mutable pointers and scripts (`manifest/v1/canary.json`, `manifest/v1/nightly.json`, `manifest/v1/wrapper.json`, `install.sh`, `install.ps1`): `Cache-Control: public, max-age=60, must-revalidate`. + - Immutable per-version snapshots (`manifest/v1/version/.json`): `Cache-Control: public, max-age=86400, immutable`. + - If Cloudflare caches these responses, it should respect the origin headers. If it does not cache JSON / scripts by default, correctness is unchanged because the wrapper also enforces its own manifest-cache TTL. +- CI publish job (Phase 2.5) uses the existing IAM role (`arn:aws:iam::277707123528:role/pkg-boundaryml-com-github-release`) and bucket. **No new GitHub secrets required** — the role is assumed via OIDC. +- The existing `release-pkg-boundaryml-com.yml` workflow is folded into the release graph (its sole behavior — upload `index.html` — becomes one of the assets the new `publish-pkg-boundaryml-com` job uploads). + +What we do NOT need to add: CloudFront distributions, Cloudflare Cache Rules, new GitHub secrets, separate buckets, separate stacks. DNS / HTTPS remain in Cloudflare; object storage and release writes remain in the existing S3 + OIDC setup. + +### 1.4 Install scripts + +Source files live at [scripts/install.sh](scripts/install.sh) and [scripts/install.ps1](scripts/install.ps1) in the repo; published at `https://pkg.boundaryml.com/install.sh` and `https://pkg.boundaryml.com/install.ps1` by the same publish job that uploads the manifest (Phase 2.5). + +Behaviors (both scripts): + +- User-scoped only. They must not use `sudo`, write system directories, mutate Homebrew/AUR/package-manager paths, install IDE extensions automatically, or write outside `BAML_HOME` except for user profile/PATH configuration. +- Defaults: + - Unix `BAML_HOME`: `$HOME/.baml` + - Windows `BAML_HOME`: `%USERPROFILE%\.baml` + - Unix wrapper: `$BAML_HOME/bin/baml` + - Windows wrapper: `%USERPROFILE%\.baml\bin\baml.exe` +- Required flags: + - `--channel ` (default `canary`) + - `--version ` (exact version; wins over `--channel`) + - `--wrapper-only` (advanced / package-maintainer mode; installs/updates `bin/baml` but skips toolchain bootstrap) + - `--no-modify-path` (CI/Docker) + - `--yes` (disable prompts / accept defaults; explicit consent for profile/PATH edits in piped/non-interactive installs) + - `--help` +- `install.ps1` exposes equivalent PowerShell-style aliases such as `--WrapperOnly`, `--NoModifyPath`, and `--Yes`; behavior is the same. +- Flag rules: + - `--version` wins over `--channel`. + - `--wrapper-only` skips toolchain bootstrap. + - `--no-modify-path` disables profile/PATH edits even when `--yes` is supplied. + - Do not add `--modify-path` in v1. + - Piped installs such as `curl ... | sh -s` are treated as non-prompting because stdin is the script stream. They install wrapper + default canary toolchain, do not edit shell profiles by default, and print PATH instructions. +- Detects platform via `uname -sm` (`install.sh`) or `$env:PROCESSOR_ARCHITECTURE` (`install.ps1`), reads `pkg.boundaryml.com/manifest/v1/wrapper.json` to find the matching wrapper download URL + sha256, downloads to `$BAML_HOME/bin/baml`, verifies sha256. +- Re-running the installer refreshes the curl-managed wrapper from `wrapper.json` before installing/using the requested toolchain. This is equivalent to `baml self-update` followed by the requested toolchain install/use flow. +- Validate wrapper archives before replacement: expected executable at `bin/baml` (`bin/baml.exe` on Windows), no `baml-cli`, no `baml-pack-host`, no VSIX, no absolute paths, no `..` paths, and no unsafe symlinks. +- Replace the wrapper atomically when possible. On Unix, extract/write to a temp path in the same directory, `chmod +x`, fsync where practical, then rename. On Windows, safely replace `%USERPROFILE%\.baml\bin\baml.exe`; if the executable is in use, use a safe replace/deferred replace path and print the action taken. +- Unless `--wrapper-only` is passed, immediately bootstraps the requested toolchain through the wrapper with one command: + + ```bash + "$BAML_HOME/bin/baml" toolchain use + ``` + + `--version` wins over `--channel`. The toolchain install path is still owned by the wrapper and uses the normal toolchain manifest (`canary.json`, `nightly.json`, or `version/.json`); the shell scripts do not know GitHub release URL patterns. +- Modeled on `rustup-init.sh`. Same flag conventions and exit-code stability for CI scripting. + +PATH behavior on Unix: + +- Installer writes a generated env file at `$BAML_HOME/env`: + + ```sh + export BAML_HOME="$HOME/.baml" + case ":$PATH:" in + *":$BAML_HOME/bin:"*) ;; + *) export PATH="$BAML_HOME/bin:$PATH" ;; + esac + ``` + +- Shell profile files only source that env file: + + ```sh + . "$HOME/.baml/env" + ``` + +- Profile edits are idempotent: if the exact source line already exists, do not add another. +- If `BAML_HOME/bin` is already on PATH, do not modify profile files unless `--yes` is explicitly supplied and the source line is missing. +- Interactive install: update PATH by default unless `--no-modify-path`. +- Piped/non-interactive install without `--yes`: do not edit profile files; print manual PATH instructions. +- Piped/non-interactive install with `--yes`: edit profile files by default unless `--no-modify-path` is also supplied. +- Shell targets: + - `zsh`: add source line to `~/.zshrc`; on macOS, also add to `~/.zprofile` if that file already exists. + - `bash`: add source line to `~/.bashrc`; on macOS, also add to `~/.bash_profile` if that file already exists. + - `fish`: write `~/.config/fish/conf.d/baml.fish` instead of editing generic shell files. + - Unknown shell: do not edit profiles; print manual PATH instructions. + +Piped install examples: + +```bash +curl -fsSL https://pkg.boundaryml.com/install.sh | sh -s +``` + +Installs/updates the wrapper, bootstraps the default canary toolchain with `baml toolchain use canary`, does not edit shell profiles, and prints: + +```text +BAML installed at ~/.baml/bin/baml + +Add BAML to your PATH by adding this to your shell profile: + + . "$HOME/.baml/env" + +Or run for this shell session: + + export PATH="$HOME/.baml/bin:$PATH" +``` + +```bash +curl -fsSL https://pkg.boundaryml.com/install.sh | sh -s -- --yes +``` + +Installs/updates the wrapper, bootstraps the default canary toolchain, and edits shell profile/PATH configuration by default. + +```bash +curl -fsSL https://pkg.boundaryml.com/install.sh | sh -s -- --yes --no-modify-path +``` + +Accepts defaults but still skips profile/PATH edits. + +PATH behavior on Windows: + +- Update user PATH only, never machine PATH. +- Do not require Administrator. +- Use `[Environment]::SetEnvironmentVariable("Path", ..., "User")`. +- Avoid duplicate entries. +- If PATH changes, tell the user to restart the terminal. +- `--NoModifyPath` skips PATH mutation. +- Non-interactive mode does not mutate user PATH unless `--Yes` is passed. +- Unless `--WrapperOnly` is passed, bootstrap with: + + ```powershell + & "$env:BAML_HOME\bin\baml.exe" toolchain use + ``` + +Stable exit codes: + +```text +0 success +1 general failure +2 unsupported platform +3 download/network failure +4 checksum verification failure +5 archive validation/extraction failure +6 PATH/profile update failure +7 toolchain bootstrap failure +``` + +Failure behavior: + +- If wrapper install succeeds but PATH update fails, leave the wrapper installed, print manual PATH instructions, and exit `6`. +- If wrapper install succeeds but toolchain bootstrap fails, leave the wrapper installed, print the exact `baml toolchain use ` command to retry, and exit `7`. +- If checksum or archive validation fails, do not replace the existing wrapper. +- Re-running the installer is idempotent: it refreshes the wrapper from `wrapper.json`, repairs PATH/env file entries if requested, and only bootstraps the requested toolchain when `--wrapper-only` is not passed. + +Acceptance: the following Dockerfile lines install cleanly on `python:slim`, `node:slim`, and Alpine: + +```dockerfile +RUN curl -fsSL https://pkg.boundaryml.com/install.sh | sh -s -- --version 0.11.0 --no-modify-path +ENV PATH="/root/.baml/bin:${PATH}" +``` + +Optional `get.baml.com` shortcut (deferred decision): if you want the shorter curl URL later, add a single DNS redirect from `get.baml.com` -> `pkg.boundaryml.com/install.sh`. No infra change, just one CNAME or HTTP-redirect record. Not in scope for v1. + +## Phase 2 - Workflow restructure + +Goal: replace [.github/workflows/release-baml-language-alpha.yml](.github/workflows/release-baml-language-alpha.yml) with a channel-aware, fan-out/fan-in/separate-publish pipeline. + +### 2.1 New release graph orchestrator + +Create `.github/workflows/release-baml-language.yml` as the release graph entrypoint (replaces the alpha file; old file deleted in same PR). It may call focused reusable/focused workflows for Rust, Python, TypeScript/VSIX, and publishing. Triggers: + +- `workflow_run` on successful `CI - BAML Language` push to the `canary` branch -> channel `nightly`, auto-publish +- If the source commit advanced `[release].canary_version` and that canary version has not already been published, the same workflow run also publishes `canary` for that commit. +- `workflow_dispatch` with at least `channel` and `dry_run` inputs for manual reruns / rehearsals. Production manual publishes use the same publish lock as automatic releases. +- GitHub tags named `baml-language-` are created by publish jobs as immutable storage/history anchors. They do **not** trigger BAML language releases. + +The workflow publishes only from actual `push` events / successful CI on `refs/heads/canary`, not from PR refs or `merge_group` branches. + +Concurrency rule: GitHub merge queue serializes normal writes to `canary`, but it is not the release lock. Manual dispatches, retries, admin bypasses, and slow publish jobs can still overlap. Add a non-cancelling concurrency group to the release graph entrypoint: + +```yaml +concurrency: + group: baml-language-release-canary + cancel-in-progress: false +``` + +Do not include the version, channel, run id, or commit SHA in the production group; those would allow overlapping runs and defeat the lock. `cancel-in-progress: false` is required so GitHub queues later releases instead of cancelling a run that may already be halfway through publishing. + +Dry-run releases may use the same group for simplicity or a separate dry-run group if dry-run validation should not block production: + +```yaml +concurrency: + group: ${{ inputs.dry_run == 'true' && 'baml-language-release-dryrun' || 'baml-language-release-canary' }} + cancel-in-progress: false +``` + +Production publish jobs still perform idempotency/uniqueness checks after acquiring the concurrency slot: check whether `baml-language-` already exists; treat an existing matching tag/release/artifact set as idempotent repair; hard-fail if content differs. Nightly suffix selection must happen inside the serialized release run, after the concurrency slot is acquired. + +### 2.2 Job graph + +```mermaid +flowchart LR + plan[plan: version, channel, refs] + plan --> buildCli + plan --> buildVsix + plan --> buildCore + plan --> buildWrapper + + subgraph builds [Build matrix] + buildCli[build cli + pack-host x N targets] + buildVsix[build vsix once] + buildCore[build baml_core wheels x python x platform] + buildWrapper[build wrapper x N targets
(compile/smoke every run; publish only if wrapper version changed)] + end + + buildCli --> gate + buildVsix --> gate + buildCore --> gate + buildWrapper --> gate + + gate[all-builds gate] + gate --> pubGh[publish GH release] + gate --> pubTap[publish homebrew-tap
(wrapper releases only)] + gate --> pubAur[publish AUR wrapper packages
(wrapper releases only)] + gate --> pubPypi[publish PyPI baml_core] + gate --> pubManifest[publish manifest to boundaryml] +``` + +Each `publish-*` job: + +- Is independently re-runnable (`workflow_dispatch` with `job=` input + a stored manifest artifact) +- Reads the shared `release-manifest.json` written by the gate job +- Is the **only** code path that talks to the corresponding registry/tap/etc + +`publish-pypi` should be implemented through a focused workflow named `.github/workflows/publish-python-pypi.yml`, renamed from the current [.github/workflows/release-sdk.yaml](.github/workflows/release-sdk.yaml) Python `baml_core` release path. The name is intentionally language-plus-registry, so future SDK publish workflows can follow the same pattern: `publish-typescript-npm.yml`, `publish-ruby-rubygems.yml`, `publish-java-maven.yml`, etc. Because PyPI trusted publishing is bound to the GitHub workflow identity, update or explicitly validate the PyPI trusted-publisher binding for project `baml-core` as part of the rename. Keep OIDC trusted publishing; do not add username/password or API-token secrets. + +Implementation gate: pause before enabling or relying on the renamed `publish-python-pypi.yml` workflow and remind the user to update PyPI's trusted publisher configuration in the PyPI portal. The user should go to PyPI, open the `baml-core` project, navigate to the publishing/trusted-publisher settings, and update the GitHub workflow filename from `.github/workflows/release-sdk.yaml` to `.github/workflows/publish-python-pypi.yml` for the `BoundaryML/baml` repository and the intended publishing environment/ref. Do not proceed with a production PyPI publish until the user confirms this portal change is complete, or until a dry-run/validation proves the existing binding still authorizes the renamed workflow. + +### 2.3 Build matrix expansion + +Update the existing matrix in the release workflow to add `aarch64-pc-windows-msvc` and reuse the cross-compile pattern already used for `aarch64-unknown-linux-musl`. Keep `x86_64-apple-darwin` (low priority but currently included). + +Target list: + +- aarch64-apple-darwin (Tier 1) +- x86_64-apple-darwin (Tier 2, low priority) +- x86_64-unknown-linux-gnu (Tier 1) +- aarch64-unknown-linux-gnu (Tier 1) +- x86_64-unknown-linux-musl (Tier 1, Docker slim) +- aarch64-unknown-linux-musl (Tier 1, Docker slim ARM) +- x86_64-pc-windows-msvc (Tier 1) +- aarch64-pc-windows-msvc (Tier 1, NEW) + +Tier labels are priority/support labels only. If a target is in this release matrix, it is required for publish in v1; any build, archive-layout, checksum, or host-runnable smoke failure blocks the entire release. + +### 2.4 Platform-neutral VSIX in every archive + +- New `build-vsix` matrix job runs `pnpm run vscode:package` in [typescript2/](typescript2/) once. +- Precondition for "build once": the VSIX must be platform-neutral. The replacement branch removes the current bundled CLI path: + - [typescript2/package.json](typescript2/package.json) `vscode:package:build` no longer runs `cargo build -p baml_cli`. + - [typescript2/package.json](typescript2/package.json) `vscode:package:stage` no longer creates `app-vscode-ext/dist/baml-cli` or copies `../baml_language/target/release/baml-cli`. + - [typescript2/app-vscode-ext/src/extension.ts](typescript2/app-vscode-ext/src/extension.ts) removes `getBundledCliPath`. + - [typescript2/app-vscode-ext/package.json](typescript2/app-vscode-ext/package.json) updates the machine setting from "path to `baml-cli`" to "path to the `baml` wrapper"; default fallback command is `baml`, not `baml-cli`. + - The extension launches the language server as `baml lsp`, allowing the wrapper to resolve `$BAML_VERSION`, `baml.toml [toolchain]`, and the user's default channel. The VSIX never pins a bundled CLI/LSP path; it uses the configured `baml` path or `baml` from `PATH`. + - VSIX process model: start one lazy VS Code `LanguageClient` / LSP process per BAML project root, not one global LSP and not one process per file. A BAML project root is the nearest ancestor of the active/open `.baml` file containing `baml.toml`; if none exists, fall back to the containing VS Code workspace folder; if the file is outside any workspace folder, use the file's directory as an ad-hoc root or run in limited-support mode. + - Maintain `Map`. When a `.baml` file opens or becomes active, find that file's BAML project root, start a client if one does not already exist, and launch it as `baml lsp` with `serverOptions.options.cwd = `. This makes wrapper toolchain resolution match the CLI behavior for that project. + - Project-root-per-client is required for monorepos where two sibling BAML projects may pin different toolchains. Workspace-folder-per-client is not sufficient because one workspace can contain multiple `baml.toml` roots. Nested roots route to the nearest BAML project root. + - To guard against VS Code document-selector leakage, each LSP receives its intended `projectRoot` in `initializationOptions.baml.projectRoot` and ignores documents whose nearest `baml.toml` root differs from that declared root. + - Playground ownership is per LSP process. Each project-root LSP owns its own playground server/port, and `baml.openPlayground` uses the client associated with the active editor's project root. Restart only the affected project-root client when that project's `[toolchain]` changes, the configured `baml` executable path changes, compatibility metadata changes after a wrapper/toolchain update, or that LSP crashes. Restart all clients only for truly global setting changes. + - If `baml` is missing or not executable, the extension shows an actionable install error and does not attempt to download its own LSP in v1. +- Existing per-target packaging step in the workflow (currently around L274) extends to copy `app-vscode-ext-.vsix` into `/assets/baml-vscode.vsix` before tar/zip. +- `baml ide install` consumes `~/.baml/toolchains//assets/baml-vscode.vsix` from the active toolchain by forwarding to the selected toolchain payload. +- Add explicit compatibility metadata for the VSIX/LSP/playground boundary. LSP compatibility rides on the existing LSP `initialize` request/result; playground compatibility is checked lazily when the playground WebSocket opens. The metadata includes BAML toolchain version for display/debugging, plus LSP/playground protocol ranges and capability flags for compatibility. This handles the important edge case where an old VSIX attempts to run a too-new wrapper/toolchain or a new VSIX opens an older project/toolchain without requiring an extension reinstall for every BAML release. Do not add VSIX activation-time network checks, extra `baml` process spawns, or a VSIX-owned downloader. +- Smoke check: unzip the VSIX and fail if it contains `dist/baml-cli/`, `baml-cli`, `baml-cli.exe`, `baml-pack-host`, or any native executable bits outside expected Node extension artifacts. + +### 2.5 `publish-pkg-boundaryml-com` job + +Single job that uploads all `pkg.boundaryml.com` assets for the release. Replaces both the manifest publishing logic and the existing standalone [.github/workflows/release-pkg-boundaryml-com.yml](.github/workflows/release-pkg-boundaryml-com.yml). + +Job steps: + +1. Assume the existing IAM role via OIDC (`arn:aws:iam::277707123528:role/pkg-boundaryml-com-github-release`). +2. Build the per-version manifest (Phase 1.2 schema) from artifacts produced upstream in the workflow. +3. `aws s3 cp` the following to bucket `pkgboundarymlcomstack-pkgboundarymlcomsitebucket4f-ybhvygkqittp`: + - `manifest/v1/version/.json` — write-once by content, `--cache-control "public, max-age=86400, immutable"`. If the object does not exist, upload it. If it exists, download and byte-compare against the newly generated canonical JSON; continue if identical, fail if different. + - `manifest/v1/.json` (`canary.json` or `nightly.json`) — overwrite, `--cache-control "public, max-age=60, must-revalidate"`. + - `manifest/v1/wrapper.json` — only when the wrapper version actually changed (rare); overwrite, `--cache-control "public, max-age=60, must-revalidate"`. + - `install.sh`, `install.ps1` — only when the install scripts changed; overwrite, `--cache-control "public, max-age=60, must-revalidate"`. + - `index.html` is replaced in the same branch with current install guidance or a redirect to docs. The standalone `release-pkg-boundaryml-com.yml` upload path is deleted, not preserved as a parallel release mechanism. +4. For a `baml-toolchain` release, gated on the toolchain GitHub release and PyPI publish completing successfully; Homebrew/AUR are not part of that release product. For a `baml-wrapper` release, gated on wrapper GitHub assets and package-manager publishing. Manifest upload is the **last** step for the relevant product. + +Rerun/idempotency rule: pointer updates (`canary.json`, `nightly.json`, `wrapper.json`) must be repairable on rerun even if the immutable per-version object already exists. Therefore, "object exists and matches" is success, not failure. "Object exists and differs" is a hard stop because it indicates two different builds attempted to claim the same version. This specifically covers the failure mode where `version/.json` uploaded successfully, but the job failed before updating `nightly.json`, `canary.json`, or `wrapper.json`; a rerun must be able to complete the pointer update. + +The CDK stack stays under [tools/pkg_boundaryml_com/](tools/pkg_boundaryml_com/) — same maintainers, same `cdk deploy` flow. No infrastructure changes required for the v1 plan. + +### 2.6 Version sync across artifacts + +The release version model has exactly one authoring source: + +```text +baml_language/release.toml +``` + +Every other version-bearing file is a stamped output. Package ecosystems still need local metadata (`pyproject.toml`, `package.json`, Rust constants, etc.), but those files are not independent sources of truth. They must either be updated by the canary bump flow or stamped from the CI release plan before building. + +#### Version authority chain + +1. **Human source of truth**: [baml_language/release.toml](baml_language/release.toml) + + ```toml + [release] + canary_version = "0.11.0" + ``` + +2. **Derivation tool**: [scripts/baml-language-version](scripts/baml-language-version) + - `bump --major|--minor|--patch`: updates `canary_version`, then runs the canary sync step. + - `compute --channel canary|nightly`: prints the canonical version. + - `compute --channel canary|nightly --pypi`: prints the PEP 440 version. + - `plan --channel canary|nightly --out release-plan.json`: writes a frozen release plan used by every CI job in the run. + - `stamp --plan release-plan.json`: writes every derived version surface in the checkout before build. + - `sync`: local canary convenience command; equivalent to planning/stamping `canary` from `release.toml`. + - `check`: fails if any committed canary surface disagrees with `release.toml`, if any public Rust/Node/Python/VSIX version API still reads Cargo package metadata, or if deprecated `BAML_RELEASE_VERSION` plumbing is reintroduced. + +3. **CI run artifact**: `release-plan.json` + + Generated once by the `plan` job and uploaded as a workflow artifact. Every matrix build downloads the same file and runs `scripts/baml-language-version stamp --plan release-plan.json` before building. This prevents Linux, macOS, Windows, Python, VSIX, and future Node/codegen jobs from independently recomputing or drifting. + + Example nightly plan: + + ```json + { + "schema": 1, + "channel": "nightly", + "canary_version": "0.11.0", + "canonical_version": "0.11.1-nightly.20260522.a", + "pypi_version": "0.11.1.dev2026052200", + "git_tag": "baml-language-0.11.1-nightly.20260522.a" + } + ``` + +#### Canary local flow + +Canary releases are explicit maintainer actions: + +```bash +scripts/baml-language-version bump --minor +scripts/baml-language-version check +``` + +The bump flow updates [baml_language/release.toml](baml_language/release.toml) and all committed canary version surfaces together. If a maintainer edits `release.toml` directly and forgets to run sync, `check` fails in CI with the list of mismatched files. + +#### Nightly CI flow + +Nightly releases never require a maintainer to edit a nightly version. CI derives the nightly version from `[release].canary_version`, freezes that derivation into `release-plan.json`, stamps the checkout in each build job, then builds from the stamped tree. These stamped nightly changes are build-local and are not committed back to the repository. + +`BAML_RELEASE_VERSION` is deprecated. It exists today in the alpha workflow and `baml-cli`, but the replacement pipeline removes it. The new design does not use an env var as a product-version source, even a temporary one. The only acceptable CI version handoff is `release-plan.json` plus stamped files. + +`scripts/baml-language-version` owns all format transformations: + +| Artifact | Transformation | Output example | +|---|---|---| +| Canonical BAML language version | identity | `0.11.0-nightly.20260522.a` | +| PyPI / PEP 440 | `0.11.0-nightly.YYYYMMDD.` -> `0.11.0.devYYYYMMDDLL` | `0.11.0.dev2026052200` | +| npm dist-tag (future) | `nightly` if pre-release else `latest` | n/a | +| Go module tag (future) | `v` prefix on git tag, no change inside `go.mod` | `sdk/go/v0.11.0-nightly.20260522.a` | +| C / C++ (future) | identity in generated `baml_version.h` | `0.11.0-nightly.20260522.a` | + +Wrapper package metadata (Homebrew/AUR) is generated from `baml_language/crates/baml/Cargo.toml` `[package].version`, not from `scripts/baml-language-version sync`. The wrapper crate must not use `version.workspace = true`, and the release graph fails if the wrapper version is not valid SemVer, is lower than the latest published `manifest/v1/wrapper.json` version, or if any workflow step tries to use `BAML_WRAPPER_VERSION` as a release authority. The old AUR `${VERSION//-/.}` transform belongs to the deleted language-toolchain package path. + +**Version surfaces that must stay in sync** (canary via the bump script; nightly via `release-plan.json` stamping): + +| File | Today's value | Patched to | +|---|---|---| +| [baml_language/release.toml](baml_language/release.toml) | `[channel] name = "alpha"` / `base_version` | `[release] canary_version = ""`; only human-authored version source | +| `release-plan.json` workflow artifact | (does not exist) | generated per run; contains `canonical_version`, `pypi_version`, `channel`, and `git_tag`; never committed | +| `baml_language/crates/baml_version/src/lib.rs` | (new crate/module) | generated constants such as `CANONICAL_VERSION`, `PYPI_VERSION`, `CHANNEL`; all Rust public version APIs consume this | +| [baml_language/sdks/python/pyproject.toml](baml_language/sdks/python/pyproject.toml) L3 | `version = "0.1.3"` | PEP 440 form | +| [typescript2/app-vscode-ext/package.json](typescript2/app-vscode-ext/package.json) L5 | `"version": "0.1.0"` | canonical SemVer (npm-compatible) | +| [baml_language/sdks/nodejs/bridge_nodejs/package.json](baml_language/sdks/nodejs/bridge_nodejs/package.json) L3 | `"version": "0.0.0-beta"` | canonical SemVer when the Node BAML language package is published | +| [baml_language/sdks/python/src/baml_core/__init__.py](baml_language/sdks/python/src/baml_core/__init__.py) | (no `__version__` today) | adds `__version__ = ""` so Python users see canonical even though PyPI stores the PEP 440 form | +| [baml_language/crates/baml_cli/src/commands.rs](baml_language/crates/baml_cli/src/commands.rs) `release_version()` | `option_env!("BAML_RELEASE_VERSION").unwrap_or(env!("CARGO_PKG_VERSION"))` | `baml_version::CANONICAL_VERSION` | +| [baml_language/sdks/python/rust/bridge_python/src/lib.rs](baml_language/sdks/python/rust/bridge_python/src/lib.rs) `get_version()` | `env!("CARGO_PKG_VERSION")` | `baml_version::CANONICAL_VERSION` | +| [baml_language/crates/bridge_cffi/src/ffi/runtime.rs](baml_language/crates/bridge_cffi/src/ffi/runtime.rs) `version()` | `env!("CARGO_PKG_VERSION")` | `baml_version::CANONICAL_VERSION` | +| [baml_language/sdks/nodejs/bridge_nodejs/src/lib.rs](baml_language/sdks/nodejs/bridge_nodejs/src/lib.rs) `get_version()` | `env!("CARGO_PKG_VERSION")` | `baml_version::CANONICAL_VERSION` | +| [baml_language/crates/bridge_wasm/src/lib.rs](baml_language/crates/bridge_wasm/src/lib.rs) `version()` | `env!("CARGO_PKG_VERSION")` | `baml_version::CANONICAL_VERSION` | +| [baml_language/crates/baml_lsp_server/src/lib.rs](baml_language/crates/baml_lsp_server/src/lib.rs) `version()` | `env!("CARGO_PKG_VERSION")` | `baml_version::CANONICAL_VERSION`, so VS Code server info reports the canonical toolchain version | + +Stamped Rust module shape: + +```rust +pub const CANONICAL_VERSION: &str = "0.11.1-nightly.20260522.a"; +pub const PYPI_VERSION: &str = "0.11.1.dev2026052200"; +pub const CHANNEL: &str = "nightly"; +pub const STABLE_VERSION: &str = "0.11.0"; +``` + +Every Rust crate that exposes a public BAML product version adds a normal workspace dependency on `baml_version` and reads these constants. The generated module is committed for canary releases by the local sync/bump flow; nightly CI stamps it in the temporary checkout before build. There is no `build.rs` shelling out to scripts and no environment-variable version override. + +**Files NOT patched** (intentional): + +- [baml_language/Cargo.toml](baml_language/Cargo.toml) workspace `version = "0.0.0-beta"` — stays. Cargo package metadata is not the public BAML language product version. +- Any `tools/versions/*.cfg` — engine-side; not touched by this plan. + +Version audit rule: any public, user-visible BAML language version API must report the same canonical version from the stamped surfaces. No public BAML language version API may read `CARGO_PKG_VERSION`, and no release workflow may pass `BAML_RELEASE_VERSION`. + +**Smoke-test gate contract** (extends existing `Smoke test version` step around L254-265 of the current alpha workflow): + +For each per-target build, the smoke-test step runs only when the build target equals the runner's host target (no cross-execution). Cross-built targets (e.g. `aarch64-unknown-linux-musl` built on x86_64 via `cross`) skip the binary-execution assertions and only check sha256 / archive layout. + +When the `baml-toolchain` archive is runnable on host, the step asserts (exact string equality, not semver-equal — drift is drift): + +```text +1. baml-cli --version | tail -n 1 == "baml-cli " +2. unzip -p assets/baml-vscode.vsix | head -c 4 == "PK\x03\x04" # ZIP magic +3. `baml pack` smoke test builds and runs a tiny host-target packaged executable, which verifies the `baml-pack-host` binary through its actual supported path. +``` + +When the `baml-wrapper` archive is runnable on host, the step asserts: + +```text +4. baml --version | tail -n 1 == "baml " +``` + +When the Python wheel is built (Linux x86_64 runner only — it's the canonical wheel-build host; other targets cross-build via `cibuildwheel`): + +```text +5. python -c "import baml_core; print(baml_core.__version__)" == "" +6. python -c "import baml_core; print(baml_core.get_version())" == "" +7. python -c "import baml_core; import importlib.metadata; print(importlib.metadata.version('baml_core'))" == "" +``` + +Add focused Rust tests for version helpers that are not naturally exercised by the Python wheel smoke test, especially `bridge_cffi::ffi::runtime::version` and `baml_lsp_server::version`. + +Archive-layout assertions (run on every target including cross-built): + +```text +8. sha256() == published sha256 (echo expected | sha256sum -c -) +9. Toolchain archive contains: bin/baml-cli[.exe], bin/baml-pack-host[.exe], assets/baml-vscode.vsix +10. Toolchain archive does NOT contain: bin/baml[.exe] +11. sha256() == published sha256 (echo expected | sha256sum -c -) +12. Wrapper archive contains: bin/baml[.exe] +13. Wrapper archive does NOT contain: bin/baml-cli[.exe], bin/baml-pack-host[.exe], assets/baml-vscode.vsix +14. Both archive types contain no path with "..", no absolute path, and no symlink pointing outside the archive. +``` + +The smoke-test step is **the** gate on `publish-*` jobs. If any assertion fails for any target, no publish job runs. Prevents the "engine and python silently drifted" class of bug and the "we shipped a broken archive" class of bug. + +`` and `` come from the single `release-plan.json` generated by the plan job. Build jobs read that artifact and stamp their checkout before compiling or packaging; they do not receive product versions through environment variables. + +### 2.7 `baml pack` unification with the release fetcher + +`baml pack` already downloads `baml-pack-host` binaries from GitHub Releases for cross-target packs (see [baml_language/crates/baml_cli/src/pack_command.rs](baml_language/crates/baml_cli/src/pack_command.rs) L481-621). The URL pattern, archive format, and SHA-256 sidecar are identical to what the wrapper will use for `baml toolchain install`. Pack is effectively a working prototype of the wrapper's fetcher; the plan unifies the two. + +**Today's pack download surface** (preserved as fallbacks + test hooks): + +- `BAML_PACK_HOST_RELEASE_BASE_URL` — full base URL override (air-gapped mirrors) +- `BAML_PACK_HOST_RELEASE_REPO` — override `BoundaryML/baml` repo +- `BAML_PACK_HOST_RELEASE_VERSION` — override the pack-host release version selected from the active toolchain's canonical version (kept for existing air-gapped/mirror workflows) +- Co-located fast path: `baml-pack-host` next to `baml-cli` is used directly when target == host (L488) + +**Changes**: + +1. **Extract a shared release fetcher** into a new `baml_language/crates/baml_release/` crate. Move from [baml_language/crates/baml_cli/src/pack_command.rs](baml_language/crates/baml_cli/src/pack_command.rs): + - `read_host_binary` (L481) + - `download_host_binary_from_release` (L509) + - `verify_release_archive_checksum*` (L530-585) + - `release_archive_url*`, `release_archive_filename`, `release_host_target_triple`, `validate_release_target_triple` (L594-663) + - `SUPPORTED_PACK_TARGETS` (L655) - expanded to match the new build matrix (Phase 2.3) including `aarch64-pc-windows-msvc`. + + Both `baml-cli` (for pack) and `baml-language/crates/baml/` (the wrapper) import this crate. No code duplication. + + **API shape** (no premature generality — only what both consumers actually need): + + ```rust + /// Identifies a release. + pub struct ReleaseSpec { + pub version: String, // canonical SemVer, e.g. "0.11.0-nightly.20260522.a" + pub target: String, // Rust target triple + } + + /// Resolved artifact entry from a manifest or fallback. + pub struct Artifact { + pub url: String, + pub sha256: String, + } + + /// 4-tier resolution chain. + pub enum Source { + BaseUrlOverride, // BAML_PACK_HOST_RELEASE_BASE_URL + LocalToolchain, // ~/.baml/toolchains//bin/ + Manifest, // pkg.boundaryml.com/manifest/v1/version/.json + GitHubReleasesFallback // hardcoded URL pattern + } + + pub struct Fetcher { /* sources, http client, cache dir */ } + + impl Fetcher { + /// Default chain used by both pack and the wrapper. + pub fn default_for(spec: ReleaseSpec) -> Self; + + /// Fetch the full per-target archive bytes (for `baml toolchain install`). + /// Side effects: sha256-verifies before returning. + pub fn fetch_archive(&self) -> Result>; + + /// Fetch and extract a named binary from the archive (for pack). + /// Side effects: sha256-verifies the archive, extracts in memory. + pub fn fetch_binary(&self, binary_name: &str) -> Result>; + + /// Download + extract + materialize on disk under ~/.baml/toolchains// + /// (for `baml toolchain install`). Idempotent — re-running with an already-installed + /// toolchain is a no-op. + pub fn install_to_toolchain_root(&self, root: &Path) -> Result; + } + ``` + + What's intentionally NOT in this crate (yet): + + - Manifest publish-side helpers. Manifest *generation* lives in the workflow (Python), not in Rust. + - Channel resolution (`canary` -> latest version). The wrapper owns that; pack always gets passed an explicit version. + - Editor/IDE concerns. Pure HTTP + extraction. + + **Error model** (`thiserror`-based, exhaustive — wrapper and pack both pattern-match on these): + + ```rust + #[derive(thiserror::Error, Debug)] + pub enum FetchError { + #[error("network error fetching {url}: {source}")] + Network { url: String, source: reqwest::Error }, + #[error("manifest 404 for version {version} (not released yet?)")] + ManifestNotFound { version: String }, + #[error("manifest schema {got} not supported (max {max}); run `baml self-update`")] + ManifestSchemaTooNew { got: u32, max: u32 }, + #[error("target {target} not built for version {version}")] + TargetNotInManifest { target: String, version: String }, + #[error("sha256 mismatch for {url}: expected {expected}, got {got}")] + ChecksumMismatch { url: String, expected: String, got: String }, + #[error("archive missing expected binary {name}")] + BinaryNotInArchive { name: String }, + #[error("disk error: {0}")] + Io(#[from] std::io::Error), + } + ``` + + **Retry and timeout policy** (kept simple — no exponential backoff complexity for v1): + + - HTTP client: `reqwest::blocking` (matches what pack uses today; no async runtime needed in the wrapper). + - Connect timeout: 10s. Read timeout: 60s for manifest, 600s for archives. + - Retries: 3 attempts with constant 2s sleep between, only on `Network` errors and HTTP 5xx. Never retry 4xx (404 is authoritative — version doesn't exist). + - User-Agent: `baml/` so we can debug from S3 access logs. + + **HTTP-level cache** is `etag`-based for `canary.json` and `nightly.json` (mutable pointers); per-version snapshots are immutable so we cache them forever. + + **On-disk cache layout** (shared cache/inventory shape used by both wrapper and pack; wrapper command resolution still uses `config.toml` + `state.toml` + `VERSION`, not manifest cache directly): + + ``` + ~/.baml/ + config.toml # wrapper user intent (see 1.1) + state.toml # active channel resolutions (see 1.1) + manifest-cache/ + prod/ + canary.json # mirrored channel pointer, TTL 24h locally + nightly.json # mirrored channel pointer, TTL 24h locally + version/ + 0.11.0-nightly.20260522.c.json # immutable; cached forever + override// # dry-run/mirror caches + toolchains/ + 0.11.0-nightly.20260522.c/ + bin/ + baml-cli[.exe] + baml-pack-host[.exe] + assets/ + baml-vscode.vsix + VERSION # exact canonical version for sanity checks + install.json # install source/checksum metadata + ``` + + The `VERSION` file is written by `install_to_toolchain_root` and read on every wrapper invocation as a tamper check (prevents `mv` between toolchain dirs from causing version-drift confusion). + + **Concurrency**: `Fetcher` is `Send + Sync`. Multiple `baml pack --target ` invocations or `baml toolchain install` runs against the same toolchain dir use a `.lock` file in `~/.baml/toolchains//` (`fs2::FileExt::try_lock_exclusive`) — second invocation waits up to 60s, then errors clearly. + +2. **Manifest-aware resolution chain.** Pack's URL/cache resolution becomes: + + ``` + 1. BAML_PACK_HOST_RELEASE_BASE_URL (kept; highest precedence) + 2. Local toolchain cache: ~/.baml/toolchains//bin/baml-pack-host[.exe] (NEW) + 3. Manifest: pkg.boundaryml.com/manifest/v1/version/.json -> URL + sha256 (NEW default) + 4. Hardcoded GitHub URL pattern (existing; kept as offline-resilient fallback) + ``` + + Same chain used by `baml toolchain install` in the wrapper. Step 2 means a developer who's installed toolchain `0.11.0-nightly.20260522.c` via `baml toolchain install` doesn't re-download for `baml pack --target `. The wrapper's `baml toolchain list` and `baml toolchain uninstall ` see and manage these binaries. + +3. **Updated cross-version error.** Existing message at L638-640 expanded to mention `baml toolchain install`: + + ``` + No released `baml-pack-host` artifact is available for -. + Install the matching toolchain via `baml toolchain install `, or install + `baml-pack-host` next to the `baml` binary for this platform. + ``` + +4. **`baml-cli` and `baml-pack-host` version coupling** stays explicit — both ship in the same per-target archive and both see the same stamped `baml_version::CANONICAL_VERSION` for that build. `baml-pack-host` remains an internal runtime host, not a user-facing CLI. Release verification exercises it through `baml pack`. + +Alpha workflow deletion is gated on this migration. Do not delete or disable [.github/workflows/release-baml-language-alpha.yml](.github/workflows/release-baml-language-alpha.yml) until the new release graph publishes equivalent per-target `baml-pack-host` archives, the per-version manifest contains those artifacts and checksums, and `baml pack --target ` succeeds against a dry-run or production new-graph release without relying on alpha-release assets. + +**On-disk pack envelope unchanged.** [baml_language/crates/baml_exec/src/envelope.rs](baml_language/crates/baml_exec/src/envelope.rs) (`PACK_SECTION_NAME`, `PackEnvelope`) is independent of the release infrastructure. The wire format between `baml pack` and `baml-pack-host` is not affected. + +### 2.8 Skip third test run on release + +- Audit [.github/workflows/release-baml-language-alpha.yml](.github/workflows/release-baml-language-alpha.yml) for redundant test invocations. Confirmed: only the per-target `--version` smoke test runs (no full test suite). Verify the same is true for the new workflow and document explicitly in the workflow header. If a test job is being added by the prepare step, remove it; release workflows only run smoke tests, never the full suite. + +### 2.9 Package-manager publishing (wrapper package only) + +Current AUR templates are language-toolchain packages and must be redrafted in the replacement branch. They cannot be carried forward as compatibility shims: + +- [packaging/aur/baml/PKGBUILD](packaging/aur/baml/PKGBUILD) currently checks out `baml-language-__VERSION_ORIG__`, builds `--bin baml-cli`, installs `/usr/bin/baml-cli`, and symlinks `/usr/bin/baml -> baml-cli`. New behavior: build only the wrapper binary (`--bin baml`) from a `baml-wrapper-` source archive/tag and install `/usr/bin/baml`. +- [packaging/aur/baml-bin/PKGBUILD](packaging/aur/baml-bin/PKGBUILD) currently downloads a `baml-language--.tar.gz` toolchain archive, installs `baml-cli`, and symlinks `baml`. New behavior: download a `baml-wrapper--.tar.gz` wrapper archive and install only `bin/baml`. +- AUR `pkgver` and Homebrew formula `version` track the wrapper version only. They are not derived from BAML language canary/nightly versions. +- No AUR/Homebrew nightly stream in v1. Nightly users get nightly toolchains through `baml toolchain use nightly` / `baml toolchain update`, backed by `manifest/v1/nightly.json`. This avoids Arch version-ordering traps where a nightly package can outrank the later canary package. +- Package-manager install behavior: + - The package artifact installs only the wrapper binary. + - Package install/reinstall/upgrade must not run `baml toolchain install`, `baml toolchain use`, or any other toolchain bootstrap. + - Package install/reinstall/upgrade must not write user-scoped `~/.baml` state. + - Caveats/install messages tell users to run: + + ```text + baml toolchain use canary + ``` + + This one command installs the current canary toolchain if missing and selects it. Package managers themselves do not install editor extensions as a side effect. + - Caveats/post-install messages should say: + + ```text + BAML wrapper installed. + + To install and select the current canary language toolchain: + baml toolchain use canary + + To use nightly: + baml toolchain use nightly + + IDE extension setup is explicit: + baml ide install --cursor + ``` + +- Homebrew: + - Repository: `BoundaryML/homebrew-tap`. + - Formula path: `Formula/baml.rb`. + - Package contents: installs only `bin/baml`. + - Formula version: wrapper version from `baml_language/crates/baml/Cargo.toml` `[package].version`. + - Formula source/archive URL: + + ```text + https://github.com/BoundaryML/baml/releases/download/baml-wrapper-/baml-wrapper--.tar.gz + ``` + + - Formula must not install, symlink, or reference `baml-cli`, `baml-pack-host`, or the VSIX. + - Formula must not run toolchain bootstrap in `post_install`. + - Publish method: release graph commits directly to `BoundaryML/homebrew-tap` using the existing `HOMEBREW_BAML_DISPATCH_TOKEN`, kept for compatibility with the current secret naming. The token needs contents write access to `BoundaryML/homebrew-tap`. +- AUR: + - Packages: `baml` and `baml-bin`. + - `baml-bin` downloads the prebuilt `baml-wrapper--.tar.gz` wrapper archive from GitHub Releases and installs only `bin/baml`. + - `baml` source-builds the wrapper from the `baml-wrapper-` source archive/tag, builds only `--bin baml`, and installs only `/usr/bin/baml`. + - Both AUR package versions track the wrapper version only. + - No AUR install hook writes `~/.baml` or runs toolchain install/use. + - AUR install messages should point users to `baml toolchain use canary`. + - Publish method: update the AUR package repositories over SSH, regenerate `.SRCINFO`, commit, and push. + - Required maintainer-configured AUR remotes: + + ```text + ssh://aur@aur.archlinux.org/baml.git + ssh://aur@aur.archlinux.org/baml-bin.git + ``` + + - Required CI secret: + + ```text + AUR_SSH_PRIVATE_KEY + ``` + +- `publish-homebrew` and `publish-aur` jobs are guarded by: + + ```text + wrapper_changed == true + dry_run == false + github.ref == refs/heads/canary + ``` + +- A `baml-toolchain` release must not dispatch or run Homebrew/AUR publishing. +- Package-manager installed wrappers refuse `baml self-update`; the wrapper prints the appropriate package-manager command instead (`brew upgrade baml`, `paru -Syu baml-bin`, etc.). +- Wrapper releases update Homebrew/AUR package definitions and `manifest/v1/wrapper.json`. Toolchain releases update `canary.json` / `nightly.json` / `version/.json` only. These paths are intentionally independent. +- Dry-run behavior: + - Generate the Homebrew formula and AUR `PKGBUILD` / `.SRCINFO` files. + - Upload generated package files as workflow artifacts. + - Do not commit to `BoundaryML/homebrew-tap`. + - Do not push to AUR. +- Required checks before publishing: + - Homebrew formula points at `baml-wrapper-`, not `baml-language-`. + - Homebrew formula version equals wrapper version. + - Homebrew formula installs only `bin/baml`. + - Homebrew formula has no `post_install` toolchain bootstrap. + - AUR `pkgver` equals wrapper version. + - AUR package sources point at wrapper release/source. + - AUR `PKGBUILD` installs only `baml`. + - AUR install hooks do not run `baml toolchain use` or `baml toolchain install`. + - Caveats/install messages include `baml toolchain use canary`. +- In interactive supported-IDE terminal contexts, package-manager caveats may print a one-time follow-up recommendation. In non-interactive/headless contexts, skip the recommendation unless the user explicitly asked for IDE setup: + + ```text + baml ide install --cursor + ``` + +## Phase 3 - IDE install, polish, deprecate old paths + +### 3.1 `baml ide install` (forwarded to the selected toolchain) + +Wrapper stays a pure pass-through (constraint). `ide install` is a new selected-toolchain subcommand added to [baml_language/crates/baml_cli/src/commands.rs](baml_language/crates/baml_cli/src/commands.rs) alongside the existing `Format`, `Describe`, `Generate`, `Test`, `Init`, etc. variants. Implementation lives in a new module `baml_language/crates/baml_cli/src/ide_command.rs`. User docs should say `baml ide install`, not `baml-cli`. + +Behavior: + +- VSIX path resolution: each `baml-cli` knows its own toolchain root (`exe_path.parent().parent()`); reads `/assets/baml-vscode.vsix`. No coupling to the wrapper or `~/.baml/` layout. +- V1 supports only Cursor and VS Code out of the box: + + ```text + baml ide install + baml ide install --cursor + baml ide install --code + ``` + + No v1 flags for `--windsurf`, `--all`, `--editor`, `--force`, or `--dry-run`. +- Probe behavior: + - `--cursor`: run `cursor --install-extension `. + - `--code`: run `code --install-extension `. + - No flag + only `cursor` found on PATH: install to Cursor. + - No flag + only `code` found on PATH: install to VS Code. + - No flag + both found + interactive terminal: prompt the user to choose. + - No flag + both found + non-interactive terminal: error and require `--cursor` or `--code`. + - Neither found: error with manual install commands and the VSIX path. +- Remove the manual extension-directory unzip fallback from v1. `baml ide install` installs only through supported editor CLIs. If no supported editor CLI is available, print the VSIX path and exact manual commands the user can run. +- Implementation may internally pass editor-specific flags needed for update/reinstall, such as `--force` if the editor CLI supports it, but those are not user-facing BAML flags in v1. +- User invocation goes through the wrapper unchanged: `baml ide install --cursor` -> wrapper execs the selected toolchain's `ide install --cursor` -> the selected toolchain installs the VSIX from its active toolchain. This is a convenience of pass-through, not wrapper-owned behavior. +- Toolchain install/update does not automatically install editor extensions. When a user runs any `baml` command from a supported IDE terminal, BAML may detect that context and print a short one-time recommendation if the extension appears missing: + + ```text + BAML IDE extension is not installed for this editor. + Run: baml ide install --cursor + ``` + + The recommendation must not block the command being run. Non-interactive contexts should not prompt, and the recommendation should be rate-limited or recorded so repeated terminal commands do not spam the user. +- Curl installer behavior: when interactive and `--wrapper-only` is not passed, `install.sh` / `install.ps1` bootstraps canary/nightly through `baml toolchain use ...`. It may print `baml ide install --cursor` as a follow-up when it can identify the IDE, but it must not install the extension unless the user explicitly runs the IDE command. +- Package-manager behavior: Homebrew/AUR install only the wrapper and do not bootstrap a toolchain. They may print `baml ide install --cursor` as an explicit follow-up, but they must not install editor extensions as a side effect. +- Marketplace publishing is deferred to Phase 4, but the new VSIX identity is fixed for v1: + + ```json + { + "publisher": "Boundary", + "name": "baml-language", + "displayName": "BAML", + "description": "Language support and playground for BAML." + } + ``` + + Stable extension ID: `Boundary.baml-language`. The release graph still builds and archives this VSIX artifact for every BAML toolchain release now, so later Marketplace publishing can consume the same artifact instead of inventing a separate build path. + +Tests: + +- Finds VSIX relative to active toolchain root. +- `--cursor` invokes `cursor --install-extension `. +- `--code` invokes `code --install-extension `. +- No flag + only Cursor detected installs to Cursor. +- No flag + only VS Code detected installs to VS Code. +- No flag + both detected + non-interactive errors and requires a flag. +- No supported CLI detected prints manual install commands and VSIX path. +- No test or implementation path manually unzips into editor extension directories. + +### 3.2 VSIX/LSP/playground compatibility protocol + +Goal: users should not need to install a new VSIX for every BAML toolchain release. The VSIX should remain a thin client that launches `baml lsp` through the wrapper, then validates protocol compatibility with the selected toolchain. Compatibility is based on explicit protocol integers and capability flags, not exact BAML semver equality. + +Old-engine prior art: [typescript/apps/vscode-ext/src/plugins/language-server-client/index.ts](typescript/apps/vscode-ext/src/plugins/language-server-client/index.ts) listens for `baml_src_generator_version`, resolves/downloads a matching `baml-cli`, and restarts the LSP when the project generator version changes. That confirms the old product needed project-driven toolchain alignment, but the new wrapper model should not recreate the VSIX-owned downloader/restart loop. Toolchain resolution belongs to the wrapper; extension compatibility belongs to the VSIX/LSP protocol boundary. + +Implementation boundaries: + +- VSIX launches `baml lsp` via the configured `baml` path or `baml` from `PATH`. It must not launch `baml-cli` directly and must not download toolchains itself. +- Wrapper resolves the selected installed toolchain before execing that toolchain's LSP. It does not auto-install or update toolchains from VSIX/LSP activation; if no matching toolchain is installed, the wrapper reports the normal local-state error and recommends `baml toolchain use `. +- VSIX/LSP compatibility is checked on the existing LSP `initialize` request/result. +- Playground compatibility is checked only when the playground opens and the webview connects to the playground WebSocket. +- The wrapper is not responsible for VSIX/playground compatibility decisions. + +Protocol constants: + +- Add a VSIX-side compatibility module, for example `typescript2/app-vscode-ext/src/compat.ts`, containing constants such as: + + ```ts + export const BAML_LSP_PROTOCOL_MIN = 1; + export const BAML_LSP_PROTOCOL_MAX = 1; + export const BAML_PLAYGROUND_PROTOCOL_MIN = 1; + export const BAML_PLAYGROUND_PROTOCOL_MAX = 1; + ``` + +- Add an LSP-side protocol module, for example `baml_language/crates/bex_project/src/bex_lsp/protocol.rs`, containing constants such as: + + ```rust + pub const BAML_LSP_PROTOCOL_VERSION: u32 = 1; + pub const BAML_PLAYGROUND_PROTOCOL_VERSION: u32 = 1; + pub const MIN_SUPPORTED_VSCODE_LSP_PROTOCOL: u32 = 1; + pub const MIN_SUPPORTED_PLAYGROUND_PROTOCOL: u32 = 1; + ``` + +- Add a playground-side compatibility module, for example `typescript2/pkg-playground/src/compat.ts` or `typescript2/pkg-playground/src/protocol.ts`, containing the playground protocol constants and range-check helper. +- Bump a protocol number only for breaking changes where an old peer would misbehave, crash, render wrong data, or send invalid requests. Additive fields/messages should use capability flags and must not require a protocol bump when old peers can safely ignore them. +- BAML toolchain version is included for display/debugging and diagnostic messages. It is not the compatibility gate. + +LSP initialize shape: + +- [typescript2/app-vscode-ext/src/extension.ts](typescript2/app-vscode-ext/src/extension.ts) wires `initializationOptions.bamlClient` into the `LanguageClientOptions`: + + ```json + { + "kind": "vscode", + "extensionVersion": "0.x.y", + "supportedLspProtocol": { "min": 1, "max": 1 }, + "supportedPlaygroundProtocol": { "min": 1, "max": 1 }, + "capabilities": ["openPlayground.v1", "listProjects.v1", "playgroundWebSocket.v1"] + } + ``` + +- [baml_language/crates/bex_project/src/bex_lsp/multi_project/request.rs](baml_language/crates/bex_project/src/bex_lsp/multi_project/request.rs) already builds `InitializeResult`. Keep that as the attachment point for BAML metadata under `capabilities.experimental.baml`: + + ```json + { + "toolchainVersion": "0.11.0", + "lspProtocol": 1, + "minSupportedClientLspProtocol": 1, + "playgroundProtocol": 1, + "minSupportedClientPlaygroundProtocol": 1, + "capabilities": ["openPlayground.v1", "listProjects.v1", "playgroundWebSocket.v1"] + } + ``` + +- Compatibility is range overlap: the client can talk to the server if the server protocol version is inside the client-supported range and the client max is at least the server's minimum supported client protocol. Equivalent logic applies to playground protocol. +- [typescript2/app-vscode-ext/src/extension.ts](typescript2/app-vscode-ext/src/extension.ts) should only wire this through and update extension state. The range-checking logic should live in `typescript2/app-vscode-ext/src/compat.ts`. +- If LSP protocol is incompatible, show an IDE diagnostic and avoid BAML custom requests. If only playground protocol is incompatible, keep the language server usable and disable/show a targeted error for playground entrypoints. +- Be careful with `serverInfo.version`: today `baml_language` crates can report crate/workspace versions such as `0.0.0-beta`. The implementation must ensure the displayed `toolchainVersion` comes from the unified BAML language release version selected by the wrapper/toolchain, not an unrelated internal crate version. + +Playground WebSocket shape: + +- [baml_language/crates/baml_lsp_server/src/playground_ws.rs](baml_language/crates/baml_lsp_server/src/playground_ws.rs) defines server/client message shapes. Add a server-to-client `hello` variant. A `clientHello` variant may be added for future bidirectional negotiation/capability reporting, but it must not be required on the v1 startup path. +- [baml_language/crates/baml_lsp_server/src/playground_server.rs](baml_language/crates/baml_lsp_server/src/playground_server.rs) currently sends `{ "type": "ready" }` immediately on connection. New v1 behavior: send a `hello` message containing `toolchainVersion`, `playgroundProtocol`, `minClientPlaygroundProtocol`, and capabilities, then send the existing `ready` message without waiting for a client round trip. The webview validates `hello` locally and suppresses playground behavior if incompatible. +- [typescript2/pkg-playground/src/ports/WebSocketRuntimePort.ts](typescript2/pkg-playground/src/ports/WebSocketRuntimePort.ts) receives `hello`, checks range overlap using the playground compatibility helper, and surfaces a clean compatibility error state instead of failing later on unknown message shapes. +- New webview connecting to old server: if the first message is `ready` without `hello`, treat the server as legacy protocol 0. Either keep an intentional legacy mode or show a "toolchain too old for this playground" error. +- Old webview connecting to new server: current TS drops unknown server messages and continues on `ready`, so the server sends `hello` followed by `ready` in v1. If a future breaking playground protocol cannot support old webviews safely, that future protocol may stop sending `ready` and return a clear error/close code instead. + +Performance rules: + +- Do not run `baml --version`, `baml toolchain list`, `baml toolchain update`, `baml toolchain install`, or any network command from VSIX activation for compatibility checking. +- Do not add an extra post-initialize LSP request solely for compatibility when the metadata can fit in `initialize`. +- Do not block editor startup on playground-only compatibility. Validate playground protocol lazily when the playground opens. +- Compute compatibility once per LSP session and once per playground WebSocket session. Do not range-check on every message. +- No compatibility check should hit `pkg.boundaryml.com`, GitHub Releases, PyPI, npm, or the manifest cache. + +Tests/merge criteria for this subsection: + +- Current VSIX + current toolchain: LSP starts, status bar shows toolchain version, playground opens. +- New VSIX + legacy `ready`-only playground server: either explicit legacy mode works or a clear "toolchain too old" playground error is shown. +- Old VSIX + new playground server: migration path still reaches `ready` when protocol 1 is compatible. +- New VSIX + server advertising higher LSP protocol than supported: custom BAML requests are disabled and the user sees "update the BAML extension" or equivalent. +- New VSIX + server advertising incompatible playground protocol but compatible LSP protocol: language features keep working and only playground entrypoints are disabled/error. +- No test or implementation path for compatibility checking spawns extra `baml` commands or performs network access during VSIX activation. + +### 3.3 Wrapper self-update flow + +- `baml self-update` reads the wrapper's own manifest (separate from toolchain manifests because wrapper versioning is intentionally slow; lives at `https://pkg.boundaryml.com/manifest/v1/wrapper.json`). The wrapper does not need separate canary/nightly tracks — it's effectively one rolling canary version. +- Curl-installed wrapper: download the latest host-target `baml-wrapper` archive, verify sha256, unpack `bin/baml`, then atomically replace `$BAML_HOME/bin/baml` (write temp file, fsync where practical, rename into place on Unix; Windows uses a safe replace/deferred replace path). +- Package-manager-installed wrapper: refuse and print the package-manager command (`brew upgrade baml`, `paru -Syu baml-bin`, etc.). The wrapper must not mutate `/usr/bin`, `/opt/homebrew`, or other package-manager-owned paths. +- Normal command execution never performs a silent wrapper self-update. If the wrapper is too old for a manifest schema, it exits with an actionable message: + + ```text + error: this BAML wrapper is too old for manifest schema 2. + Run: baml self-update + ``` + + For managed installs, replace the final command with the package-manager upgrade command. + +### 3.4 Deprecation banners and workflow cleanup + +- Old install command (`brew install BoundaryML/baml/baml`) keeps working via GitHub repo redirect; on next `brew update` the formula's `caveats` block tells users to re-tap as `boundaryml/tap/baml`. Banner lands with the tap rename and is removed after the migration window. +- Workflow files [release-baml-language-alpha.yml](.github/workflows/release-baml-language-alpha.yml), [release-pkg-boundaryml-com.yml](.github/workflows/release-pkg-boundaryml-com.yml), and [release-cli.yaml](.github/workflows/release-cli.yaml) are deleted on the replacement branch before merge to `canary`. [release-sdk.yaml](.github/workflows/release-sdk.yaml) is renamed to `.github/workflows/publish-python-pypi.yml` and invoked by the release graph after the PyPI trusted-publisher binding for `baml-core` is updated or validated. The rollback path is reverting the branch merge, which restores the old files exactly as they were. + +### 3.5 Docs + +- Create in-repo implementation/user-flow docs under `TASK/docs/`. There is no established public `baml_language` documentation home yet; public product-docs migration is a later product/docs decision and should not block this release-infra implementation. +- `TASK/docs/install.md` + - Audience: users, support, and people testing the new installer. + - Required contents: `brew install boundaryml/tap/baml` installs the wrapper only; package-manager installs do not bootstrap toolchains; `curl ... | sh -s` installs the wrapper and default canary toolchain; Docker/CI install example using `--no-modify-path`; PATH behavior and `~/.baml/env`; `baml toolchain use canary`; `baml toolchain use nightly`; `baml toolchain install `; `baml toolchain update`; `baml self-update`; package-manager wrapper upgrade behavior (`brew upgrade baml`, AUR upgrade); IDE setup with `baml ide install --cursor` and `baml ide install --code`. +- `TASK/docs/release-maintainer.md` + - Audience: maintainers and implementation agents. + - Required contents: release graph overview; canary vs nightly model; how to bump canary; how nightly suffixes are chosen; release workflow concurrency rule; dry-run release workflow and `BAML_MANIFEST_BASE_URL`; production publish guards; PyPI trusted-publisher portal pause; Homebrew/AUR wrapper release flow; rerun/idempotency behavior; rollback / mutable pointer repair flow. +- `TASK/docs/toolchain-system.md` + - Audience: engineers. + - Required contents: wrapper vs toolchain product boundary; `~/.baml` layout; `config.toml` user intent; `state.toml` active channel resolutions; manifest cache role; manifest schema; network/cache policy; `toolchain use` vs `toolchain install` semantics; VSIX/LSP/playground compatibility protocol; one-LSP-per-BAML-project-root model; direct `baml-cli` warning behavior; `baml pack` fetcher unification. +- Migration notes for `homebrew-baml` users. +- Reference to PEP 440 translation table for Python users who pin exact nightly versions. + +## Phase 4 - Future / explicitly deferred + +Not in this plan, called out so they're tracked: + +- npm SDK publish path. Plumbing: add a `publish-npm` job that reads the shared release manifest; package source lives separately in `typescript2/` so it can publish independently when ready. +- apt repo (`apt install baml`). Until then, the bash installer covers Linux. +- Code signing / notarization (macOS, Windows). Significant build-matrix complexity; add when there's actual user friction. +- "Release in a branch with zeroed sub-versions" idea: explicitly rejected; problem solved instead by patching versions in CI rather than committing them. +- Marketplace publishing and old-extension deprecation. V1 ships a stable, distinct, toolchain-bundled VSIX with extension ID `Boundary.baml-language`, installed explicitly by `baml ide install`; it is not published to the VS Code Marketplace in v1. The old Marketplace extension remains untouched. No old-extension detection, warning, disable, uninstall, or migration behavior is part of v1. If a user has both extensions installed and sees duplicate behavior, docs/support guidance is to disable one manually. + +## Risks and open follow-ups + +- **PyPI translation collision.** If two nightly builds on the same day target the same letter index due to a tag-cleanup mistake, PEP 440 collision is possible (`0.11.0.dev2026052200` already exists on PyPI). Mitigation: the `letter` selector in `scripts/baml-language-version` validates uniqueness against existing PyPI versions of `baml_core` via the simple JSON API (`https://pypi.org/pypi/baml_core/json`) before publishing. The workflow runs this check in the `prepare` job before any build kicks off. +- **Letter overflow.** `z` is the 26th nightly in a single calendar day, which is well above any realistic merge cadence. Mitigation: hard-fail at `z` with this operational playbook: + - The `compute` step in `scripts/baml-language-version` returns exit code 2 (distinct from "no work to do" exit 1) with a message: + `error: 26 nightly builds today for base 0.11.1; wait for the next UTC day or cut a new canary release by bumping baml_language/release.toml [release].canary_version.` + - The workflow's `prepare` job surfaces this as a workflow-level failure with a link to the operational doc. + - Two-letter scheme (`aa`, `ab`...) is **explicitly not** implemented in v1: it would require a parallel PEP 440 numbering scheme (3-digit `LL` becomes ambiguous) and a manifest schema bump. We'd rather catch it as a forcing function for a base_version bump. + - If the 27th merge on the day is truly time-critical, the operational override is: cut a new canary release by bumping `[release].canary_version` (for example `0.11.0` -> `0.11.1`), commit, and push to `canary`. The nightly base then derives to the next patch (`0.11.2`), and the letter resets to `a`. This is intentionally a real release decision, not a hidden nightly-only edit. +- **VSIX size.** Bundling VSIX into every target archive adds ~MB per platform. Acceptable; called out for future split if the bundle grows. Alternative: VSIX as a separate per-release asset, downloaded by `baml ide install` on demand. +- **`baml.toml` schema drift.** Adding `[toolchain]` is a non-breaking extension (existing files without the table keep working). Document the table in the same place as `[package]` and `[scripts]`. +- **Pack/wrapper fetcher divergence.** Two separate code paths fetching the same artifacts are guaranteed to drift over time (different retry behavior, different proxy support, different cache semantics). Mitigation: shared crate in Phase 2.7 from day one of pack-aware manifest support. CI test that pack and wrapper both successfully fetch the same target archive from the same manifest. +- **Pack env override compatibility.** The three `BAML_PACK_HOST_RELEASE_*` env vars are likely in use by users today for air-gapped builds, internal mirrors, and CI test fixtures. The shared fetcher preserves all three at the highest precedence so existing setups continue working; the wrapper inherits the same overrides for symmetry. +- **VSIX compatibility startup cost.** A naive implementation could slow editor startup by spawning `baml --version`, querying toolchain manifests, or performing a second LSP compatibility request. Mitigation: protocol compatibility is constant-based metadata on the existing LSP `initialize` result, and playground protocol validation is lazy on WebSocket connect. No compatibility check performs network access. +- **Workspace version footgun.** `baml_language/Cargo.toml` workspace `version = "0.0.0-beta"` is intentionally never patched. This works only because every `baml_language` crate is `publish = false` except [tools_stow](baml_language/crates/tools_stow/Cargo.toml) (which has its own hardcoded `0.1.0`). If anyone flips `publish = true` on another crate without first deciding the versioning story, they'd publish a confused `0.0.0-beta` to crates.io. PR 2 adds a `cargo metadata` check in `scripts/baml-language-version check` that fails CI if any non-allow-listed crate has `publish = true` while the workspace version is `0.0.0-beta`. +- **Two VSCode extensions exist.** [typescript/apps/vscode-ext/](typescript/apps/vscode-ext/) (`baml-extension`, on Marketplace at version 0.222.0) is the engine-side extension; [typescript2/app-vscode-ext/](typescript2/app-vscode-ext/) is the new baml_language extension bundled in toolchain archives. The new VSIX has stable ID `Boundary.baml-language`, publisher `Boundary`, name `baml-language`, and display name `BAML`. It is not published to Marketplace in v1, and `baml ide install --cursor` / `baml ide install --code` install this new toolchain-bundled VSIX only. Old-extension deprecation, Marketplace publishing, and migration messaging are product decisions tracked under Phase 4. +- **`pkg.boundaryml.com` index page.** The legacy job uploads `tools/pkg_boundaryml_com/data/index.html` (currently a static install instructions page). The replacement branch updates or redirects that page in the same cutover so the standalone upload workflow can be deleted without leaving stale install instructions behind. + +## Cutover sequencing + +This work lands on a dedicated feature branch and merges to `canary` only when the branch contains the complete replacement. We do **not** run two production language-release systems side by side, and we do **not** carry an alpha-era compatibility switch forward. + +The end state after merge to `canary`: + +- One coherent BAML language release graph, with one release plan and fan-out/fan-in gates. It may be implemented as an orchestrator plus focused reusable/focused workflows for Rust, Python, TypeScript/VSIX, and publishing. +- Deleted old language-release workflows: + - `.github/workflows/release-baml-language-alpha.yml` + - `.github/workflows/release-pkg-boundaryml-com.yml` + - `.github/workflows/release-cli.yaml` +- `.github/workflows/release-sdk.yaml` is renamed to `.github/workflows/publish-python-pypi.yml`, invoked by the release graph, and covered by an updated or validated PyPI trusted-publisher binding for `baml-core`. It must not remain as an independent, separately-versioned release path. +- One version script: `scripts/baml-language-version`. +- One shared fetcher crate: `baml_release`. +- One manifest system and one hosting domain: `pkg.boundaryml.com`. +- Engine release workflows remain untouched. + +### Branch milestones + +| Milestone | Scope | Risk | +|---|---|---| +| 1 | Infra prep: `pkg.boundaryml.com` CDK trust-policy narrowing to the `canary` branch and `pnpm` workspace inclusion for the CDK package are already done in this branch / deployed to prod; Homebrew tap rename remains a low-risk standalone prep item. | Low | +| 2 | Versioning foundation: `release.toml` canary intent schema, `scripts/baml-language-version` rewrite (`canary` / derived `nightly`, nightly suffix selection, `--pypi`, `release-plan.json`, `stamp`, `sync`, metadata checks), stamped `baml_version` module, removal of `BAML_RELEASE_VERSION`, and `baml_release` extraction from pack with no behavior change. | Low-Med | +| 3 | Wrapper and manifest foundation: new `baml_language/crates/baml/`, manifest schema structs, install/list/use/uninstall/update/self-update skeletons, `baml.toml [toolchain]` parsing, test fixtures. | Med | +| 4 | New release graph from scratch: plan job, build matrices, smoke gate, separate publish jobs, toolchain manifest publish, Python `baml_core` publish, wrapper archives, Homebrew/AUR wrapper publishing only on wrapper-version changes, VSIX packaging, and archive-layout checks. | High | +| 5 | Hard cleanup on the same branch: delete the old alpha/pkg/CLI release workflows, rename `release-sdk.yaml` to `publish-python-pypi.yml`, update or validate the PyPI trusted-publisher binding, and remove old install-page behavior that is replaced by `install.sh` / `install.ps1` and docs. No old release code remains as an independent fallback path inside the repo. | Med | +| 6 | Polish: `baml ide install`, wrapper self-update/refuse-when-managed, docs for install matrix, channel selection, `baml.toml [toolchain]`, PyPI translation, and migration notes. | Low | + +### Branch validation + +The new release graph is validated on the feature branch without publishing production artifacts: + +- `push` / `pull_request` rehearsal paths run build + smoke-test jobs only. +- `workflow_dispatch` supports an explicit dry-run mode with at least: + + ```text + channel: nightly | canary + dry_run: true + ``` + + Example: + + ```bash + gh workflow run release-baml-language.yml \ + -f channel=nightly \ + -f dry_run=true + ``` + +- Dry-run mode builds, packages, and smoke-tests exactly like production, but publish jobs do not write production channel pointers and do not create production user-visible release state. +- Dry-run manifests and artifacts are uploaded under an isolated namespace, for example: + + ```text + https://pkg.boundaryml.com/dryrun//manifest/v1/nightly.json + https://pkg.boundaryml.com/dryrun//manifest/v1/version/.json + https://pkg.boundaryml.com/dryrun//artifacts/ + ``` + + They may instead be kept as workflow artifacts when public HTTP access is not required. If wrapper install validation is required, the dry-run manifest must point at HTTP-accessible dry-run artifact URLs. +- The workflow summary prints the dry-run manifest base URL and example validation command: + + ```bash + BAML_HOME="$(mktemp -d)" \ + BAML_MANIFEST_BASE_URL="https://pkg.boundaryml.com/dryrun//manifest/v1" \ + baml toolchain use nightly + ``` + + Use a temporary `BAML_HOME` so production user/developer state is not polluted. +- Production publish jobs are guarded by ref checks, not by a long-lived kill switch: + - nightly publish only from `refs/heads/canary` + - canary publish only from `refs/heads/canary` when `[release].canary_version` advanced and that canary version has not already been published + - GitHub tags/releases are produced by publish jobs and ignored as workflow triggers +- The old alpha workflow continues to exist only on `canary` until the replacement branch merges. It is not modified into a second compatibility system. + +### Merge criteria + +Merge the branch to `canary` only when all are true: + +1. The new workflow completes branch rehearsal builds and smoke tests on macOS, Linux, and Windows. +2. The generated manifest is byte-identical (modulo `released_at`) to a checked-in reference fixture for at least one nightly version. +3. `baml toolchain use ` against a dry-run manifest, using `BAML_MANIFEST_BASE_URL` and a temporary `BAML_HOME`, installs and selects a toolchain from a clean macOS, Linux, and Windows VM. +4. `baml pack --target ` succeeds against the new release graph's dry-run manifest/artifacts and fetches `baml-pack-host` through the new shared fetcher path, not through alpha-release-only assets. +5. The branch deletes or integrates the old language-release workflows listed above, and `publish-python-pypi.yml` is invoked by the release graph rather than running as an independent separately-versioned publish path. +6. Engineer-on-call has read the rollback playbook and is available for the first nightly release after merge. + +### Rollback + +Rollback is code revert plus mutable pointer repair. `git revert ` restores workflow files, but it does not undo external release state. The playbook must distinguish immutable release artifacts from mutable pointers. + +Immutable artifacts are never overwritten or deleted during normal rollback: + +```text +manifest/v1/version/.json +GitHub release baml-language- +GitHub release baml-wrapper- +PyPI baml_core +``` + +If a bad version was published, leave the immutable record intact and move the channel pointer away from it. This preserves reproducibility and avoids disappearing versions. + +Mutable pointers may be repaired: + +```text +manifest/v1/canary.json +manifest/v1/nightly.json +manifest/v1/wrapper.json +install.sh +install.ps1 +index.html +Homebrew formula / AUR metadata, for wrapper releases +``` + +Channel pointer repair flow: + +1. Choose the last known good concrete version. +2. Download `manifest/v1/version/.json`. +3. Validate schema and target completeness. +4. Upload that JSON as `manifest/v1/canary.json` or `manifest/v1/nightly.json`. +5. Use mutable cache headers: + + ```text + Cache-Control: public, max-age=60, must-revalidate + ``` + +Wrapper rollback is separate. If a bad wrapper release was published, repair `manifest/v1/wrapper.json` and any package-manager definitions that point to it. Do not delete the bad `baml-wrapper-` GitHub release. Prefer publishing a newer fixed wrapper when package-manager version ordering makes repointing awkward. + +PyPI rollback is fix-forward. Do not try to delete/reuse a published `baml_core` version. Move channel pointers away from the bad release when possible and publish the next nightly/canary with a fix. + +Add an explicit rollback script or manual workflow, for example: + +```bash +scripts/baml-release-rollback point-channel \ + --channel nightly \ + --version 0.11.1-nightly.20260601.b + +scripts/baml-release-rollback point-channel \ + --channel canary \ + --version 0.11.0 + +scripts/baml-release-rollback point-wrapper \ + --version 0.1.0 +``` + +The rollback script must not publish new immutable version artifacts. It only validates existing immutable manifests/releases and repairs mutable pointers. + +If the release graph itself is broken: + +1. Stop or avoid further production publishes. +2. Repair user-facing mutable pointers if users are affected. +3. Revert the merge commit. +4. Confirm old workflow files are restored. +5. Run one controlled release or dry run if needed. + +Repo/workflow rollback: + +```bash +git revert +``` + +Effect: the old canary alpha release workflow files are restored by the revert, and the new release graph is removed. Because old release files are deleted/integrated only in the replacement branch, reverting the merge commit restores the previous production path cleanly. Short cache TTLs on mutable pointers help pointer repair reach wrappers quickly. diff --git a/TASK/USER-PATHS.md b/TASK/USER-PATHS.md new file mode 100644 index 00000000000..7bebfa5298c --- /dev/null +++ b/TASK/USER-PATHS.md @@ -0,0 +1,417 @@ +# BAML Language Release Pipeline Alignment + +We are replacing the current alpha-era release flow with a clean, unified release system for the new BAML language. + +The new system keeps Engine releases working as they do today, while giving BAML language users a modern install and update experience: + +- `brew install baml` installs the lightweight `baml` wrapper and bootstraps the latest canary BAML toolchain. +- `baml toolchain use nightly` lets users follow every successful `canary` branch release. +- `baml toolchain update` updates the active language toolchain. +- `baml self-update` updates only the wrapper. +- The VSIX uses the locally installed `baml` command and does not bundle its own platform-specific CLI. + +The release pipeline builds fast by fanning out independent jobs, then fans in through one smoke-test gate before publishing. The publishing jobs are separate and rerunnable, but they all consume the same release plan. That gives us speed without creating separate release systems. + +This design is intentionally future-looking. It does not implement `apt` or `dnf` now, but it creates the right primitives for them later: HTTP-hosted manifests, immutable per-version artifacts, checksums, and a package-manager-friendly wrapper. + +## Why We Are Doing This + +Today, BAML language releases grew out of the alpha flow. It works, but it is too easy to accumulate adjacent systems: + +- GitHub prerelease naming from the alpha era. +- Homebrew/AUR packaging that installs the language CLI directly. +- Python publishing that can drift from the CLI/toolchain version. +- VSIX packaging that currently bundles a platform-specific CLI. +- A standalone `pkg.boundaryml.com` publishing path. + +For the Turing-complete BAML language, we need one coherent release graph and one source of version truth. The goal is not necessarily "one giant workflow file." The goal is one release plan, one version decision, one manifest system, and no independent publishing paths that can contradict each other. + +Engine remains separate. Existing Engine users keep their current release pipeline. + +## What Success Looks Like + +For users: + +- New users can install BAML with one package-manager command and get a working canary CLI plus an explicit IDE setup follow-up when relevant. +- Nightly users can stay on the latest successful `canary` branch commit. +- Canary users are not exposed to nightly releases unless they opt in. +- Project teams can pin a toolchain in `baml.toml`. +- The VS Code/Cursor/Windsurf/other IDE extension uses the installed `baml` command, so it follows the same toolchain selection as the CLI. + +For maintainers: + +- Canary release intent is explicit and human-controlled. +- Nightly release versions are automatic and never manually edited. +- Python, VSIX, Rust, Node, and future SDK version surfaces are stamped from the same release plan. +- The pipeline is fast because build jobs run in parallel. +- Publishing is safe because all outputs pass through one smoke-test gate. +- Failed publish reruns are repairable. +- Rollback is a code revert, not a long-lived kill switch. + +For the codebase: + +- Old alpha-era workflows are deleted or integrated. +- Package managers install a wrapper, not a language toolchain. +- The wrapper and the toolchain have separate lifecycles. +- `BAML_RELEASE_VERSION` is removed as alpha-era plumbing. +- `baml-pack-host` stays internal and is verified through `baml pack`, not exposed as a user-facing CLI. + +## Product Model + +The release system has two products. + +| Product | What It Is | How Often It Changes | Who Installs It | +|---|---|---|---| +| `baml-wrapper` | Small `baml` binary that selects, installs, updates, and forwards to toolchains | Rarely | Homebrew, AUR, curl installer | +| `baml-toolchain` | Versioned language payload: `baml-cli`, `baml-pack-host`, and VSIX artifact | Every nightly and canary release | Installed by the wrapper | + +This split is the core of the design. + +Package managers publish the wrapper. The wrapper fetches toolchains from manifests. A nightly/canary language release does not require a Homebrew or AUR package update. + +## User Stories + +### New Canary User + +As a new user, I want to run: + +```bash +brew install baml +``` + +and end up with: + +- `baml` on my path. +- the current canary toolchain installed under the wrapper-managed toolchain directory. +- no automatic editor extension install as a side effect. +- a clear follow-up command such as `baml ide install --cursor` when BAML can detect a supported IDE terminal. + +The package artifact itself still contains only the wrapper. The canary toolchain is installed through the normal wrapper-owned manifest flow. + +### Existing Canary User + +As an existing canary user, I want: + +```bash +baml toolchain update +``` + +to update my active canary language toolchain, not the wrapper binary. + +If I installed the wrapper through Homebrew or AUR, wrapper updates come from: + +```bash +brew upgrade baml +``` + +or the equivalent package-manager command. + +### Nightly User + +As a nightly user, I want: + +```bash +baml toolchain use nightly +baml toolchain update +``` + +to keep me on the newest successful `canary` branch release. Every successful `canary` branch CI run publishes a nightly toolchain. + +Nightly versions are always ahead of the latest canary patch line. If canary is `0.11.0`, nightly releases are `0.11.1-nightly.YYYYMMDD.a`, `0.11.1-nightly.YYYYMMDD.b`, and so on. + +### Project Maintainer + +As a maintainer of a BAML project, I want to pin a known-good toolchain in `baml.toml`: + +```toml +[toolchain] +channel = "nightly" +``` + +or: + +```toml +[toolchain] +version = "0.11.1-nightly.20260522.a" +``` + +The wrapper resolves the toolchain before forwarding to the selected toolchain payload. + +### IDE User + +As an IDE user, I want: + +```bash +baml ide install +``` + +to install the VSIX from my active BAML toolchain. + +`baml ide install` works because the wrapper forwards the command to the selected toolchain payload. Common editors should support flags such as `--cursor`, `--code`, `--windsurf`, and `--all` so users can avoid an interactive prompt. The VSIX launches `baml lsp` using the configured `baml` path or `baml` from `PATH`; it does not bundle a platform-specific CLI/LSP. + +### Release Engineer + +As a release engineer, I want a release run to: + +1. Decide the version once. +2. Build everything in parallel. +3. Smoke-test the outputs. +4. Publish only after the fan-in gate passes. +5. Be safely rerunnable if a publish step fails halfway. + +That is the release graph. + +## Release Channels + +There are two channels. + +| Channel | Trigger | Version Example | Notes | +|---|---|---|---| +| Canary | A change to `baml_language/release.toml` canary intent | `0.11.0` | Human-controlled | +| Nightly | Every successful `canary` branch CI run | `0.11.1-nightly.20260522.a` | Automatic | + +There is no `stable` channel in v1. Daily/commit-following users subscribe to `nightly`; canary remains the human-controlled channel. + +GitHub tags and releases are outputs of the release process. They are not release triggers. + +This matters because tag-triggered releases can self-trigger duplicate runs. The new design avoids that class of bug entirely. + +## Versioning Story + +### Source Of Truth + +The only human-edited source of truth for BAML language canary intent is: + +```text +baml_language/release.toml +``` + +with: + +```toml +[release] +canary_version = "0.11.0" +``` + +Nightly versions are derived from that canary version. No one manually edits a nightly version. We update this file with a new repo command; it updates this file, playground, SDKs, and related version surfaces. + +### Release Plan + +The CI plan job produces one `release-plan.json`. Every build job consumes that same artifact. + +Example: + +```json +{ + "schema": 1, + "channel": "nightly", + "canary_version": "0.11.0", + "canonical_version": "0.11.1-nightly.20260522.a", + "pypi_version": "0.11.1.dev2026052200", + "git_tag": "baml-language-0.11.1-nightly.20260522.a" +} +``` + +This prevents macOS, Linux, Windows, Python, VSIX, Rust, Node, and future SDK jobs from recomputing versions independently. + +### Compatibility Across SDKs + +Most ecosystems can use the canonical SemVer string directly: + +```text +0.11.1-nightly.20260522.a +``` + +PyPI cannot. Python uses PEP 440, so nightly releases are translated: + +| Canonical | PyPI | +|---|---| +| `0.11.0` | `0.11.0` | +| `0.11.1-nightly.20260522.a` | `0.11.1.dev2026052200` | +| `0.11.1-nightly.20260522.b` | `0.11.1.dev2026052201` | + +Python users still see the canonical BAML version from `baml_core.__version__` and `baml_core.get_version()`. PyPI stores the PEP 440 form because it must. All other foresseable SDKs don't have any issues with this system. + +### Deprecated Version Plumbing + +The current alpha flow uses `BAML_RELEASE_VERSION` as a compile-time override. The new design removes it. + +Instead, the version script stamps explicit version surfaces from `release-plan.json`, including a shared Rust `baml_version` module. Public BAML version APIs must not read Cargo package metadata such as `CARGO_PKG_VERSION`. + +## Install And Update Behavior + +| Action | Result | +|---|---| +| `brew install baml` | Installs wrapper, bootstraps canary toolchain, and may print an explicit `baml ide install` follow-up | +| `curl .../install.sh` | Installs wrapper, bootstraps requested canary/nightly/version | +| `baml toolchain install canary` | Installs current canary toolchain without necessarily making it active | +| `baml toolchain install nightly` | Installs current nightly toolchain without necessarily making it active | +| `baml toolchain use canary` | Selects canary as the active default and installs the resolved toolchain if missing | +| `baml toolchain use nightly` | Selects nightly as the active default and installs the resolved toolchain if missing | +| `baml toolchain update` | Updates active language toolchain | +| `baml self-update` | Updates wrapper only, refused for Homebrew/AUR installs | +| `baml ide install` | Installs IDE extension from active toolchain | + +Wrapper upgrades are idempotent. If a user is already on nightly or has an explicit toolchain, a package-manager wrapper upgrade must not reset them back to canary. + +## CI/CD Shape + +The pipeline is optimized around a fan-out/fan-in model. + +```mermaid +flowchart LR + plan["plan: version, channel, release-plan.json"] + plan --> cli["build cli + pack-host matrix"] + plan --> core["build baml_core wheels"] + plan --> vsix["build platform-neutral VSIX"] + plan --> wrapper["build wrapper"] + cli --> gate["smoke-test gate"] + core --> gate + vsix --> gate + wrapper --> gate + gate --> gh["publish GitHub release"] + gate --> pypi["publish PyPI"] + gate --> manifest["publish pkg.boundaryml.com manifests"] + gate --> pkg["publish wrapper packages when wrapper changed"] +``` + +The publish jobs are separate so they can be retried independently. They do not make independent version decisions. + +The smoke gate checks: + +- CLI version. +- Python canonical version. +- Python PyPI version. +- VSIX archive validity. +- archive layout. +- checksums. +- `baml pack` exercising `baml-pack-host`. + +If the gate fails, nothing publishes. + +## Hosting And Manifests + +We reuse the existing `pkg.boundaryml.com` infrastructure. + +`pkg.boundaryml.com` is the directory: + +- `manifest/v1/canary.json` +- `manifest/v1/nightly.json` +- `manifest/v1/version/.json` +- `manifest/v1/wrapper.json` +- `install.sh` +- `install.ps1` + +GitHub Releases are the binary storage layer: + +- toolchain archives. +- wrapper archives. +- checksums. +- VSIX artifacts. + +This split supports future package repositories such as `apt` and `dnf` because the system already has HTTP-hosted metadata, immutable artifacts, and checksums. + +## Safety And Edge Cases + +### Reruns Are Idempotent + +Per-version manifests are write-once by content: + +- missing object: upload. +- existing identical object: continue. +- existing different object: fail. + +Mutable pointers such as `canary.json`, `nightly.json`, and `wrapper.json` can be repaired on rerun. This handles the failure case where `version/.json` uploaded successfully, but the job failed before updating the channel pointer. + +### Nightly Stays Ahead Of Canary + +Nightly is derived from the next patch after canary. + +If canary is: + +```text +0.11.0 +``` + +then nightly is: + +```text +0.11.1-nightly.YYYYMMDD.a +``` + +This matters for PyPI because `0.11.0.dev...` sorts before canary `0.11.0`. Nightly subscribers should not accidentally fall behind canary. + +### Letter Overflow Is Explicit + +Nightly suffixes use `a` through `z` per UTC day. If we ever exceed `z`, the release job fails loudly. The operational answer is to cut a new canary version or wait for the next day. We are not adding a hidden second suffix scheme in v1. + +### No Double Publishing During Cutover + +The replacement lands on a dedicated branch. We merge it to `canary` only when it contains the complete replacement. We do not run the old alpha release system and the new release graph as two production systems side by side. + +Rollback is a code revert: + +```bash +git revert +``` + +### Tags Do Not Trigger Releases + +GitHub tags/releases are records created by the publish jobs. They do not trigger BAML language releases. This prevents self-triggered duplicate release runs. + +### Package Managers Do Not Own Toolchain Versions + +Homebrew and AUR versions track the wrapper version. They do not track the BAML language version. + +Nightly users get nightly through manifests, not package-manager churn. + +### VSIX Is Platform Neutral + +The release-built VSIX does not bundle `baml-cli`. It uses the user's installed `baml` command and therefore respects: + +- `BAML_VERSION`. +- `baml.toml [toolchain]`. +- the user's default channel. +- exact version pins. + +The VSIX does not need to be reinstalled for every BAML toolchain release. It checks compatibility by protocol metadata: + +- LSP compatibility is validated from the normal LSP `initialize` result. +- Playground compatibility is validated lazily when the playground WebSocket connects. +- The compatibility gate is integer protocol ranges and capability flags, not exact BAML semver equality. +- These checks must not spawn extra `baml` commands or hit the network during editor startup. + +Marketplace publishing is deferred, but every toolchain release already builds and archives the VSIX artifact. + +### Python Publishing Is Handled Carefully + +`release-sdk.yaml` currently owns Python `baml_core` publishing and PyPI trusted publishing may be bound to that workflow identity. The implementation must resolve whether to retain that file as a focused Python publish workflow or fold/rename it after updating PyPI trusted-publisher configuration. + +It must not remain an independent, separately versioned release path. + +## What We Are Removing + +The replacement branch removes or integrates the old adjacent systems: + +- `release-baml-language-alpha.yml` +- `release-pkg-boundaryml-com.yml` +- `release-cli.yaml` +- `BAML_RELEASE_VERSION` version injection. +- AUR templates that package `baml-cli`. +- VSIX-bundled `baml-cli`. +- standalone release paths that publish outside the release graph. + +`release-sdk.yaml` must be integrated deliberately because PyPI trusted publishing may be bound to that workflow identity. It can stay as a focused Python publish workflow invoked by the release graph, or be folded only after the PyPI trusted-publisher binding is updated or validated. + +## What We Are Not Doing In V1 + +We are not: + +- changing the Engine release pipeline. +- building `apt` or `dnf` repositories yet. +- publishing the new VSIX to the Marketplace yet. +- adding code signing/notarization yet. +- adding yanking. +- adding a runtime kill switch or long-lived dual pipeline. + +Bad releases are handled by publishing a newer fixed version and moving the channel pointer forward. diff --git a/TASK/docs/install.md b/TASK/docs/install.md new file mode 100644 index 00000000000..7a2fb14ed10 --- /dev/null +++ b/TASK/docs/install.md @@ -0,0 +1,85 @@ +# BAML Install Guide + +## Package Managers + +Homebrew and AUR install the `baml` wrapper only. They do not install, select, or update a language toolchain during package install or upgrade. + +```sh +brew install boundaryml/tap/baml +baml toolchain use canary +``` + +Nightly users opt in explicitly: + +```sh +baml toolchain use nightly +baml toolchain update +``` + +Package-manager wrapper upgrades use the package manager: + +```sh +brew upgrade baml +``` + +AUR users upgrade with their normal AUR helper, for example `paru -Syu baml-bin`. + +## Curl Installer + +The curl installer is user-scoped. It installs or updates `~/.baml/bin/baml` and, unless `--wrapper-only` is used, bootstraps the requested toolchain through the wrapper. + +```sh +curl -fsSL https://pkg.boundaryml.com/install.sh | sh -s +curl -fsSL https://pkg.boundaryml.com/install.sh | sh -s -- --channel nightly +curl -fsSL https://pkg.boundaryml.com/install.sh | sh -s -- --version 0.11.0 --no-modify-path +``` + +Docker/CI: + +```dockerfile +RUN curl -fsSL https://pkg.boundaryml.com/install.sh | sh -s -- --version 0.11.0 --no-modify-path +ENV PATH="/root/.baml/bin:${PATH}" +``` + +The Unix installer writes `~/.baml/env`: + +```sh +export BAML_HOME="${BAML_HOME:-$HOME/.baml}" +case ":$PATH:" in + *":$BAML_HOME/bin:"*) ;; + *) export PATH="$BAML_HOME/bin:$PATH" ;; +esac +``` + +## Toolchains + +Use selects the default and installs if missing: + +```sh +baml toolchain use canary +baml toolchain use nightly +baml toolchain use 0.11.0 +``` + +Install downloads without changing the default: + +```sh +baml toolchain install 0.11.0 +``` + +Update advances a channel selector only: + +```sh +baml toolchain update +``` + +## IDE + +IDE installation is explicit and owned by the selected toolchain: + +```sh +baml ide install --cursor +baml ide install --code +``` + +`baml self-update` updates curl-installed wrappers only. Package-manager wrappers refuse self-update and point back to the package manager. diff --git a/TASK/docs/release-maintainer.md b/TASK/docs/release-maintainer.md new file mode 100644 index 00000000000..af5e905cbf6 --- /dev/null +++ b/TASK/docs/release-maintainer.md @@ -0,0 +1,66 @@ +# BAML Language Release Maintainer Notes + +## Model + +`baml_language/release.toml` is the human-authored source of truth: + +```toml +[release] +canary_version = "0.11.0" +``` + +Canary publishes exactly that version. Nightly derives the next patch and a UTC date suffix, for example `0.11.1-nightly.20260602.a`. + +## Common Commands + +```sh +scripts/baml-language-version bump --patch +scripts/baml-language-version sync +scripts/baml-language-version check +scripts/baml-language-version compute --channel nightly +scripts/baml-language-version compute --channel nightly --pypi +scripts/baml-language-version plan --channel nightly --out release-plan.json +``` + +PyPI nightly versions use PEP 440 dev releases, for example `0.11.1-nightly.20260602.a` becomes `0.11.1.dev2026060200`. + +## Workflow + +`.github/workflows/release-baml-language.yml` is the release graph entrypoint. It uses a non-cancelling concurrency lock: + +```yaml +group: baml-language-release-canary +cancel-in-progress: false +``` + +Build jobs download the single `release-plan.json` artifact and run `scripts/baml-language-version stamp --plan release-plan.json` before compiling. + +Dry-run dispatch: + +```sh +gh workflow run release-baml-language.yml -f channel=nightly -f dry_run=true +``` + +Dry runs generate a `dry-run-release-files` workflow artifact containing `manifest/v1/**`, `homebrew/Formula/baml.rb`, and AUR `PKGBUILD` / `.SRCINFO` files. When the run is on `refs/heads/canary`, the workflow also uploads public manifests under `https://pkg.boundaryml.com/dryrun//manifest/v1`. + +Dry-run wrapper validation: + +```sh +BAML_HOME="$(mktemp -d)" \ +BAML_MANIFEST_BASE_URL="https://pkg.boundaryml.com/dryrun//manifest/v1" \ +baml toolchain use nightly +``` + +## Publishing + +Production publishes are guarded to `refs/heads/canary`. Mutable pointers (`canary.json`, `nightly.json`, `wrapper.json`, install scripts) are repairable on rerun. Immutable per-version manifests and GitHub release assets are never overwritten. + +Automatic canary branch releases publish nightly by default. If the source commit advances `baml_language/release.toml` and `baml-language-` does not already exist, the workflow publishes canary first and queues a serialized nightly run after the canary manifest is live. + +The renamed PyPI workflow is `.github/workflows/publish-python-pypi.yml`. Before production PyPI publishing, update or validate the PyPI trusted-publisher binding for project `baml-core` so it authorizes this workflow filename. + +Homebrew and AUR publish only wrapper packages when `wrapper_changed == true`. Toolchain releases never dispatch package-manager updates. `scripts/baml-package-manager-artifacts` generates the formula and AUR files from wrapper archive checksums; publish jobs refuse to run unless the Homebrew token or AUR SSH key is configured. + +## Rollback + +Rollback is a code revert plus mutable pointer repair. Leave immutable artifacts in place and move channel pointers to a fixed version. diff --git a/TASK/docs/toolchain-system.md b/TASK/docs/toolchain-system.md new file mode 100644 index 00000000000..f60edc4168d --- /dev/null +++ b/TASK/docs/toolchain-system.md @@ -0,0 +1,68 @@ +# BAML Toolchain System + +## Product Boundary + +`baml-wrapper` contains only `bin/baml`. It is installed by package managers and curl installers. + +`baml-toolchain` contains `bin/baml-cli`, `bin/baml-pack-host`, and `assets/baml-vscode.vsix`. Normal commands are forwarded by the wrapper to the selected toolchain. + +## Local Layout + +```text +~/.baml/ + config.toml + state.toml + bin/baml + manifest-cache/ + toolchains// + VERSION + install.json + bin/baml-cli + bin/baml-pack-host + assets/baml-vscode.vsix +``` + +`config.toml` stores user intent: + +```toml +[default] +selector = "canary" + +[update] +auto_check = false +``` + +`state.toml` stores active channel resolutions. Manifest cache is not authoritative for normal command resolution. + +## Resolution + +Precedence: + +1. `BAML_VERSION` +2. nearest `baml.toml [toolchain]` +3. `~/.baml/config.toml [default].selector` +4. hardcoded `canary` + +Normal commands never hit the network and never auto-install. Toolchain commands are the allowlisted network surface. + +`toolchain use` installs if missing and selects the default. `toolchain install` downloads without selecting. `toolchain update` advances channel selectors only. `toolchain list` is local-only. + +## Manifests + +Toolchain and wrapper manifests use schema `1`, HTTPS artifact URLs, lowercase SHA-256 checksums, and exact target sets. `BAML_MANIFEST_BASE_URL` can point wrapper validation at dry-run or mirror manifests and is not persisted. + +Manifest cache entries are namespaced by manifest base URL: production uses `manifest-cache/prod`, while overrides use `manifest-cache/override/`. Channel pointers have a 24-hour cache TTL for toolchain-management commands. Normal pass-through commands never read remote metadata; they use `config.toml`, `state.toml`, and installed `VERSION` files only. Channel state records the manifest base URL that produced it, so dry-run or mirror state is not silently reused under production. + +## IDE And Compatibility + +The VSIX launches `baml lsp` through the wrapper. It starts one lazy LSP client per nearest BAML project root, so sibling projects can select different toolchains. + +LSP compatibility metadata is returned on `initialize` under `capabilities.experimental.baml`. Playground compatibility is checked when the WebSocket receives `hello`; the server sends `hello` and then the legacy `ready`. + +## Direct Internal CLI + +Direct `baml-cli` invocation prints a stderr warning unless `BAML_WRAPPER_EXEC=1` or `BAML_CLI_ALLOW_DIRECT=1` is set. The warning never goes to stdout. + +## Pack Fetcher + +`baml_release` owns shared target naming, checksums, manifest schema structs, retrying downloads, install locks, and archive extraction. `baml pack` keeps existing `BAML_PACK_HOST_RELEASE_*` overrides while sharing release artifact resolution with the wrapper. diff --git a/baml_language/Cargo.lock b/baml_language/Cargo.lock index 1de9b33c9c1..fbdaa126bc9 100644 --- a/baml_language/Cargo.lock +++ b/baml_language/Cargo.lock @@ -972,6 +972,18 @@ dependencies = [ "tower-service", ] +[[package]] +name = "baml" +version = "0.1.0" +dependencies = [ + "anyhow", + "baml_release", + "reqwest", + "serde", + "serde_json", + "toml 0.8.23", +] + [[package]] name = "baml_base" version = "0.0.0-beta" @@ -1055,8 +1067,10 @@ dependencies = [ "baml_lsp2_actions", "baml_lsp_server", "baml_project", + "baml_release", "baml_tests", "baml_type", + "baml_version", "baml_workspace", "bex_engine", "bex_events", @@ -1351,6 +1365,7 @@ dependencies = [ "axum", "baml_compiler2_visualization", "baml_type", + "baml_version", "base64", "bex_events", "bex_events_native", @@ -1436,6 +1451,21 @@ dependencies = [ "text-size", ] +[[package]] +name = "baml_release" +version = "0.0.0-beta" +dependencies = [ + "anyhow", + "flate2", + "reqwest", + "serde", + "serde_json", + "sha2 0.10.9", + "tar", + "thiserror 2.0.18", + "zip", +] + [[package]] name = "baml_tests" version = "0.0.0-beta" @@ -1490,6 +1520,10 @@ dependencies = [ "num-bigint", ] +[[package]] +name = "baml_version" +version = "0.0.0-beta" + [[package]] name = "baml_workspace" version = "0.0.0-beta" @@ -1649,6 +1683,7 @@ dependencies = [ "baml_lsp2_actions", "baml_project", "baml_type", + "baml_version", "baml_workspace", "bex_engine", "bex_events", @@ -1883,6 +1918,7 @@ checksum = "36f64beae40a84da1b4b26ff2761a5b895c12adc41dc25aaee1c4f2bbfe97a6e" name = "bridge_cffi" version = "0.218.1" dependencies = [ + "baml_version", "bex_events", "bex_events_native", "bex_project", @@ -1923,6 +1959,7 @@ dependencies = [ name = "bridge_nodejs" version = "0.0.0-beta" dependencies = [ + "baml_version", "bex_events", "bex_project", "bridge_cffi", @@ -1941,6 +1978,7 @@ dependencies = [ name = "bridge_python" version = "0.0.0-beta" dependencies = [ + "baml_version", "bex_events", "bex_project", "bridge_cffi", @@ -1962,6 +2000,7 @@ version = "0.1.0" dependencies = [ "async-trait", "baml_type", + "baml_version", "bex_events", "bex_heap", "bex_project", @@ -6624,6 +6663,7 @@ checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ "base64", "bytes", + "futures-channel", "futures-core", "futures-util", "http 1.4.1", diff --git a/baml_language/Cargo.toml b/baml_language/Cargo.toml index 89a75666eb4..9a56b735aff 100644 --- a/baml_language/Cargo.toml +++ b/baml_language/Cargo.toml @@ -87,7 +87,9 @@ baml_lsp2_actions = { path = "crates/baml_lsp2_actions" } baml_lsp_server = { path = "crates/baml_lsp_server", default-features = false } baml_lsp_types = { path = "crates/baml_lsp_types" } baml_project = { path = "crates/baml_project" } +baml_release = { path = "crates/baml_release" } baml_tests = { path = "crates/baml_tests" } +baml_version = { path = "crates/baml_version" } tools_onionskin = { path = "crates/tools_onionskin" } bex_vm = { path = "crates/bex_vm" } bex_vm_types = { path = "crates/bex_vm_types" } diff --git a/baml_language/crates/baml/Cargo.toml b/baml_language/crates/baml/Cargo.toml new file mode 100644 index 00000000000..6c1bd633277 --- /dev/null +++ b/baml_language/crates/baml/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "baml" +version = "0.1.0" +publish = false +authors = { workspace = true } +edition = { workspace = true } +rust-version = { workspace = true } +homepage = { workspace = true } +documentation = { workspace = true } +repository = { workspace = true } +license = { workspace = true } + +[lints] +workspace = true + +[dependencies] +baml_release = { workspace = true } +anyhow = { workspace = true } +reqwest = { workspace = true, features = [ "blocking", "rustls" ] } +serde = { workspace = true, features = [ "derive" ] } +serde_json = { workspace = true } +toml = { workspace = true } + +[[bin]] +name = "baml" +path = "src/main.rs" + +[package.metadata.ci] +wasm_support = false diff --git a/baml_language/crates/baml/src/main.rs b/baml_language/crates/baml/src/main.rs new file mode 100644 index 00000000000..2c6ecb5cdfc --- /dev/null +++ b/baml_language/crates/baml/src/main.rs @@ -0,0 +1,662 @@ +#![allow(clippy::print_stdout, clippy::print_stderr)] + +use std::{ + collections::{BTreeMap, hash_map::DefaultHasher}, + env, fs, + hash::{Hash, Hasher}, + path::{Path, PathBuf}, + process::Command, + time::{Duration, SystemTime}, +}; + +use anyhow::{Context, Result, anyhow}; +use baml_release::{Artifact, Product, ReleaseSpec, ToolchainManifest, WrapperManifest}; +use serde::{Deserialize, Serialize}; + +const CONFIG_FILE: &str = "config.toml"; +const STATE_FILE: &str = "state.toml"; +const CHANNEL_CACHE_TTL: Duration = Duration::from_secs(24 * 60 * 60); + +#[derive(Debug, Default, Serialize, Deserialize)] +struct Config { + #[serde(default)] + default: DefaultConfig, + #[serde(default)] + update: UpdateConfig, +} + +#[derive(Debug, Serialize, Deserialize)] +struct DefaultConfig { + selector: String, +} + +impl Default for DefaultConfig { + fn default() -> Self { + Self { + selector: "canary".to_string(), + } + } +} + +#[derive(Debug, Default, Serialize, Deserialize)] +struct UpdateConfig { + auto_check: bool, +} + +#[derive(Debug, Default, Serialize, Deserialize)] +struct State { + #[serde(default)] + channels: BTreeMap, +} + +#[derive(Debug, Serialize, Deserialize)] +struct ChannelState { + active_version: String, + resolved_at: String, + manifest_path: String, + #[serde(skip_serializing_if = "Option::is_none")] + manifest_base_url: Option, +} + +#[derive(Debug, Deserialize)] +struct ProjectConfig { + toolchain: Option, +} + +#[derive(Debug, Deserialize)] +struct ProjectToolchain { + version: Option, + channel: Option, +} + +enum SelectorSource { + Env, + Project(PathBuf), + Config, + Fallback, +} + +struct ResolvedSelector { + selector: String, + source: SelectorSource, +} + +fn main() { + let exit = match run() { + Ok(code) => code, + Err(err) => { + eprintln!("{err:#}"); + 1 + } + }; + std::process::exit(exit); +} + +fn run() -> Result { + let mut args: Vec = env::args().skip(1).collect(); + if args.iter().any(|arg| arg == "--version" || arg == "-V") { + println!("baml {}", env!("CARGO_PKG_VERSION")); + return Ok(0); + } + if args.first().map(String::as_str) == Some("toolchain") { + args.remove(0); + return toolchain(args).map(|()| 0); + } + if args.first().map(String::as_str) == Some("self-update") { + return self_update().map(|()| 0); + } + pass_through(args) +} + +fn baml_home() -> PathBuf { + env::var_os("BAML_HOME") + .map(PathBuf::from) + .or_else(|| { + env::var_os("HOME") + .map(PathBuf::from) + .or_else(|| env::var_os("USERPROFILE").map(PathBuf::from)) + .map(|home| home.join(".baml")) + }) + .unwrap_or_else(|| PathBuf::from(".baml")) +} + +fn config_path() -> PathBuf { + baml_home().join(CONFIG_FILE) +} + +fn state_path() -> PathBuf { + baml_home().join(STATE_FILE) +} + +fn toolchains_dir() -> PathBuf { + baml_home().join("toolchains") +} + +fn manifest_cache_dir(base_url: &str) -> PathBuf { + if base_url == baml_release::DEFAULT_MANIFEST_BASE_URL { + return baml_home().join("manifest-cache").join("prod"); + } + let mut hasher = DefaultHasher::new(); + base_url.hash(&mut hasher); + baml_home() + .join("manifest-cache") + .join("override") + .join(format!("{:016x}", hasher.finish())) +} + +fn read_config() -> Config { + fs::read_to_string(config_path()) + .ok() + .and_then(|text| toml::from_str(&text).ok()) + .unwrap_or_default() +} + +fn write_config(config: &Config) -> Result<()> { + write_toml_atomic(&config_path(), config) +} + +fn read_state() -> State { + fs::read_to_string(state_path()) + .ok() + .and_then(|text| toml::from_str(&text).ok()) + .unwrap_or_default() +} + +fn write_state(state: &State) -> Result<()> { + write_toml_atomic(&state_path(), state) +} + +fn write_toml_atomic(path: &Path, value: &T) -> Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + let tmp = path.with_extension("tmp"); + fs::write(&tmp, toml::to_string_pretty(value)?)?; + fs::rename(tmp, path)?; + Ok(()) +} + +fn toolchain(args: Vec) -> Result<()> { + let (args, manifest_base_url) = parse_manifest_base_url(args)?; + match args.first().map(String::as_str) { + Some("install") => { + let selector = args + .get(1) + .ok_or_else(|| anyhow!("usage: baml toolchain install "))?; + let force = args.iter().any(|arg| arg == "--force"); + install_toolchain(selector, false, manifest_base_url.as_deref(), force) + } + Some("use") => { + let selector = args + .get(1) + .ok_or_else(|| anyhow!("usage: baml toolchain use "))?; + use_toolchain(selector, manifest_base_url.as_deref()) + } + Some("update") => update_toolchain(manifest_base_url.as_deref()), + Some("list") => { + list_toolchains(); + Ok(()) + } + Some("uninstall") => { + let version = args + .get(1) + .ok_or_else(|| anyhow!("usage: baml toolchain uninstall "))?; + uninstall_toolchain(version) + } + _ => Err(anyhow!( + "usage: baml toolchain ..." + )), + } +} + +fn parse_manifest_base_url(mut args: Vec) -> Result<(Vec, Option)> { + let mut manifest_base_url = None; + let mut i = 0; + while i < args.len() { + if args[i] == "--manifest-base-url" { + let value = args + .get(i + 1) + .ok_or_else(|| anyhow!("--manifest-base-url requires a value"))? + .trim_end_matches('/') + .to_string(); + args.drain(i..=i + 1); + manifest_base_url = Some(value); + } else { + i += 1; + } + } + Ok((args, manifest_base_url)) +} + +fn pass_through(args: Vec) -> Result { + let selector = active_selector()?; + let version = concrete_version_for_selector(&selector)?; + let cli = toolchain_cli_path(&version); + if !cli.exists() { + return Err(missing_toolchain_error(&selector, &version)); + } + verify_toolchain_version_file(&version)?; + + let mut command = Command::new(cli); + command.args(args); + command.env("BAML_WRAPPER_EXEC", "1"); + command.env("BAML_WRAPPER_RESOLVED_TOOLCHAIN", &version); + + #[cfg(unix)] + { + use std::os::unix::process::CommandExt; + let err = command.exec(); + Err(anyhow!("failed to exec baml-cli: {err}")) + } + #[cfg(not(unix))] + { + let status = command.status().context("failed to run baml-cli")?; + Ok(status.code().unwrap_or(1)) + } +} + +fn active_selector() -> Result { + if let Ok(value) = env::var("BAML_VERSION") { + if !value.trim().is_empty() { + return Ok(ResolvedSelector { + selector: value, + source: SelectorSource::Env, + }); + } + } + if let Some((path, selector)) = project_toolchain_selector()? { + return Ok(ResolvedSelector { + selector, + source: SelectorSource::Project(path), + }); + } + let config = read_config(); + if !config.default.selector.trim().is_empty() { + return Ok(ResolvedSelector { + selector: config.default.selector, + source: SelectorSource::Config, + }); + } + Ok(ResolvedSelector { + selector: "canary".to_string(), + source: SelectorSource::Fallback, + }) +} + +fn project_toolchain_selector() -> Result> { + let mut dir = env::current_dir()?; + let home = env::var_os("HOME").map(PathBuf::from); + loop { + let candidate = dir.join("baml.toml"); + if candidate.exists() { + let text = fs::read_to_string(&candidate)?; + if let Ok(config) = toml::from_str::(&text) { + if let Some(toolchain) = config.toolchain { + if let Some(version) = toolchain.version { + return Ok(Some((candidate, version))); + } + if let Some(channel) = toolchain.channel { + return Ok(Some((candidate, channel))); + } + } + } + } + if home.as_ref().is_some_and(|home| dir == *home) || !dir.pop() { + break; + } + } + Ok(None) +} + +fn concrete_version_for_selector(selector: &ResolvedSelector) -> Result { + if is_channel(&selector.selector) { + let state = read_state(); + if let Some(channel) = state.channels.get(&selector.selector) { + let current_base = baml_release::manifest_base_url(); + if channel + .manifest_base_url + .as_deref() + .is_some_and(|base| base.trim_end_matches('/') != current_base) + { + return Err(anyhow!( + "channel {} was selected from a different manifest source.\nRun: baml toolchain use {} under the current manifest source.", + selector.selector, + selector.selector + )); + } + return Ok(channel.active_version.clone()); + } + return match &selector.source { + SelectorSource::Project(path) => Err(anyhow!( + "baml.toml [toolchain] selects channel {}, but no active concrete version is recorded locally.\nRun: baml toolchain use {}", + selector.selector, + selector.selector + ) + .context(format!("project config: {}", path.display()))), + _ => Err(anyhow!( + "error: no BAML toolchain is installed.\nRun: baml toolchain use canary\nOr: baml toolchain use nightly" + )), + }; + } + Ok(selector.selector.clone()) +} + +fn missing_toolchain_error(selector: &ResolvedSelector, version: &str) -> anyhow::Error { + let installed = installed_toolchains().join(", "); + match &selector.source { + SelectorSource::Project(path) => anyhow!( + "{} [toolchain] pins version {version}, but it isn't installed.\nInstalled toolchains: {}\nRun: baml toolchain install {version}", + path.display(), + if installed.is_empty() { + "(none)" + } else { + &installed + } + ), + _ => anyhow!( + "BAML toolchain {version} is not installed.\nInstalled toolchains: {}\nRun: baml toolchain install {version}", + if installed.is_empty() { + "(none)" + } else { + &installed + } + ), + } +} + +fn toolchain_cli_path(version: &str) -> PathBuf { + let exe = if cfg!(windows) { + "baml-cli.exe" + } else { + "baml-cli" + }; + toolchains_dir().join(version).join("bin").join(exe) +} + +fn verify_toolchain_version_file(version: &str) -> Result<()> { + let path = toolchains_dir().join(version).join("VERSION"); + let actual = fs::read_to_string(&path).with_context(|| { + format!( + "failed to read toolchain VERSION file at {}", + path.display() + ) + })?; + if actual.trim() != version { + anyhow::bail!( + "BAML toolchain {version} is corrupt: VERSION contains {}.\nRun: baml toolchain install {version} --force", + actual.trim() + ); + } + Ok(()) +} + +fn is_channel(selector: &str) -> bool { + selector == "canary" || selector == "nightly" +} + +fn manifest_base_url(override_url: Option<&str>) -> String { + override_url + .map(|value| value.trim_end_matches('/').to_string()) + .unwrap_or_else(baml_release::manifest_base_url) +} + +fn fetch_manifest(selector: &str, override_url: Option<&str>) -> Result { + let base = manifest_base_url(override_url); + let url = if is_channel(selector) { + format!("{base}/{selector}.json") + } else { + format!("{base}/version/{selector}.json") + }; + let cache_path = if is_channel(selector) { + manifest_cache_dir(&base).join(format!("{selector}.json")) + } else { + manifest_cache_dir(&base) + .join("version") + .join(format!("{selector}.json")) + }; + let use_cache = !is_channel(selector) + || cache_path + .metadata() + .and_then(|metadata| metadata.modified()) + .ok() + .and_then(|modified| SystemTime::now().duration_since(modified).ok()) + .is_some_and(|age| age <= CHANNEL_CACHE_TTL); + let mut fetched_remote = false; + let text = if use_cache { + fs::read_to_string(&cache_path).ok() + } else { + None + }; + let text = match text { + Some(text) => text, + None => { + fetched_remote = true; + reqwest::blocking::get(&url) + .with_context(|| format!("failed to fetch {url}"))? + .error_for_status() + .with_context(|| format!("failed to fetch {url}"))? + .text() + .with_context(|| format!("failed to read {url}"))? + } + }; + let manifest: ToolchainManifest = toml_or_json(&text)?; + manifest.validate()?; + if fetched_remote { + write_text_atomic(&cache_path, &text)?; + } + let version_cache_path = manifest_cache_dir(&base) + .join("version") + .join(format!("{}.json", manifest.version)); + if !version_cache_path.exists() { + write_text_atomic(&version_cache_path, &text)?; + } + Ok(manifest) +} + +fn toml_or_json Deserialize<'de>>(text: &str) -> Result { + serde_json::from_str(text).context("invalid JSON manifest") +} + +fn install_toolchain( + selector: &str, + activate_channel: bool, + override_url: Option<&str>, + force: bool, +) -> Result<()> { + let manifest = fetch_manifest(selector, override_url)?; + let target = baml_release::release_host_target_triple()?; + let artifact = manifest.artifact_for_target(target)?.clone(); + install_manifest_artifact(&manifest.version, target, artifact, force)?; + if activate_channel && is_channel(selector) { + let mut state = read_state(); + state.channels.insert( + selector.to_string(), + ChannelState { + active_version: manifest.version.clone(), + resolved_at: manifest.released_at.clone(), + manifest_path: manifest_cache_dir(&manifest_base_url(override_url)) + .join("version") + .join(format!("{}.json", manifest.version)) + .display() + .to_string(), + manifest_base_url: Some(manifest_base_url(override_url)), + }, + ); + write_state(&state)?; + } + println!("installed BAML toolchain {}", manifest.version); + Ok(()) +} + +fn install_manifest_artifact( + version: &str, + target: &str, + artifact: Artifact, + force: bool, +) -> Result<()> { + fs::create_dir_all(toolchains_dir())?; + if force { + let dir = toolchains_dir().join(version); + if dir.exists() { + fs::remove_dir_all(dir)?; + } + } + let fetcher = baml_release::Fetcher::from_artifact( + ReleaseSpec { + version: version.to_string(), + target: target.to_string(), + }, + Product::Toolchain, + artifact, + ); + fetcher + .install_to_toolchain_root(&toolchains_dir()) + .map(|_| ()) + .map_err(|err| anyhow!("{err}")) +} + +fn use_toolchain(selector: &str, override_url: Option<&str>) -> Result<()> { + if is_channel(selector) { + install_toolchain(selector, true, override_url, false)?; + } else { + let target = baml_release::release_host_target_triple()?; + if !toolchain_cli_path(selector).exists() { + let manifest = fetch_manifest(selector, override_url)?; + let artifact = manifest.artifact_for_target(target)?.clone(); + install_manifest_artifact(&manifest.version, target, artifact, false)?; + } + } + + let mut config = read_config(); + config.default.selector = selector.to_string(); + write_config(&config)?; + println!("selected BAML toolchain {selector}"); + Ok(()) +} + +fn update_toolchain(override_url: Option<&str>) -> Result<()> { + let config = read_config(); + if !is_channel(&config.default.selector) { + println!( + "active selector {} is an exact version and does not advance automatically.\nRun: baml toolchain use canary\nOr: baml toolchain use nightly", + config.default.selector + ); + return Ok(()); + } + install_toolchain(&config.default.selector, true, override_url, false) +} + +fn installed_toolchains() -> Vec { + let mut versions = fs::read_dir(toolchains_dir()) + .ok() + .into_iter() + .flat_map(|entries| entries.filter_map(std::result::Result::ok)) + .filter_map(|entry| { + entry + .file_type() + .ok() + .filter(std::fs::FileType::is_dir) + .and_then(|_| entry.file_name().into_string().ok()) + }) + .filter(|name| !name.starts_with('.')) + .collect::>(); + versions.sort(); + versions +} + +fn list_toolchains() { + let config = read_config(); + let state = read_state(); + println!("default selector: {}", config.default.selector); + for (channel, state) in state.channels { + println!("{channel}: {}", state.active_version); + } + let installed = installed_toolchains(); + if installed.is_empty() { + println!("installed toolchains: (none)"); + } else { + println!("installed toolchains:"); + for version in installed { + println!(" {version}"); + } + } +} + +fn uninstall_toolchain(version: &str) -> Result<()> { + let dir = toolchains_dir().join(version); + if !dir.exists() { + return Err(anyhow!("BAML toolchain {version} is not installed")); + } + fs::remove_dir_all(dir)?; + println!("uninstalled BAML toolchain {version}"); + Ok(()) +} + +fn self_update() -> Result<()> { + let current = env::current_exe()?; + if is_managed_install(¤t) { + return Err(anyhow!( + "this BAML wrapper appears to be managed by a package manager.\nRun: brew upgrade baml or your system package-manager upgrade command." + )); + } + let manifest = fetch_wrapper_manifest()?; + let target = baml_release::release_host_target_triple()?; + let artifact = manifest.artifact_for_target(target)?.clone(); + let binary = if cfg!(windows) { "baml.exe" } else { "baml" }; + let fetcher = baml_release::Fetcher::from_artifact( + ReleaseSpec { + version: manifest.version.clone(), + target: target.to_string(), + }, + Product::Wrapper, + artifact, + ); + let bytes = fetcher + .fetch_binary(binary) + .map_err(|err| anyhow!("{err}"))?; + let tmp = current.with_extension("tmp"); + fs::write(&tmp, bytes)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&tmp, fs::Permissions::from_mode(0o755))?; + } + fs::rename(tmp, current)?; + println!("updated BAML wrapper to {}", manifest.version); + Ok(()) +} + +fn fetch_wrapper_manifest() -> Result { + let base = baml_release::manifest_base_url(); + let url = format!("{base}/wrapper.json"); + let cache_path = manifest_cache_dir(&base).join("wrapper.json"); + let text = reqwest::blocking::get(&url) + .with_context(|| format!("failed to fetch {url}"))? + .error_for_status() + .with_context(|| format!("failed to fetch {url}"))? + .text() + .with_context(|| format!("failed to read {url}"))?; + write_text_atomic(&cache_path, &text)?; + let manifest: WrapperManifest = + serde_json::from_str(&text).context("invalid wrapper manifest")?; + manifest.validate()?; + Ok(manifest) +} + +fn write_text_atomic(path: &Path, text: &str) -> Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + let tmp = path.with_extension("tmp"); + fs::write(&tmp, text)?; + fs::rename(tmp, path)?; + Ok(()) +} + +fn is_managed_install(path: &Path) -> bool { + let text = path.to_string_lossy(); + text.contains("/opt/homebrew/") + || text.contains("/usr/local/Cellar/") + || text.starts_with("/usr/bin/") + || text.starts_with("/opt/") +} diff --git a/baml_language/crates/baml_cli/Cargo.toml b/baml_language/crates/baml_cli/Cargo.toml index a4a533a1820..4915cd71182 100644 --- a/baml_language/crates/baml_cli/Cargo.toml +++ b/baml_language/crates/baml_cli/Cargo.toml @@ -24,7 +24,9 @@ baml_fmt = { workspace = true } baml_lsp2_actions = { workspace = true } baml_lsp_server = { workspace = true } baml_project = { workspace = true } +baml_release = { workspace = true } baml_type = { workspace = true } +baml_version = { workspace = true } baml_workspace = { workspace = true } bex_engine = { workspace = true } bex_events = { workspace = true } diff --git a/baml_language/crates/baml_cli/src/commands.rs b/baml_language/crates/baml_cli/src/commands.rs index 4912857800a..00c73cfbf03 100644 --- a/baml_language/crates/baml_cli/src/commands.rs +++ b/baml_language/crates/baml_cli/src/commands.rs @@ -6,14 +6,11 @@ use anyhow::Result; use clap::{CommandFactory, Parser, Subcommand}; pub(crate) const fn release_version() -> &'static str { - match option_env!("BAML_RELEASE_VERSION") { - Some(version) => version, - None => env!("CARGO_PKG_VERSION"), - } + baml_version::CANONICAL_VERSION } #[derive(Parser, Debug)] -#[command(author, version = release_version(), about = "A CLI tool for working with BAML. Learn more at https://docs.boundaryml.com.", long_about = None)] +#[command(name = "baml-cli", author, version = release_version(), about = "A CLI tool for working with BAML. Learn more at https://docs.boundaryml.com.", long_about = None)] #[command(styles = crate::reporter::CLAP_STYLING)] #[command(propagate_version = true)] pub(crate) struct RuntimeCli { @@ -88,6 +85,9 @@ pub(crate) enum Commands { #[command(about = "Package a BAML target as a standalone executable")] Pack(crate::pack_command::PackArgs), + #[command(about = "Install or manage IDE integration assets")] + Ide(crate::ide_command::IdeArgs), + #[command(about = "Starts a language server", name = "lsp")] LanguageServer(crate::lsp::LanguageServerArgs), // #[command(about = "Start an interactive REPL for BAML expressions", hide = true)] @@ -149,6 +149,7 @@ impl RuntimeCli { Commands::New(args) => args.run(), Commands::Run(args) => args.run(), Commands::Pack(args) => args.run(), + Commands::Ide(args) => args.run(), Commands::Describe(args) => args.run(), Commands::Generate(args) => args.run(), Commands::Grep(args) => args.run(), @@ -177,7 +178,6 @@ mod tests { #[test] fn release_version_matches_compile_time_setting() { - let expected = option_env!("BAML_RELEASE_VERSION").unwrap_or(env!("CARGO_PKG_VERSION")); - assert_eq!(release_version(), expected); + assert_eq!(release_version(), baml_version::CANONICAL_VERSION); } } diff --git a/baml_language/crates/baml_cli/src/ide_command.rs b/baml_language/crates/baml_cli/src/ide_command.rs new file mode 100644 index 00000000000..e368d524762 --- /dev/null +++ b/baml_language/crates/baml_cli/src/ide_command.rs @@ -0,0 +1,128 @@ +use std::{ + env, + ffi::OsString, + io::Write, + path::{Path, PathBuf}, + process::Command, +}; + +use anyhow::{Context, Result, anyhow}; +use clap::Args; + +use crate::ExitCode; + +#[derive(Args, Clone, Debug)] +pub(crate) struct IdeArgs { + #[command(subcommand)] + pub command: IdeCommand, +} + +#[derive(clap::Subcommand, Clone, Debug)] +pub(crate) enum IdeCommand { + #[command(about = "Install the active toolchain's BAML IDE extension")] + Install(IdeInstallArgs), +} + +#[derive(Args, Clone, Debug)] +pub(crate) struct IdeInstallArgs { + #[arg(long)] + pub cursor: bool, + #[arg(long)] + pub code: bool, +} + +impl IdeArgs { + pub fn run(&self) -> Result { + match &self.command { + IdeCommand::Install(args) => args.run(), + } + } +} + +impl IdeInstallArgs { + pub fn run(&self) -> Result { + let vsix = active_toolchain_vsix()?; + let editor = self.resolve_editor()?; + let status = Command::new(&editor) + .arg("--install-extension") + .arg(&vsix) + .status() + .with_context(|| format!("failed to run {}", editor.to_string_lossy()))?; + + if !status.success() { + anyhow::bail!( + "{} --install-extension {} exited with {status}", + editor.to_string_lossy(), + vsix.display() + ); + } + + writeln!( + std::io::stdout(), + "installed BAML IDE extension from {}", + vsix.display() + )?; + Ok(ExitCode::Success) + } + + fn resolve_editor(&self) -> Result { + if self.cursor && self.code { + return Err(anyhow!("choose only one of --cursor or --code")); + } + if self.cursor { + return Ok(OsString::from("cursor")); + } + if self.code { + return Ok(OsString::from("code")); + } + + let cursor = command_on_path("cursor"); + let code = command_on_path("code"); + match (cursor, code) { + (Some(cursor), None) => Ok(cursor), + (None, Some(code)) => Ok(code), + (Some(_), Some(_)) => Err(anyhow!( + "both Cursor and VS Code CLIs were found; rerun with --cursor or --code" + )), + (None, None) => Err(anyhow!( + "no supported editor CLI was found.\nManual install commands:\n cursor --install-extension {}\n code --install-extension {}", + active_toolchain_vsix()?.display(), + active_toolchain_vsix()?.display() + )), + } + } +} + +fn active_toolchain_vsix() -> Result { + let exe = env::current_exe().context("failed to locate baml-cli executable")?; + let toolchain_root = exe + .parent() + .and_then(Path::parent) + .ok_or_else(|| anyhow!("failed to determine active BAML toolchain root"))?; + let vsix = toolchain_root.join("assets").join("baml-vscode.vsix"); + if !vsix.exists() { + anyhow::bail!( + "active BAML toolchain does not include assets/baml-vscode.vsix at {}", + vsix.display() + ); + } + Ok(vsix) +} + +fn command_on_path(command: &str) -> Option { + let path = env::var_os("PATH")?; + for dir in env::split_paths(&path) { + let candidate = dir.join(command); + if candidate.exists() { + return Some(OsString::from(command)); + } + #[cfg(windows)] + { + let candidate = dir.join(format!("{command}.cmd")); + if candidate.exists() { + return Some(OsString::from(format!("{command}.cmd"))); + } + } + } + None +} diff --git a/baml_language/crates/baml_cli/src/lib.rs b/baml_language/crates/baml_cli/src/lib.rs index 4b01a43a3db..0531dd030df 100644 --- a/baml_language/crates/baml_cli/src/lib.rs +++ b/baml_language/crates/baml_cli/src/lib.rs @@ -16,6 +16,7 @@ mod describe_command_tests; pub(crate) mod format; pub(crate) mod generate; pub(crate) mod grep_command; +pub(crate) mod ide_command; pub(crate) mod init_command; pub(crate) mod lsp; pub(crate) mod pack_command; diff --git a/baml_language/crates/baml_cli/src/main.rs b/baml_language/crates/baml_cli/src/main.rs index 7bb0ab15676..38047ce2492 100644 --- a/baml_language/crates/baml_cli/src/main.rs +++ b/baml_language/crates/baml_cli/src/main.rs @@ -3,10 +3,14 @@ // TODO: baml_runtime is disabled for now // use baml_runtime::RuntimeCliDefaults; +use std::io::Write as _; + fn main() { // TODO: baml_log is disabled for now // baml_log::init()?; + warn_if_direct_invocation(); + let argv: Vec = std::env::args().collect(); // Route every top-level error through the ariadne-style printer @@ -23,3 +27,19 @@ fn main() { }; std::process::exit(exit_code.into()); } + +fn warn_if_direct_invocation() { + if env_flag("BAML_WRAPPER_EXEC") || env_flag("BAML_CLI_ALLOW_DIRECT") { + return; + } + let _ = writeln!( + std::io::stderr(), + "warning: using the internal BAML toolchain binary directly is not recommended. Use `baml` instead." + ); +} + +fn env_flag(name: &str) -> bool { + std::env::var(name) + .map(|value| matches!(value.trim().to_ascii_lowercase().as_str(), "1" | "true")) + .unwrap_or(false) +} diff --git a/baml_language/crates/baml_cli/src/pack_command.rs b/baml_language/crates/baml_cli/src/pack_command.rs index 7acd263c146..aaf7c1ce473 100644 --- a/baml_language/crates/baml_cli/src/pack_command.rs +++ b/baml_language/crates/baml_cli/src/pack_command.rs @@ -25,10 +25,8 @@ use std::{ collections::HashMap, - io::{Cursor, Read}, path::{Path, PathBuf}, sync::Arc, - time::Duration, }; use anyhow::{Context, Result, anyhow}; @@ -38,7 +36,6 @@ use baml_project::ProjectDatabase; use bex_engine::BexEngine; use bex_vm_types::types::Program; use clap::Args; -use sha2::{Digest, Sha256}; use sys_native::SysOpsExt; use crate::{ @@ -509,92 +506,16 @@ fn host_binary_name(target_triple: &str) -> String { fn download_host_binary_from_release(target: &str, host_name: &str) -> Result> { let version = release_version_for_download(); - let url = release_archive_url(&version, target); - - let archive_bytes = download_release_asset(&url)?; - verify_release_archive_checksum(&archive_bytes, &url)?; - - extract_host_from_archive(&archive_bytes, &url, host_name) -} - -fn download_release_asset(url: &str) -> Result> { - let rt = tokio::runtime::Runtime::new() - .context("Failed to create tokio runtime for release asset download")?; - rt.block_on(async { - let client = reqwest::Client::builder() - .timeout(Duration::from_secs(120)) - .build() - .context("Failed to build HTTP client for release asset download")?; - let response = client - .get(url) - .send() - .await - .with_context(|| format!("Failed to download BAML release asset from {url}"))? - .error_for_status() - .with_context(|| format!("Failed to download BAML release asset from {url}"))?; - let bytes = response - .bytes() - .await - .with_context(|| format!("Failed to read BAML release asset from {url}"))?; - Ok(bytes.to_vec()) - }) -} - -fn verify_release_archive_checksum(archive_bytes: &[u8], archive_url: &str) -> Result<()> { - let checksum_url = release_archive_checksum_url(archive_url); - let checksum_text = download_release_asset(&checksum_url)?; - let checksum_text = std::str::from_utf8(&checksum_text) - .with_context(|| format!("Checksum asset {checksum_url} was not valid UTF-8"))?; - verify_release_archive_checksum_text(archive_bytes, archive_url, checksum_text).with_context( - || format!("Failed to verify BAML release archive checksum from {checksum_url}"), - ) -} - -fn verify_release_archive_checksum_text( - archive_bytes: &[u8], - archive_url: &str, - checksum_text: &str, -) -> Result<()> { - let archive_name = archive_url - .rsplit('/') - .next() - .ok_or_else(|| anyhow!("Archive URL did not contain a file name: {archive_url}"))?; - let expected = parse_release_checksum(checksum_text, archive_name)?; - let actual = format!("{:x}", Sha256::digest(archive_bytes)); - if actual != expected { - anyhow::bail!( - "Checksum mismatch for BAML release archive {archive_name}: expected {expected}, got {actual}" - ); - } - Ok(()) -} - -fn release_archive_checksum_url(archive_url: &str) -> String { - format!("{archive_url}.sha256") -} - -fn parse_release_checksum(checksum_text: &str, archive_name: &str) -> Result { - for line in checksum_text.lines() { - let mut parts = line.split_whitespace(); - let Some(hash) = parts.next() else { - continue; - }; - let Some(name) = parts.next() else { - continue; - }; - if name == archive_name { - return validate_sha256(hash); - } - } - anyhow::bail!("Checksum file did not contain an entry for {archive_name}") -} - -fn validate_sha256(hash: &str) -> Result { - if hash.len() == 64 && hash.chars().all(|c| c.is_ascii_hexdigit()) { - Ok(hash.to_ascii_lowercase()) - } else { - anyhow::bail!("Invalid SHA-256 checksum `{hash}`") - } + let fetcher = baml_release::Fetcher::default_for( + baml_release::ReleaseSpec { + version, + target: target.to_string(), + }, + baml_release::Product::Toolchain, + ); + fetcher + .fetch_binary(host_name) + .map_err(|err| anyhow!("{err}")) } fn release_version_for_download() -> String { @@ -604,77 +525,15 @@ fn release_version_for_download() -> String { .unwrap_or_else(|| release_version().to_string()) } -fn release_archive_url(version: &str, target: &str) -> String { - if let Ok(base_url) = std::env::var("BAML_PACK_HOST_RELEASE_BASE_URL") { - let base = base_url.trim_end_matches('/'); - return format!("{base}/{}", release_archive_filename(version, target)); - } - - let repo = std::env::var("BAML_PACK_HOST_RELEASE_REPO") - .ok() - .filter(|v| !v.trim().is_empty()) - .unwrap_or_else(|| "BoundaryML/baml".to_string()); - release_archive_url_for_repo(version, target, &repo) -} - -fn release_archive_url_for_repo(version: &str, target: &str, repo: &str) -> String { - format!( - "https://github.com/{repo}/releases/download/baml-language-{version}/{}", - release_archive_filename(version, target) - ) -} - -fn release_archive_filename(version: &str, target: &str) -> String { - let ext = if target.ends_with("windows-msvc") { - "zip" - } else { - "tar.gz" - }; - format!("baml-language-{version}-{target}.{ext}") -} - fn release_host_target_triple() -> Result<&'static str> { - #[cfg(target_env = "musl")] - const IS_MUSL: bool = true; - #[cfg(not(target_env = "musl"))] - const IS_MUSL: bool = false; - - match (std::env::consts::OS, std::env::consts::ARCH, IS_MUSL) { - ("macos", "aarch64", _) => Ok("aarch64-apple-darwin"), - ("macos", "x86_64", _) => Ok("x86_64-apple-darwin"), - ("linux", "aarch64", true) => Ok("aarch64-unknown-linux-musl"), - ("linux", "aarch64", false) => Ok("aarch64-unknown-linux-gnu"), - ("linux", "x86_64", true) => Ok("x86_64-unknown-linux-musl"), - ("linux", "x86_64", false) => Ok("x86_64-unknown-linux-gnu"), - ("windows", "x86_64", _) => Ok("x86_64-pc-windows-msvc"), - (os, arch, _) => anyhow::bail!( - "No released `baml-pack-host` artifact is available for {arch}-{os}. \ - Install `baml-pack-host` next to the `baml` binary to use `baml pack` on this platform." - ), - } + baml_release::release_host_target_triple() } fn validate_release_target_triple(target: &str) -> Result<&str> { - if SUPPORTED_PACK_TARGETS.contains(&target) { - Ok(target) - } else { - anyhow::bail!( - "Unsupported pack target `{target}`. Supported targets: {}", - SUPPORTED_PACK_TARGETS.join(", ") - ) - } + baml_release::validate_release_target_triple(target) + .map_err(|err| anyhow!("Unsupported pack target `{target}`. {err}")) } -const SUPPORTED_PACK_TARGETS: &[&str] = &[ - "aarch64-apple-darwin", - "x86_64-apple-darwin", - "aarch64-unknown-linux-gnu", - "aarch64-unknown-linux-musl", - "x86_64-unknown-linux-gnu", - "x86_64-unknown-linux-musl", - "x86_64-pc-windows-msvc", -]; - /// Heuristic: does this positional `` look like a filesystem /// path rather than a function name? Triggers when the user typed /// something like `baml_src/main.baml` and we want to redirect them to @@ -695,57 +554,6 @@ fn default_output_path(default_basename: &str, target_triple: &str) -> PathBuf { path } -fn extract_host_from_archive(archive_bytes: &[u8], url: &str, host_name: &str) -> Result> { - if url.ends_with(".zip") { - extract_host_from_zip(archive_bytes, host_name) - } else { - extract_host_from_tar_gz(archive_bytes, host_name) - } - .with_context(|| format!("Failed to extract `{host_name}` from release archive {url}")) -} - -fn extract_host_from_tar_gz(archive_bytes: &[u8], host_name: &str) -> Result> { - let decoder = flate2::read::GzDecoder::new(Cursor::new(archive_bytes)); - let mut archive = tar::Archive::new(decoder); - for entry in archive.entries().context("Failed to read tar entries")? { - let mut entry = entry.context("Failed to read tar entry")?; - if entry - .path() - .ok() - .and_then(|path| path.file_name().map(|name| name == host_name)) - .unwrap_or(false) - { - let mut bytes = Vec::new(); - entry - .read_to_end(&mut bytes) - .context("Failed to read host binary from tar archive")?; - return Ok(bytes); - } - } - anyhow::bail!("Release archive did not contain `{host_name}`") -} - -fn extract_host_from_zip(archive_bytes: &[u8], host_name: &str) -> Result> { - let mut archive = - zip::ZipArchive::new(Cursor::new(archive_bytes)).context("Failed to read zip archive")?; - for i in 0..archive.len() { - let mut file = archive - .by_index(i) - .with_context(|| format!("Failed to read zip entry {i}"))?; - if Path::new(file.name()) - .file_name() - .map(|name| name == host_name) - .unwrap_or(false) - { - let mut bytes = Vec::new(); - file.read_to_end(&mut bytes) - .context("Failed to read host binary from zip archive")?; - return Ok(bytes); - } - } - anyhow::bail!("Release archive did not contain `{host_name}`") -} - fn write_executable( host_bytes: &[u8], data: &[u8], @@ -1217,62 +1025,9 @@ mod tests { assert_eq!(args.resolve_output_basename().unwrap(), "hello"); } - // ── GitHub release fallback ─────────────────────────────────────── - - #[test] - fn test_release_archive_filename_uses_platform_extension() { - assert_eq!( - release_archive_filename("1.2.3-alpha.4", "x86_64-unknown-linux-gnu"), - "baml-language-1.2.3-alpha.4-x86_64-unknown-linux-gnu.tar.gz" - ); - assert_eq!( - release_archive_filename("1.2.3-alpha.4", "x86_64-pc-windows-msvc"), - "baml-language-1.2.3-alpha.4-x86_64-pc-windows-msvc.zip" - ); - } - - #[test] - fn test_release_archive_url_defaults_to_github_release_asset() { - let url = release_archive_url_for_repo( - "1.2.3-alpha.4", - "x86_64-unknown-linux-gnu", - "BoundaryML/baml", - ); - assert_eq!( - url, - "https://github.com/BoundaryML/baml/releases/download/baml-language-1.2.3-alpha.4/baml-language-1.2.3-alpha.4-x86_64-unknown-linux-gnu.tar.gz" - ); - } - - #[test] - fn test_release_archive_checksum_url_matches_uploaded_asset() { - assert_eq!( - release_archive_checksum_url( - "https://github.com/BoundaryML/baml/releases/download/baml-language-1.2.3-alpha.4/baml-language-1.2.3-alpha.4-x86_64-unknown-linux-gnu.tar.gz" - ), - "https://github.com/BoundaryML/baml/releases/download/baml-language-1.2.3-alpha.4/baml-language-1.2.3-alpha.4-x86_64-unknown-linux-gnu.tar.gz.sha256" - ); - } - - #[test] - fn test_verify_release_archive_checksum_text() { - let archive_name = "baml-language-1.2.3-alpha.4-x86_64-unknown-linux-gnu.tar.gz"; - let archive_url = format!("https://example.com/releases/{archive_name}"); - let archive_bytes = b"fake archive bytes"; - let digest = format!("{:x}", Sha256::digest(archive_bytes)); - let checksum_text = format!("{digest} {archive_name}\n"); - - verify_release_archive_checksum_text(archive_bytes, &archive_url, &checksum_text).unwrap(); - - let err = - verify_release_archive_checksum_text(b"different bytes", &archive_url, &checksum_text) - .unwrap_err(); - assert!(format!("{err}").contains("Checksum mismatch")); - } - #[test] fn test_validate_release_target_triple_accepts_supported_targets() { - for target in SUPPORTED_PACK_TARGETS { + for target in baml_release::SUPPORTED_RELEASE_TARGETS { assert_eq!(validate_release_target_triple(target).unwrap(), *target); } } @@ -1313,49 +1068,6 @@ mod tests { ); } - #[test] - fn test_extract_host_from_tar_gz() { - let mut gzip = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); - { - let mut builder = tar::Builder::new(&mut gzip); - let bytes = b"fake host"; - let mut header = tar::Header::new_gnu(); - header.set_size(bytes.len() as u64); - header.set_mode(0o755); - header.set_cksum(); - builder - .append_data(&mut header, "nested/baml-pack-host", &bytes[..]) - .unwrap(); - builder.finish().unwrap(); - } - let archive = gzip.finish().unwrap(); - assert_eq!( - extract_host_from_tar_gz(&archive, "baml-pack-host").unwrap(), - b"fake host" - ); - } - - #[test] - fn test_extract_host_from_zip() { - use std::io::Write; - - let mut archive = Cursor::new(Vec::new()); - { - let mut zip = zip::ZipWriter::new(&mut archive); - zip.start_file( - "nested/baml-pack-host.exe", - zip::write::SimpleFileOptions::default(), - ) - .unwrap(); - zip.write_all(b"fake windows host").unwrap(); - zip.finish().unwrap(); - } - assert_eq!( - extract_host_from_zip(&archive.into_inner(), "baml-pack-host.exe").unwrap(), - b"fake windows host" - ); - } - // ── Envelope roundtrip ──────────────────────────────────────────── /// The PackEnvelope borsh roundtrip is the wire contract between diff --git a/baml_language/crates/baml_lsp_server/Cargo.toml b/baml_language/crates/baml_lsp_server/Cargo.toml index 60033c35b8b..fbe7d149723 100644 --- a/baml_language/crates/baml_lsp_server/Cargo.toml +++ b/baml_language/crates/baml_lsp_server/Cargo.toml @@ -30,6 +30,7 @@ rustls = [ "aws-crypto" ] [dependencies] baml_compiler2_visualization = { workspace = true } baml_type = { workspace = true } +baml_version = { workspace = true } bex_events = { workspace = true } bex_events_native = { workspace = true } bex_heap = { workspace = true } diff --git a/baml_language/crates/baml_lsp_server/src/lib.rs b/baml_language/crates/baml_lsp_server/src/lib.rs index 5e3d8a989e1..b13e4793e7a 100644 --- a/baml_language/crates/baml_lsp_server/src/lib.rs +++ b/baml_language/crates/baml_lsp_server/src/lib.rs @@ -51,7 +51,7 @@ use playground_ws::WsOutMessage; use tokio::net::TcpListener; pub fn version() -> &'static str { - env!("CARGO_PKG_VERSION") + baml_version::CANONICAL_VERSION } /// Build `SysOps` for a playground-connected project. diff --git a/baml_language/crates/baml_lsp_server/src/playground_server.rs b/baml_language/crates/baml_lsp_server/src/playground_server.rs index df988ac3115..167cca23b54 100644 --- a/baml_language/crates/baml_lsp_server/src/playground_server.rs +++ b/baml_language/crates/baml_lsp_server/src/playground_server.rs @@ -143,6 +143,23 @@ async fn playground_ws_session(socket: WebSocket, state: WsState) { tracing::info!("Playground: WS session started"); let (mut sink, mut stream) = socket.split(); + if let Some(hello) = to_ws_text(&WsOutMessage::Hello { + toolchain_version: baml_version::CANONICAL_VERSION.to_string(), + playground_protocol: 1, + min_client_playground_protocol: 1, + capabilities: vec![ + "playgroundWebSocket.v1".to_string(), + "callFunction.v1".to_string(), + "collectTests.v1".to_string(), + ], + }) { + if sink.send(hello).await.is_err() { + return; + } + } else { + return; + } + if let Some(ready) = to_ws_text(&WsOutMessage::Ready) { if sink.send(ready).await.is_err() { return; diff --git a/baml_language/crates/baml_lsp_server/src/playground_ws.rs b/baml_language/crates/baml_lsp_server/src/playground_ws.rs index 913d9b87293..3b1a488f4db 100644 --- a/baml_language/crates/baml_lsp_server/src/playground_ws.rs +++ b/baml_language/crates/baml_lsp_server/src/playground_ws.rs @@ -82,6 +82,16 @@ pub enum WsInMessage { #[derive(Debug, Serialize, Clone)] #[serde(tag = "type")] pub enum WsOutMessage { + #[serde(rename = "hello")] + Hello { + #[serde(rename = "toolchainVersion")] + toolchain_version: String, + #[serde(rename = "playgroundProtocol")] + playground_protocol: u32, + #[serde(rename = "minClientPlaygroundProtocol")] + min_client_playground_protocol: u32, + capabilities: Vec, + }, #[serde(rename = "ready")] Ready, #[serde(rename = "playgroundNotification")] diff --git a/baml_language/crates/baml_release/Cargo.toml b/baml_language/crates/baml_release/Cargo.toml new file mode 100644 index 00000000000..4510b6c3adb --- /dev/null +++ b/baml_language/crates/baml_release/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "baml_release" +version = { workspace = true } +publish = false +authors = { workspace = true } +edition = { workspace = true } +rust-version = { workspace = true } +homepage = { workspace = true } +documentation = { workspace = true } +repository = { workspace = true } +license = { workspace = true } + +[lib] +doctest = false + +[lints] +workspace = true + +[dependencies] +anyhow = { workspace = true } +flate2 = { workspace = true } +reqwest = { workspace = true, features = [ "blocking", "rustls" ] } +serde = { workspace = true, features = [ "derive" ] } +serde_json = { workspace = true } +sha2 = { workspace = true } +tar = { workspace = true } +thiserror = { workspace = true } +zip = { workspace = true } + +[package.metadata.ci] +wasm_support = false diff --git a/baml_language/crates/baml_release/src/lib.rs b/baml_language/crates/baml_release/src/lib.rs new file mode 100644 index 00000000000..846109c43e9 --- /dev/null +++ b/baml_language/crates/baml_release/src/lib.rs @@ -0,0 +1,825 @@ +#![allow(clippy::print_stdout, clippy::print_stderr)] + +pub mod manifest; + +use std::{ + fs::{self, OpenOptions}, + io::{Cursor, Read}, + path::{Path, PathBuf}, + thread, + time::Duration, +}; + +use anyhow::{Context, Result}; +pub use manifest::{Artifact, Channel, ToolchainManifest, WrapperManifest}; +use sha2::{Digest, Sha256}; + +pub const MANIFEST_SCHEMA: u32 = 1; +pub const DEFAULT_MANIFEST_BASE_URL: &str = "https://pkg.boundaryml.com/manifest/v1"; +pub const DEFAULT_RELEASE_REPO: &str = "BoundaryML/baml"; + +pub const SUPPORTED_RELEASE_TARGETS: &[&str] = &[ + "aarch64-apple-darwin", + "x86_64-apple-darwin", + "aarch64-unknown-linux-gnu", + "aarch64-unknown-linux-musl", + "x86_64-unknown-linux-gnu", + "x86_64-unknown-linux-musl", + "x86_64-pc-windows-msvc", + "aarch64-pc-windows-msvc", +]; + +#[derive(Debug, Clone)] +pub struct ReleaseSpec { + pub version: String, + pub target: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Product { + Toolchain, + Wrapper, +} + +impl Product { + pub fn tag_prefix(self) -> &'static str { + match self { + Product::Toolchain => "baml-language", + Product::Wrapper => "baml-wrapper", + } + } +} + +#[derive(Debug, thiserror::Error)] +pub enum FetchError { + #[error("network error fetching {url}: {source}")] + Network { url: String, source: reqwest::Error }, + #[error("manifest 404 for version {version} (not released yet?)")] + ManifestNotFound { version: String }, + #[error("manifest schema {got} not supported (max {max}); run `baml self-update`")] + ManifestSchemaTooNew { got: u32, max: u32 }, + #[error("target {target} not built for version {version}")] + TargetNotInManifest { target: String, version: String }, + #[error("sha256 mismatch for {url}: expected {expected}, got {got}")] + ChecksumMismatch { + url: String, + expected: String, + got: String, + }, + #[error("archive missing expected binary {name}")] + BinaryNotInArchive { name: String }, + #[error("archive contains unsafe path {path}")] + UnsafeArchivePath { path: String }, + #[error("disk error: {0}")] + Io(#[from] std::io::Error), + #[error("zip archive error: {0}")] + Zip(#[from] zip::result::ZipError), +} + +#[derive(Debug, Clone)] +pub struct Fetcher { + pub spec: ReleaseSpec, + pub product: Product, + pub manifest_base_url: String, + pub release_repo: String, + artifact: Option, +} + +impl Fetcher { + pub fn default_for(spec: ReleaseSpec, product: Product) -> Self { + Self { + spec, + product, + manifest_base_url: manifest_base_url(), + release_repo: release_repo(), + artifact: None, + } + } + + pub fn from_artifact(spec: ReleaseSpec, product: Product, artifact: Artifact) -> Self { + Self { + spec, + product, + manifest_base_url: manifest_base_url(), + release_repo: release_repo(), + artifact: Some(artifact), + } + } + + pub fn artifact_url(&self) -> String { + if let Some(artifact) = &self.artifact { + return artifact.url.clone(); + } + if let Ok(base_url) = std::env::var("BAML_PACK_HOST_RELEASE_BASE_URL") { + let base = base_url.trim_end_matches('/'); + return format!( + "{base}/{}", + release_archive_filename(self.product, &self.spec.version, &self.spec.target) + ); + } + release_archive_url_for_repo( + self.product, + &self.spec.version, + &self.spec.target, + &self.release_repo, + ) + } + + pub fn fetch_archive(&self) -> Result, FetchError> { + let url = self.artifact_url(); + let archive = download_bytes(&url)?; + if let Some(artifact) = &self.artifact { + verify_sha256(&archive, &url, &artifact.sha256)?; + } else { + let checksum_url = format!("{url}.sha256"); + let checksum = download_bytes(&checksum_url)?; + let checksum_text = + std::str::from_utf8(&checksum).map_err(|_| FetchError::UnsafeArchivePath { + path: format!("{checksum_url} was not valid UTF-8"), + })?; + verify_release_archive_checksum_text(&archive, &url, checksum_text)?; + } + validate_archive_layout(self.product, &self.spec.target, &archive, &url)?; + Ok(archive) + } + + pub fn fetch_binary(&self, binary_name: &str) -> Result, FetchError> { + let url = self.artifact_url(); + let archive = self.fetch_archive()?; + extract_binary_from_archive(&archive, &url, binary_name) + } + + pub fn install_to_toolchain_root(&self, root: &Path) -> Result { + fs::create_dir_all(root)?; + let _lock = InstallLock::acquire(root.join(format!(".{}.lock", self.spec.version)))?; + let archive = self.fetch_archive()?; + let install_dir = root.join(&self.spec.version); + if fs::read_to_string(install_dir.join("VERSION")) + .map(|version| version.trim() == self.spec.version) + .unwrap_or(false) + { + return Ok(install_dir); + } + + let tmp_dir = root.join(format!(".{}.tmp", self.spec.version)); + if tmp_dir.exists() { + fs::remove_dir_all(&tmp_dir)?; + } + fs::create_dir_all(&tmp_dir)?; + extract_archive_to_dir(&archive, &self.artifact_url(), &tmp_dir)?; + fs::write(tmp_dir.join("VERSION"), format!("{}\n", self.spec.version))?; + fs::write( + tmp_dir.join("install.json"), + serde_json::json!({ + "version": self.spec.version, + "target": self.spec.target, + "archive_url": self.artifact_url(), + }) + .to_string(), + )?; + + if install_dir.exists() { + fs::remove_dir_all(&install_dir)?; + } + fs::rename(&tmp_dir, &install_dir)?; + Ok(install_dir) + } +} + +struct InstallLock { + path: PathBuf, +} + +impl InstallLock { + fn acquire(path: PathBuf) -> Result { + for _ in 0..60 { + match OpenOptions::new().write(true).create_new(true).open(&path) { + Ok(_) => return Ok(Self { path }), + Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => { + thread::sleep(Duration::from_secs(1)); + } + Err(err) => return Err(FetchError::Io(err)), + } + } + Err(FetchError::Io(std::io::Error::new( + std::io::ErrorKind::TimedOut, + format!("timed out waiting for install lock {}", path.display()), + ))) + } +} + +impl Drop for InstallLock { + fn drop(&mut self) { + let _ = fs::remove_file(&self.path); + } +} + +pub fn manifest_base_url() -> String { + std::env::var("BAML_MANIFEST_BASE_URL") + .ok() + .filter(|value| !value.trim().is_empty()) + .unwrap_or_else(|| DEFAULT_MANIFEST_BASE_URL.to_string()) + .trim_end_matches('/') + .to_string() +} + +pub fn release_repo() -> String { + std::env::var("BAML_PACK_HOST_RELEASE_REPO") + .ok() + .filter(|value| !value.trim().is_empty()) + .unwrap_or_else(|| DEFAULT_RELEASE_REPO.to_string()) +} + +pub fn release_host_target_triple() -> Result<&'static str> { + #[cfg(target_env = "musl")] + const IS_MUSL: bool = true; + #[cfg(not(target_env = "musl"))] + const IS_MUSL: bool = false; + + match (std::env::consts::OS, std::env::consts::ARCH, IS_MUSL) { + ("macos", "aarch64", _) => Ok("aarch64-apple-darwin"), + ("macos", "x86_64", _) => Ok("x86_64-apple-darwin"), + ("linux", "aarch64", true) => Ok("aarch64-unknown-linux-musl"), + ("linux", "aarch64", false) => Ok("aarch64-unknown-linux-gnu"), + ("linux", "x86_64", true) => Ok("x86_64-unknown-linux-musl"), + ("linux", "x86_64", false) => Ok("x86_64-unknown-linux-gnu"), + ("windows", "x86_64", _) => Ok("x86_64-pc-windows-msvc"), + ("windows", "aarch64", _) => Ok("aarch64-pc-windows-msvc"), + (os, arch, _) => anyhow::bail!( + "No released BAML artifact is available for {arch}-{os}. \ + Install the matching toolchain via `baml toolchain install `." + ), + } +} + +pub fn validate_release_target_triple(target: &str) -> Result<&str> { + if SUPPORTED_RELEASE_TARGETS.contains(&target) { + Ok(target) + } else { + anyhow::bail!( + "Unsupported release target `{target}`. Supported targets: {}", + SUPPORTED_RELEASE_TARGETS.join(", ") + ) + } +} + +pub fn release_archive_filename(product: Product, version: &str, target: &str) -> String { + let ext = if target.ends_with("windows-msvc") { + "zip" + } else { + "tar.gz" + }; + format!("{}-{version}-{target}.{ext}", product.tag_prefix()) +} + +pub fn release_archive_url_for_repo( + product: Product, + version: &str, + target: &str, + repo: &str, +) -> String { + format!( + "https://github.com/{repo}/releases/download/{}-{version}/{}", + product.tag_prefix(), + release_archive_filename(product, version, target) + ) +} + +pub fn validate_sha256(hash: &str) -> Result { + if hash.len() == 64 && hash.chars().all(|c| c.is_ascii_hexdigit()) { + Ok(hash.to_ascii_lowercase()) + } else { + anyhow::bail!("Invalid SHA-256 checksum `{hash}`") + } +} + +pub fn verify_release_archive_checksum_text( + archive_bytes: &[u8], + archive_url: &str, + checksum_text: &str, +) -> Result<(), FetchError> { + let archive_name = archive_url.rsplit('/').next().unwrap_or(archive_url); + let expected = parse_release_checksum(checksum_text, archive_name).map_err(|_| { + FetchError::BinaryNotInArchive { + name: archive_name.to_string(), + } + })?; + let got = format!("{:x}", Sha256::digest(archive_bytes)); + compare_sha256(archive_url, &expected, &got) +} + +pub fn verify_sha256( + archive_bytes: &[u8], + archive_url: &str, + expected: &str, +) -> Result<(), FetchError> { + let expected = validate_sha256(expected).map_err(|_| FetchError::ChecksumMismatch { + url: archive_url.to_string(), + expected: expected.to_string(), + got: "invalid sha256".to_string(), + })?; + let got = format!("{:x}", Sha256::digest(archive_bytes)); + compare_sha256(archive_url, &expected, &got) +} + +fn compare_sha256(archive_url: &str, expected: &str, got: &str) -> Result<(), FetchError> { + if got != expected { + return Err(FetchError::ChecksumMismatch { + url: archive_url.to_string(), + expected: expected.to_string(), + got: got.to_string(), + }); + } + Ok(()) +} + +pub fn parse_release_checksum(checksum_text: &str, archive_name: &str) -> Result { + for line in checksum_text.lines() { + let mut parts = line.split_whitespace(); + let Some(hash) = parts.next() else { + continue; + }; + let Some(name) = parts.next() else { + continue; + }; + if name == archive_name { + return validate_sha256(hash); + } + } + anyhow::bail!("Checksum file did not contain an entry for {archive_name}") +} + +fn download_bytes(url: &str) -> Result, FetchError> { + let client = reqwest::blocking::Client::builder() + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(600)) + .user_agent("baml-release/1") + .build() + .map_err(|source| FetchError::Network { + url: url.to_string(), + source, + })?; + + let mut last_error = None; + for attempt in 1..=3 { + match client.get(url).send() { + Ok(response) if response.status() == reqwest::StatusCode::NOT_FOUND => { + return Err(FetchError::ManifestNotFound { + version: url.to_string(), + }); + } + Ok(response) if response.status().is_success() => { + return response + .bytes() + .map(|bytes| bytes.to_vec()) + .map_err(|source| FetchError::Network { + url: url.to_string(), + source, + }); + } + Ok(response) + if response.status().is_server_error() || response.status().is_redirection() => + { + last_error = response.error_for_status().err(); + } + Ok(response) => { + return response + .error_for_status() + .map(|_| Vec::new()) + .map_err(|source| FetchError::Network { + url: url.to_string(), + source, + }); + } + Err(source) => { + last_error = Some(source); + } + } + if attempt < 3 { + thread::sleep(Duration::from_secs(2)); + } + } + Err(FetchError::Network { + url: url.to_string(), + source: last_error.expect("retry loop should capture a network error"), + }) +} + +pub fn extract_binary_from_archive( + archive_bytes: &[u8], + url: &str, + binary_name: &str, +) -> Result, FetchError> { + if archive_url_is_zip(url) { + extract_binary_from_zip(archive_bytes, binary_name) + } else { + extract_binary_from_tar_gz(archive_bytes, binary_name) + } +} + +fn extract_binary_from_tar_gz( + archive_bytes: &[u8], + binary_name: &str, +) -> Result, FetchError> { + let decoder = flate2::read::GzDecoder::new(Cursor::new(archive_bytes)); + let mut archive = tar::Archive::new(decoder); + for entry in archive.entries()? { + let mut entry = entry?; + if entry + .path() + .ok() + .and_then(|path| path.file_name().map(|name| name == binary_name)) + .unwrap_or(false) + { + let mut bytes = Vec::new(); + entry.read_to_end(&mut bytes)?; + return Ok(bytes); + } + } + Err(FetchError::BinaryNotInArchive { + name: binary_name.to_string(), + }) +} + +fn extract_binary_from_zip(archive_bytes: &[u8], binary_name: &str) -> Result, FetchError> { + let mut archive = zip::ZipArchive::new(Cursor::new(archive_bytes))?; + for i in 0..archive.len() { + let mut file = archive.by_index(i)?; + if Path::new(file.name()) + .file_name() + .map(|name| name == binary_name) + .unwrap_or(false) + { + let mut bytes = Vec::new(); + file.read_to_end(&mut bytes)?; + return Ok(bytes); + } + } + Err(FetchError::BinaryNotInArchive { + name: binary_name.to_string(), + }) +} + +pub fn extract_archive_to_dir( + archive_bytes: &[u8], + url: &str, + dest: &Path, +) -> Result<(), FetchError> { + if archive_url_is_zip(url) { + extract_zip_to_dir(archive_bytes, dest) + } else { + extract_tar_gz_to_dir(archive_bytes, dest) + } +} + +fn safe_join(dest: &Path, entry_path: &Path) -> Result { + if entry_path.is_absolute() + || entry_path + .components() + .any(|c| matches!(c, std::path::Component::ParentDir)) + { + return Err(FetchError::UnsafeArchivePath { + path: entry_path.display().to_string(), + }); + } + Ok(dest.join(entry_path)) +} + +fn extract_tar_gz_to_dir(archive_bytes: &[u8], dest: &Path) -> Result<(), FetchError> { + let decoder = flate2::read::GzDecoder::new(Cursor::new(archive_bytes)); + let mut archive = tar::Archive::new(decoder); + for entry in archive.entries()? { + let mut entry = entry?; + let path = entry.path()?.into_owned(); + let out = safe_join(dest, &path)?; + if entry.header().entry_type().is_dir() { + fs::create_dir_all(out)?; + } else if entry.header().entry_type().is_file() { + if let Some(parent) = out.parent() { + fs::create_dir_all(parent)?; + } + entry + .unpack(out) + .context("extract tar entry") + .map_err(|err| FetchError::Io(std::io::Error::other(err)))?; + } + } + Ok(()) +} + +fn validate_archive_layout( + product: Product, + target: &str, + archive_bytes: &[u8], + url: &str, +) -> Result<(), FetchError> { + let paths = archive_paths(archive_bytes, url)?; + let exe_suffix = if target.ends_with("windows-msvc") { + ".exe" + } else { + "" + }; + let required = match product { + Product::Toolchain => vec![ + format!("bin/baml-cli{exe_suffix}"), + format!("bin/baml-pack-host{exe_suffix}"), + "assets/baml-vscode.vsix".to_string(), + ], + Product::Wrapper => vec![format!("bin/baml{exe_suffix}")], + }; + for path in &required { + if !paths.iter().any(|entry| entry == path) { + return Err(FetchError::BinaryNotInArchive { name: path.clone() }); + } + } + let forbidden = match product { + Product::Toolchain => vec![format!("bin/baml{exe_suffix}")], + Product::Wrapper => vec![ + format!("bin/baml-cli{exe_suffix}"), + format!("bin/baml-pack-host{exe_suffix}"), + "assets/baml-vscode.vsix".to_string(), + ], + }; + for path in &forbidden { + if paths.iter().any(|entry| entry == path) { + return Err(FetchError::UnsafeArchivePath { + path: format!("archive unexpectedly contains {path}"), + }); + } + } + Ok(()) +} + +fn normalize_archive_path(path: &Path) -> Result { + let mut parts = Vec::new(); + for component in path.components() { + match component { + std::path::Component::Normal(part) => parts.push(part.to_string_lossy().to_string()), + std::path::Component::CurDir => {} + std::path::Component::ParentDir + | std::path::Component::RootDir + | std::path::Component::Prefix(_) => { + return Err(FetchError::UnsafeArchivePath { + path: path.display().to_string(), + }); + } + } + } + Ok(parts.join("/")) +} + +fn archive_paths(archive_bytes: &[u8], url: &str) -> Result, FetchError> { + if archive_url_is_zip(url) { + archive_paths_zip(archive_bytes) + } else { + archive_paths_tar_gz(archive_bytes) + } +} + +fn archive_paths_tar_gz(archive_bytes: &[u8]) -> Result, FetchError> { + let decoder = flate2::read::GzDecoder::new(Cursor::new(archive_bytes)); + let mut archive = tar::Archive::new(decoder); + let mut paths = Vec::new(); + for entry in archive.entries()? { + let entry = entry?; + let entry_type = entry.header().entry_type(); + if entry_type.is_symlink() || entry_type.is_hard_link() { + return Err(FetchError::UnsafeArchivePath { + path: entry.path()?.display().to_string(), + }); + } + if entry_type.is_file() { + paths.push(normalize_archive_path(&entry.path()?)?); + } + } + Ok(paths) +} + +fn archive_paths_zip(archive_bytes: &[u8]) -> Result, FetchError> { + let mut archive = zip::ZipArchive::new(Cursor::new(archive_bytes))?; + let mut paths = Vec::new(); + for i in 0..archive.len() { + let file = archive.by_index(i)?; + let Some(path) = file.enclosed_name() else { + return Err(FetchError::UnsafeArchivePath { + path: file.name().to_string(), + }); + }; + #[cfg(unix)] + if file + .unix_mode() + .is_some_and(|mode| mode & 0o170_000 == 0o120_000) + { + return Err(FetchError::UnsafeArchivePath { + path: file.name().to_string(), + }); + } + if file.is_file() { + paths.push(normalize_archive_path(&path)?); + } + } + Ok(paths) +} + +fn archive_url_is_zip(url: &str) -> bool { + Path::new(url) + .extension() + .is_some_and(|extension| extension.eq_ignore_ascii_case("zip")) +} + +fn extract_zip_to_dir(archive_bytes: &[u8], dest: &Path) -> Result<(), FetchError> { + let mut archive = zip::ZipArchive::new(Cursor::new(archive_bytes))?; + for i in 0..archive.len() { + let mut file = archive.by_index(i)?; + let Some(path) = file.enclosed_name() else { + return Err(FetchError::UnsafeArchivePath { + path: file.name().to_string(), + }); + }; + let out = safe_join(dest, &path)?; + if file.is_dir() { + fs::create_dir_all(out)?; + } else { + if let Some(parent) = out.parent() { + fs::create_dir_all(parent)?; + } + let mut output = fs::File::create(out)?; + std::io::copy(&mut file, &mut output)?; + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_release_archive_filename_uses_platform_extension() { + assert_eq!( + release_archive_filename( + Product::Toolchain, + "1.2.3-nightly.20260602.a", + "x86_64-unknown-linux-gnu" + ), + "baml-language-1.2.3-nightly.20260602.a-x86_64-unknown-linux-gnu.tar.gz" + ); + assert_eq!( + release_archive_filename(Product::Wrapper, "0.1.0", "aarch64-pc-windows-msvc"), + "baml-wrapper-0.1.0-aarch64-pc-windows-msvc.zip" + ); + } + + #[test] + fn test_release_archive_url_defaults_to_github_release_asset() { + let url = release_archive_url_for_repo( + Product::Toolchain, + "1.2.3-nightly.20260602.a", + "x86_64-unknown-linux-gnu", + "BoundaryML/baml", + ); + assert_eq!( + url, + "https://github.com/BoundaryML/baml/releases/download/baml-language-1.2.3-nightly.20260602.a/baml-language-1.2.3-nightly.20260602.a-x86_64-unknown-linux-gnu.tar.gz" + ); + } + + #[test] + fn test_verify_release_archive_checksum_text() { + let archive_name = "baml-language-1.2.3-nightly.20260602.a-x86_64-unknown-linux-gnu.tar.gz"; + let archive_url = format!("https://example.com/releases/{archive_name}"); + let archive_bytes = b"fake archive bytes"; + let digest = format!("{:x}", Sha256::digest(archive_bytes)); + let checksum_text = format!("{digest} {archive_name}\n"); + + verify_release_archive_checksum_text(archive_bytes, &archive_url, &checksum_text).unwrap(); + + let err = + verify_release_archive_checksum_text(b"different bytes", &archive_url, &checksum_text) + .unwrap_err(); + assert!(format!("{err}").contains("sha256 mismatch")); + } + + #[test] + fn test_validate_release_target_triple_accepts_supported_targets() { + for target in SUPPORTED_RELEASE_TARGETS { + assert_eq!(validate_release_target_triple(target).unwrap(), *target); + } + } + + #[test] + fn test_validate_release_target_triple_rejects_unknown_target() { + let err = validate_release_target_triple("wasm32-unknown-unknown").unwrap_err(); + let msg = format!("{err}"); + assert!(msg.contains("Unsupported release target"), "got: {msg}"); + assert!(msg.contains("aarch64-pc-windows-msvc"), "got: {msg}"); + } + + #[test] + fn test_extract_binary_from_tar_gz() { + let mut gzip = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + { + let mut builder = tar::Builder::new(&mut gzip); + let bytes = b"fake host"; + let mut header = tar::Header::new_gnu(); + header.set_size(bytes.len() as u64); + header.set_mode(0o755); + header.set_cksum(); + builder + .append_data(&mut header, "nested/baml-pack-host", &bytes[..]) + .unwrap(); + builder.finish().unwrap(); + } + let archive = gzip.finish().unwrap(); + assert_eq!( + extract_binary_from_archive( + &archive, + "https://example.com/baml-language-1.2.3-x86_64-unknown-linux-gnu.tar.gz", + "baml-pack-host" + ) + .unwrap(), + b"fake host" + ); + } + + #[test] + fn test_extract_binary_from_zip() { + use std::io::Write; + + let mut archive = Cursor::new(Vec::new()); + { + let mut zip = zip::ZipWriter::new(&mut archive); + zip.start_file( + "nested/baml-pack-host.exe", + zip::write::SimpleFileOptions::default(), + ) + .unwrap(); + zip.write_all(b"fake windows host").unwrap(); + zip.finish().unwrap(); + } + assert_eq!( + extract_binary_from_archive( + &archive.into_inner(), + "https://example.com/baml-language-1.2.3-x86_64-pc-windows-msvc.zip", + "baml-pack-host.exe" + ) + .unwrap(), + b"fake windows host" + ); + } + + #[test] + fn test_validate_toolchain_archive_layout_rejects_wrapper_binary() { + let mut gzip = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + { + let mut builder = tar::Builder::new(&mut gzip); + for path in [ + "bin/baml-cli", + "bin/baml-pack-host", + "assets/baml-vscode.vsix", + "bin/baml", + ] { + let bytes = b"fake"; + let mut header = tar::Header::new_gnu(); + header.set_size(bytes.len() as u64); + header.set_mode(0o755); + header.set_cksum(); + builder.append_data(&mut header, path, &bytes[..]).unwrap(); + } + builder.finish().unwrap(); + } + let archive = gzip.finish().unwrap(); + let err = validate_archive_layout( + Product::Toolchain, + "x86_64-unknown-linux-gnu", + &archive, + "https://example.com/archive.tar.gz", + ) + .unwrap_err(); + assert!(format!("{err}").contains("bin/baml")); + } + + #[test] + fn test_validate_wrapper_archive_layout_rejects_toolchain_payload() { + use std::io::Write; + + let mut archive = Cursor::new(Vec::new()); + { + let mut zip = zip::ZipWriter::new(&mut archive); + for path in ["bin/baml.exe", "bin/baml-cli.exe"] { + zip.start_file(path, zip::write::SimpleFileOptions::default()) + .unwrap(); + zip.write_all(b"fake").unwrap(); + } + zip.finish().unwrap(); + } + let err = validate_archive_layout( + Product::Wrapper, + "x86_64-pc-windows-msvc", + &archive.into_inner(), + "https://example.com/archive.zip", + ) + .unwrap_err(); + assert!(format!("{err}").contains("baml-cli")); + } +} diff --git a/baml_language/crates/baml_release/src/manifest.rs b/baml_language/crates/baml_release/src/manifest.rs new file mode 100644 index 00000000000..9c474a2840e --- /dev/null +++ b/baml_language/crates/baml_release/src/manifest.rs @@ -0,0 +1,119 @@ +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +use crate::{MANIFEST_SCHEMA, SUPPORTED_RELEASE_TARGETS, validate_sha256}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Channel { + Canary, + Nightly, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Artifact { + pub url: String, + pub sha256: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct VsixArtifact { + pub url: String, + pub sha256: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BamlCorePypi { + pub version: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ToolchainManifest { + pub schema: u32, + pub version: String, + pub channel: Channel, + pub released_at: String, + pub artifacts: BTreeMap, + #[serde(skip_serializing_if = "Option::is_none")] + pub vsix: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub baml_core_pypi: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WrapperManifest { + pub schema: u32, + pub version: String, + pub released_at: String, + pub artifacts: BTreeMap, +} + +impl ToolchainManifest { + pub fn validate(&self) -> anyhow::Result<()> { + validate_schema(self.schema)?; + validate_artifacts(&self.version, &self.artifacts)?; + if let Some(vsix) = &self.vsix { + validate_artifact( + "vsix", + &Artifact { + url: vsix.url.clone(), + sha256: vsix.sha256.clone(), + }, + )?; + } + Ok(()) + } + + pub fn artifact_for_target(&self, target: &str) -> anyhow::Result<&Artifact> { + self.artifacts.get(target).ok_or_else(|| { + anyhow::anyhow!("target {target} not built for version {}", self.version) + }) + } +} + +impl WrapperManifest { + pub fn validate(&self) -> anyhow::Result<()> { + validate_schema(self.schema)?; + validate_artifacts(&self.version, &self.artifacts) + } + + pub fn artifact_for_target(&self, target: &str) -> anyhow::Result<&Artifact> { + self.artifacts.get(target).ok_or_else(|| { + anyhow::anyhow!("target {target} not built for wrapper {}", self.version) + }) + } +} + +fn validate_schema(schema: u32) -> anyhow::Result<()> { + if schema > MANIFEST_SCHEMA { + anyhow::bail!( + "manifest schema {schema} is newer than this wrapper; run `baml self-update`" + ); + } + if schema != MANIFEST_SCHEMA { + anyhow::bail!("unsupported manifest schema {schema}"); + } + Ok(()) +} + +fn validate_artifacts(version: &str, artifacts: &BTreeMap) -> anyhow::Result<()> { + let expected: std::collections::BTreeSet<_> = + SUPPORTED_RELEASE_TARGETS.iter().copied().collect(); + let actual: std::collections::BTreeSet<_> = artifacts.keys().map(String::as_str).collect(); + if actual != expected { + anyhow::bail!("manifest for {version} has target set {actual:?}; expected {expected:?}"); + } + for (target, artifact) in artifacts { + validate_artifact(target, artifact)?; + } + Ok(()) +} + +fn validate_artifact(name: &str, artifact: &Artifact) -> anyhow::Result<()> { + if !artifact.url.starts_with("https://") { + anyhow::bail!("artifact {name} URL must use HTTPS"); + } + validate_sha256(&artifact.sha256)?; + Ok(()) +} diff --git a/baml_language/crates/baml_version/Cargo.toml b/baml_language/crates/baml_version/Cargo.toml new file mode 100644 index 00000000000..34e07937a38 --- /dev/null +++ b/baml_language/crates/baml_version/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "baml_version" +version = { workspace = true } +publish = false +authors = { workspace = true } +edition = { workspace = true } +rust-version = { workspace = true } +homepage = { workspace = true } +documentation = { workspace = true } +repository = { workspace = true } +license = { workspace = true } + +[lib] +doctest = false + +[lints] +workspace = true + +[package.metadata.ci] +wasm_support = true diff --git a/baml_language/crates/baml_version/src/lib.rs b/baml_language/crates/baml_version/src/lib.rs new file mode 100644 index 00000000000..9a7ee6bcae2 --- /dev/null +++ b/baml_language/crates/baml_version/src/lib.rs @@ -0,0 +1,6 @@ +//! Stamped BAML language product version. + +pub const CANONICAL_VERSION: &str = "0.11.0"; +pub const PYPI_VERSION: &str = "0.11.0"; +pub const CHANNEL: &str = "canary"; +pub const STABLE_VERSION: &str = "0.11.0"; diff --git a/baml_language/crates/bex_project/Cargo.toml b/baml_language/crates/bex_project/Cargo.toml index d78f686282e..5a5edf59ef1 100644 --- a/baml_language/crates/bex_project/Cargo.toml +++ b/baml_language/crates/bex_project/Cargo.toml @@ -28,6 +28,7 @@ baml_fmt = { workspace = true } baml_lsp2_actions = { workspace = true } baml_project = { workspace = true } baml_type = { workspace = true } +baml_version = { workspace = true } baml_workspace = { workspace = true } bex_engine = { workspace = true } bex_events = { workspace = true } diff --git a/baml_language/crates/bex_project/src/bex_lsp/mod.rs b/baml_language/crates/bex_project/src/bex_lsp/mod.rs index 87c4c43f01b..5165eee73cf 100644 --- a/baml_language/crates/bex_project/src/bex_lsp/mod.rs +++ b/baml_language/crates/bex_project/src/bex_lsp/mod.rs @@ -5,6 +5,7 @@ pub(crate) mod notification; mod request; mod multi_project; +mod protocol; use async_trait::async_trait; diff --git a/baml_language/crates/bex_project/src/bex_lsp/multi_project/request.rs b/baml_language/crates/bex_project/src/bex_lsp/multi_project/request.rs index 614b7e9dc88..1950a975155 100644 --- a/baml_language/crates/bex_project/src/bex_lsp/multi_project/request.rs +++ b/baml_language/crates/bex_project/src/bex_lsp/multi_project/request.rs @@ -8,7 +8,7 @@ use lsp_types::{ }; use super::{BexMulitProject, LspError, WithDiagnostics, commands, wasm_helpers}; -use crate::bex_lsp::{multi_project::commands::BexLspCommand, request::BexLspRequest}; +use crate::bex_lsp::{multi_project::commands::BexLspCommand, protocol, request::BexLspRequest}; /// Server capabilities advertised during the LSP `initialize` handshake. /// @@ -77,6 +77,16 @@ pub(super) fn server_capabilities() -> ServerCapabilities { resolve_provider: Some(false), }), )), + experimental: Some(serde_json::json!({ + "baml": { + "toolchainVersion": baml_version::CANONICAL_VERSION, + "lspProtocol": protocol::BAML_LSP_PROTOCOL_VERSION, + "minSupportedClientLspProtocol": protocol::MIN_SUPPORTED_VSCODE_LSP_PROTOCOL, + "playgroundProtocol": protocol::BAML_PLAYGROUND_PROTOCOL_VERSION, + "minSupportedClientPlaygroundProtocol": protocol::MIN_SUPPORTED_PLAYGROUND_PROTOCOL, + "capabilities": protocol::CAPABILITIES, + } + })), ..Default::default() } } @@ -139,7 +149,7 @@ impl BexLspRequest for BexMulitProject { capabilities: server_capabilities(), server_info: Some(lsp_types::ServerInfo { name: "baml-lsp".to_string(), - version: Some(env!("CARGO_PKG_VERSION").to_string()), + version: Some(baml_version::CANONICAL_VERSION.to_string()), }), }) } diff --git a/baml_language/crates/bex_project/src/bex_lsp/protocol.rs b/baml_language/crates/bex_project/src/bex_lsp/protocol.rs new file mode 100644 index 00000000000..f9c1779e467 --- /dev/null +++ b/baml_language/crates/bex_project/src/bex_lsp/protocol.rs @@ -0,0 +1,10 @@ +pub(super) const BAML_LSP_PROTOCOL_VERSION: u32 = 1; +pub(super) const BAML_PLAYGROUND_PROTOCOL_VERSION: u32 = 1; +pub(super) const MIN_SUPPORTED_VSCODE_LSP_PROTOCOL: u32 = 1; +pub(super) const MIN_SUPPORTED_PLAYGROUND_PROTOCOL: u32 = 1; + +pub(super) const CAPABILITIES: &[&str] = &[ + "openPlayground.v1", + "listProjects.v1", + "playgroundWebSocket.v1", +]; diff --git a/baml_language/crates/bridge_cffi/Cargo.toml b/baml_language/crates/bridge_cffi/Cargo.toml index 62185ba925d..efae11f7b7c 100644 --- a/baml_language/crates/bridge_cffi/Cargo.toml +++ b/baml_language/crates/bridge_cffi/Cargo.toml @@ -21,6 +21,7 @@ rustls = [ "sys_native/rustls" ] bundle-http = [ "sys_native/bundle-http" ] [dependencies] +baml_version = { workspace = true } bex_events = { workspace = true } bex_events_native = { workspace = true } bex_project = { workspace = true } diff --git a/baml_language/crates/bridge_cffi/src/ffi/runtime.rs b/baml_language/crates/bridge_cffi/src/ffi/runtime.rs index 4831c8bcc56..a927eea7b39 100644 --- a/baml_language/crates/bridge_cffi/src/ffi/runtime.rs +++ b/baml_language/crates/bridge_cffi/src/ffi/runtime.rs @@ -4,13 +4,11 @@ use std::{collections::HashMap, ffi::CStr}; use crate::{Buffer, initialize_runtime, panic::ffi_safe_ptr}; -const VERSION: &str = env!("CARGO_PKG_VERSION"); - /// Returns the BAML version as a Buffer containing raw UTF-8 bytes. /// Caller must free with free_buffer(). #[unsafe(no_mangle)] pub extern "C" fn version() -> Buffer { - Buffer::from(VERSION.as_bytes().to_vec()) + Buffer::from(baml_version::CANONICAL_VERSION.as_bytes().to_vec()) } /// Create/initialize the BAML runtime (global BexEngine). diff --git a/baml_language/crates/bridge_wasm/Cargo.toml b/baml_language/crates/bridge_wasm/Cargo.toml index ead1fa92836..107315ec625 100644 --- a/baml_language/crates/bridge_wasm/Cargo.toml +++ b/baml_language/crates/bridge_wasm/Cargo.toml @@ -12,6 +12,7 @@ crate-type = [ "cdylib", "rlib" ] [dependencies] baml_type = { workspace = true } +baml_version = { workspace = true } bex_events = { workspace = true } bex_heap = { workspace = true } bex_project = { workspace = true } diff --git a/baml_language/crates/bridge_wasm/src/lib.rs b/baml_language/crates/bridge_wasm/src/lib.rs index ba78be5f5fe..aab1e5793f8 100644 --- a/baml_language/crates/bridge_wasm/src/lib.rs +++ b/baml_language/crates/bridge_wasm/src/lib.rs @@ -91,7 +91,7 @@ pub fn start() { /// Get the version of the `bridge_wasm` crate. #[wasm_bindgen] pub fn version() -> String { - env!("CARGO_PKG_VERSION").to_string() + baml_version::CANONICAL_VERSION.to_string() } /// Returns the build timestamp (unix seconds) for hot-reload / build-identity checks. diff --git a/baml_language/release.toml b/baml_language/release.toml index 959e1a403c5..5a1ceb90f21 100644 --- a/baml_language/release.toml +++ b/baml_language/release.toml @@ -1,3 +1,2 @@ -[channel] -name = "alpha" -base_version = "0.11.0" +[release] +canary_version = "0.11.0" diff --git a/baml_language/sdks/nodejs/bridge_nodejs/Cargo.toml b/baml_language/sdks/nodejs/bridge_nodejs/Cargo.toml index 33f1be23f0e..92c6d9b00de 100644 --- a/baml_language/sdks/nodejs/bridge_nodejs/Cargo.toml +++ b/baml_language/sdks/nodejs/bridge_nodejs/Cargo.toml @@ -22,6 +22,7 @@ aws-crypto = [ "bex_project/aws-crypto", "bridge_cffi/aws-crypto" ] ring-crypto = [ "bex_project/ring-crypto", "bridge_cffi/ring-crypto" ] [dependencies] +baml_version = { workspace = true } bex_events = { workspace = true } bex_project = { workspace = true } bridge_cffi = { workspace = true } diff --git a/baml_language/sdks/nodejs/bridge_nodejs/package.json b/baml_language/sdks/nodejs/bridge_nodejs/package.json index 74b7b27b623..de06ace9187 100644 --- a/baml_language/sdks/nodejs/bridge_nodejs/package.json +++ b/baml_language/sdks/nodejs/bridge_nodejs/package.json @@ -1,6 +1,6 @@ { "name": "@boundaryml/baml-node", - "version": "0.0.0-beta", + "version": "0.11.0", "main": "index.js", "types": "index.d.ts", "napi": { diff --git a/baml_language/sdks/nodejs/bridge_nodejs/src/lib.rs b/baml_language/sdks/nodejs/bridge_nodejs/src/lib.rs index 159d3133ff6..05b42e46ee4 100644 --- a/baml_language/sdks/nodejs/bridge_nodejs/src/lib.rs +++ b/baml_language/sdks/nodejs/bridge_nodejs/src/lib.rs @@ -28,7 +28,7 @@ fn init() { #[napi] pub fn get_version() -> &'static str { - env!("CARGO_PKG_VERSION") + baml_version::CANONICAL_VERSION } /// Flush all buffered trace events to the JSONL file (if BAML_TRACE_FILE is set). diff --git a/baml_language/sdks/python/pyproject.toml b/baml_language/sdks/python/pyproject.toml index 89bf5912119..3bc538fa7ae 100644 --- a/baml_language/sdks/python/pyproject.toml +++ b/baml_language/sdks/python/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "baml_core" -version = "0.1.3" +version = "0.11.0" description = "BAML python bindings (new system, bex_engine)" authors = [{ "name" = "Boundary", "email" = "contact@boundaryml.com" }] license = "Apache-2.0" diff --git a/baml_language/sdks/python/rust/bridge_python/Cargo.toml b/baml_language/sdks/python/rust/bridge_python/Cargo.toml index 1fbd39bb831..153c9e35a25 100644 --- a/baml_language/sdks/python/rust/bridge_python/Cargo.toml +++ b/baml_language/sdks/python/rust/bridge_python/Cargo.toml @@ -23,6 +23,7 @@ ring-crypto = [ "bundle-http", "bex_project/ring-crypto", "bridge_cffi/ring-cryp extension-module = [ "pyo3/extension-module" ] [dependencies] +baml_version = { workspace = true } bex_events = { workspace = true } bex_project = { workspace = true } bridge_cffi = { workspace = true } diff --git a/baml_language/sdks/python/rust/bridge_python/src/lib.rs b/baml_language/sdks/python/rust/bridge_python/src/lib.rs index c25dc8bd705..03420161cdf 100644 --- a/baml_language/sdks/python/rust/bridge_python/src/lib.rs +++ b/baml_language/sdks/python/rust/bridge_python/src/lib.rs @@ -22,7 +22,7 @@ use pyo3_stub_gen::{define_stub_info_gatherer, derive::gen_stub_pyfunction}; #[gen_stub_pyfunction] #[pyfunction] fn get_version() -> &'static str { - env!("CARGO_PKG_VERSION") + baml_version::CANONICAL_VERSION } /// Flush all buffered trace events to the JSONL file (if BAML_TRACE_FILE is set). diff --git a/baml_language/sdks/python/src/baml_core/__init__.py b/baml_language/sdks/python/src/baml_core/__init__.py index 5b977669235..0a3f44059bf 100644 --- a/baml_language/sdks/python/src/baml_core/__init__.py +++ b/baml_language/sdks/python/src/baml_core/__init__.py @@ -36,6 +36,8 @@ # Flush buffered trace events on process exit so nothing is lost. atexit.register(flush_events) +__version__ = "0.11.0" + # --------------------------------------------------------------------------- # FunctionLog / Collector wrappers (decode protobuf result → Python value) diff --git a/baml_language/sdks/python/uv.lock b/baml_language/sdks/python/uv.lock index fecb109b8e6..56d93ef4078 100644 --- a/baml_language/sdks/python/uv.lock +++ b/baml_language/sdks/python/uv.lock @@ -22,7 +22,7 @@ wheels = [ [[package]] name = "baml-core" -version = "0.1.3" +version = "0.11.0" source = { editable = "." } dependencies = [ { name = "protobuf" }, diff --git a/baml_language/stow.toml b/baml_language/stow.toml index 385fa00bbd6..315f2e1b249 100644 --- a/baml_language/stow.toml +++ b/baml_language/stow.toml @@ -71,6 +71,12 @@ allowed_crates = [ # the same reason — top-level error surfacing. "baml_exec", "baml_pack_host", + # `baml` is the wrapper binary users install. `baml_release` is + # shared release-boundary fetch/install code used by the wrapper + # and pack; it exposes typed fetch errors but uses anyhow for + # validation helpers and caller-facing context. + "baml", + "baml_release", ] regular_deps_only = true reason = "Use thiserror for proper error types in library crates." @@ -154,4 +160,4 @@ name = "surface" # `baml_pack_host` is the runtime binary embedded in packed # executables. Listed here so they're exempt from the baml-namespace # rules forbidding bex_* deps. -name_exceptions = {"baml_cli" = "baml_cli", "baml_exec" = "baml_exec", "baml_pack_host" = "baml_pack_host" } +name_exceptions = {"baml" = "baml", "baml_cli" = "baml_cli", "baml_exec" = "baml_exec", "baml_pack_host" = "baml_pack_host" } diff --git a/packaging/aur/baml-bin/PKGBUILD b/packaging/aur/baml-bin/PKGBUILD index 1a95ddcd87d..6e1a06db0e4 100644 --- a/packaging/aur/baml-bin/PKGBUILD +++ b/packaging/aur/baml-bin/PKGBUILD @@ -1,19 +1,18 @@ pkgname=baml-bin pkgver=__VERSION__ pkgrel=1 -pkgdesc="BAML - the language for agents (prebuilt binary)" +pkgdesc="BAML wrapper - toolchain manager and launcher (prebuilt binary)" arch=('x86_64' 'aarch64') url="https://github.com/BoundaryML/baml" license=('Apache-2.0') provides=('baml') conflicts=('baml') -source_x86_64=("https://github.com/BoundaryML/baml/releases/download/baml-language-__VERSION_ORIG__/baml-language-__VERSION_ORIG__-x86_64-unknown-linux-gnu.tar.gz") -source_aarch64=("https://github.com/BoundaryML/baml/releases/download/baml-language-__VERSION_ORIG__/baml-language-__VERSION_ORIG__-aarch64-unknown-linux-gnu.tar.gz") +source_x86_64=("https://github.com/BoundaryML/baml/releases/download/baml-wrapper-__VERSION_ORIG__/baml-wrapper-__VERSION_ORIG__-x86_64-unknown-linux-gnu.tar.gz") +source_aarch64=("https://github.com/BoundaryML/baml/releases/download/baml-wrapper-__VERSION_ORIG__/baml-wrapper-__VERSION_ORIG__-aarch64-unknown-linux-gnu.tar.gz") sha256sums_x86_64=('__SHA256_X86_64__') sha256sums_aarch64=('__SHA256_AARCH64__') package() { - install -Dm755 baml-cli "$pkgdir/usr/bin/baml-cli" - ln -s baml-cli "$pkgdir/usr/bin/baml" + install -Dm755 bin/baml "$pkgdir/usr/bin/baml" } diff --git a/packaging/aur/baml/PKGBUILD b/packaging/aur/baml/PKGBUILD index 18652ab18d7..4dd7cbffdd7 100644 --- a/packaging/aur/baml/PKGBUILD +++ b/packaging/aur/baml/PKGBUILD @@ -1,7 +1,7 @@ pkgname=baml pkgver=__VERSION__ pkgrel=1 -pkgdesc="BAML - the language for agents (built from source)" +pkgdesc="BAML wrapper - toolchain manager and launcher (built from source)" arch=('x86_64' 'aarch64') url="https://github.com/BoundaryML/baml" license=('Apache-2.0') @@ -9,24 +9,23 @@ makedepends=('rust' 'cargo' 'cmake' 'gcc') provides=('baml') conflicts=('baml-bin') -source=("$pkgname-$pkgver.tar.gz::https://github.com/BoundaryML/baml/archive/refs/tags/baml-language-__VERSION_ORIG__.tar.gz") +source=("$pkgname-$pkgver.tar.gz::https://github.com/BoundaryML/baml/archive/refs/tags/baml-wrapper-__VERSION_ORIG__.tar.gz") sha256sums=('__SHA256__') prepare() { - cd "baml-baml-language-__VERSION_ORIG__/baml_language" + cd "baml-baml-wrapper-__VERSION_ORIG__/baml_language" export RUSTUP_TOOLCHAIN=stable cargo fetch --locked } build() { - cd "baml-baml-language-__VERSION_ORIG__/baml_language" + cd "baml-baml-wrapper-__VERSION_ORIG__/baml_language" export RUSTUP_TOOLCHAIN=stable export CARGO_TARGET_DIR=target - cargo build --release --frozen --bin baml-cli + cargo build --release --frozen --bin baml } package() { - cd "baml-baml-language-__VERSION_ORIG__/baml_language" - install -Dm755 "target/release/baml-cli" "$pkgdir/usr/bin/baml-cli" - ln -s baml-cli "$pkgdir/usr/bin/baml" + cd "baml-baml-wrapper-__VERSION_ORIG__/baml_language" + install -Dm755 "target/release/baml" "$pkgdir/usr/bin/baml" } diff --git a/scripts/baml-language-version b/scripts/baml-language-version index d4df1a5151c..2c5010d9135 100755 --- a/scripts/baml-language-version +++ b/scripts/baml-language-version @@ -1,9 +1,11 @@ #!/usr/bin/env python3 -"""Manage the baml_language alpha release base version.""" +"""Manage the BAML language release version surfaces.""" from __future__ import annotations import argparse +import datetime as dt +import json import os import re import subprocess @@ -14,7 +16,18 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] RELEASE_TOML = ROOT / "baml_language" / "release.toml" +BAML_VERSION_RS = ROOT / "baml_language" / "crates" / "baml_version" / "src" / "lib.rs" +PYPROJECT = ROOT / "baml_language" / "sdks" / "python" / "pyproject.toml" +PY_INIT = ROOT / "baml_language" / "sdks" / "python" / "src" / "baml_core" / "__init__.py" +VSIX_PACKAGE = ROOT / "typescript2" / "app-vscode-ext" / "package.json" +NODE_PACKAGE = ROOT / "baml_language" / "sdks" / "nodejs" / "bridge_nodejs" / "package.json" +WRAPPER_CARGO_TOML = ROOT / "baml_language" / "crates" / "baml" / "Cargo.toml" + SEMVER_RE = re.compile(r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$") +CANONICAL_RE = re.compile( + r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-nightly\.(\d{8})\.([a-z]))?$" +) +SHA_RE = re.compile(r"^[a-z]$") @dataclass(frozen=True, order=True) @@ -27,7 +40,7 @@ class Version: def parse(cls, value: str) -> "Version": match = SEMVER_RE.fullmatch(value) if not match: - raise ValueError(f"invalid base version {value!r}; expected MAJOR.MINOR.PATCH") + raise ValueError(f"invalid canary version {value!r}; expected MAJOR.MINOR.PATCH") return cls(*(int(part) for part in match.groups())) def __str__(self) -> str: @@ -42,15 +55,31 @@ class Version: def bump_patch(self) -> "Version": return Version(self.major, self.minor, self.patch + 1) + def next_patch(self) -> "Version": + return self.bump_patch() + + +@dataclass(frozen=True) +class ReleasePlan: + schema: int + channel: str + canary_version: str + canonical_version: str + pypi_version: str + git_tag: str + + def to_json(self) -> str: + return json.dumps(self.__dict__, indent=2, sort_keys=True) + "\n" + -def fail(message: str) -> None: +def fail(message: str, code: int = 1) -> None: print(f"error: {message}", file=sys.stderr) - raise SystemExit(1) + raise SystemExit(code) -def parse_release_content(content: str) -> tuple[str, Version]: +def parse_release_content(content: str) -> Version: section: str | None = None - channel: dict[str, str] = {} + release: dict[str, str] = {} for raw_line in content.splitlines(): line = raw_line.split("#", 1)[0].strip() @@ -61,7 +90,7 @@ def parse_release_content(content: str) -> tuple[str, Version]: section = line[1:-1].strip() continue - if section != "channel": + if section != "release": continue key, separator, value = line.partition("=") @@ -71,39 +100,58 @@ def parse_release_content(content: str) -> tuple[str, Version]: key = key.strip() value = value.strip() if len(value) < 2 or not value.startswith('"') or not value.endswith('"'): - fail(f"channel.{key} must be a quoted string") + fail(f"release.{key} must be a quoted string") - channel[key] = value[1:-1] + release[key] = value[1:-1] - name = channel.get("name") - if name is None: - fail("channel.name must be a string") - - base_version = channel.get("base_version") - if base_version is None: - fail("channel.base_version must be a string") + canary_version = release.get("canary_version") + if canary_version is None: + fail("release.canary_version must be a string") try: - version = Version.parse(base_version) + return Version.parse(canary_version) except ValueError as exc: fail(str(exc)) - return name, version + +def parse_previous_release_content(content: str) -> Version: + if "[release]" in content: + return parse_release_content(content) + + section: str | None = None + channel: dict[str, str] = {} + for raw_line in content.splitlines(): + line = raw_line.split("#", 1)[0].strip() + if not line: + continue + if line.startswith("[") and line.endswith("]"): + section = line[1:-1].strip() + continue + if section != "channel": + continue + key, separator, value = line.partition("=") + if separator and value.strip().startswith('"') and value.strip().endswith('"'): + channel[key.strip()] = value.strip()[1:-1] + + base_version = channel.get("base_version") + if base_version is None: + fail("previous baml_language/release.toml is invalid") + try: + return Version.parse(base_version) + except ValueError as exc: + fail(f"previous baml_language/release.toml is invalid: {exc}") -def load_release(path: Path = RELEASE_TOML) -> tuple[str, Version]: +def load_release(path: Path = RELEASE_TOML) -> Version: try: - content = path.read_text() + content = path.read_text(encoding="utf-8") except FileNotFoundError: fail(f"{path.relative_to(ROOT)} does not exist") return parse_release_content(content) def write_release(version: Version) -> None: - RELEASE_TOML.write_text( - f'[channel]\nname = "alpha"\nbase_version = "{version}"\n', - encoding="utf-8", - ) + RELEASE_TOML.write_text(f'[release]\ncanary_version = "{version}"\n', encoding="utf-8") def git_output(*args: str) -> str | None: @@ -138,37 +186,236 @@ def previous_release_toml() -> str | None: return None -def parse_release_toml_content(content: str) -> tuple[str, Version]: +def today_utc() -> str: + override = os.environ.get("BAML_LANGUAGE_VERSION_DATE") + if override: + if not re.fullmatch(r"\d{8}", override): + fail("BAML_LANGUAGE_VERSION_DATE must be YYYYMMDD") + return override + return dt.datetime.now(dt.UTC).strftime("%Y%m%d") + + +def command_output(command: list[str]) -> str | None: try: - return parse_release_content(content) - except Exception as exc: - fail(f"previous baml_language/release.toml is invalid: {exc}") + return subprocess.check_output( + command, + cwd=ROOT, + stderr=subprocess.DEVNULL, + text=True, + timeout=30, + ) + except (subprocess.CalledProcessError, FileNotFoundError, subprocess.TimeoutExpired): + return None -def check_release() -> None: - name, current = load_release() - if name != "alpha": - fail('channel.name must be "alpha"') +def release_tags() -> list[str]: + output = command_output( + [ + "gh", + "release", + "list", + "--repo", + "BoundaryML/baml", + "--limit", + "200", + "--json", + "tagName", + "--jq", + ".[].tagName", + ] + ) + if output: + return [line.strip() for line in output.splitlines() if line.strip()] + + output = command_output(["git", "ls-remote", "--tags", "https://github.com/BoundaryML/baml.git"]) + if output: + tags: list[str] = [] + for line in output.splitlines(): + ref = line.rsplit("/", 1)[-1].removesuffix("^{}") + if ref: + tags.append(ref) + return tags + + return [] + + +def next_nightly_letter(base: Version, date: str, tags: list[str] | None = None) -> str: + prefix = f"baml-language-{base}-nightly.{date}." + used = sorted( + tag.removeprefix(prefix) + for tag in (tags if tags is not None else release_tags()) + if tag.startswith(prefix) and SHA_RE.fullmatch(tag.removeprefix(prefix)) + ) + if not used: + return "a" + + max_index = max(ord(letter) - ord("a") for letter in used) + if max_index >= 25: + fail( + f"26 nightly builds today for base {base}; wait for the next UTC day or cut a new canary release by bumping baml_language/release.toml [release].canary_version.", + code=2, + ) + return chr(ord("a") + max_index + 1) + + +def compute_version(channel: str) -> str: + canary = load_release() + if channel == "canary": + return str(canary) + if channel == "nightly": + base = canary.next_patch() + date = today_utc() + letter = next_nightly_letter(base, date) + return f"{base}-nightly.{date}.{letter}" + fail(f"unsupported channel {channel!r}") + + +def to_pypi_version(canonical: str) -> str: + match = CANONICAL_RE.fullmatch(canonical) + if not match: + fail(f"invalid canonical version {canonical!r}") + major, minor, patch, date, letter = match.groups() + if date is None: + return f"{major}.{minor}.{patch}" + index = ord(letter) - ord("a") + return f"{major}.{minor}.{patch}.dev{date}{index:02d}" + + +def make_plan(channel: str) -> ReleasePlan: + canonical = compute_version(channel) + return ReleasePlan( + schema=1, + channel=channel, + canary_version=str(load_release()), + canonical_version=canonical, + pypi_version=to_pypi_version(canonical), + git_tag=f"baml-language-{canonical}", + ) + + +def load_plan(path: Path) -> ReleasePlan: + try: + data = json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError: + fail(f"{path} does not exist") + except json.JSONDecodeError as exc: + fail(f"{path} is not valid JSON: {exc}") + + try: + return ReleasePlan( + schema=int(data["schema"]), + channel=str(data["channel"]), + canary_version=str(data["canary_version"]), + canonical_version=str(data["canonical_version"]), + pypi_version=str(data["pypi_version"]), + git_tag=str(data["git_tag"]), + ) + except KeyError as exc: + fail(f"{path} is missing {exc.args[0]!r}") + + +def replace_regex(path: Path, pattern: str, replacement: str) -> None: + text = path.read_text(encoding="utf-8") + new_text, count = re.subn(pattern, replacement, text, count=1, flags=re.MULTILINE) + if count != 1: + fail(f"could not update expected version surface in {path.relative_to(ROOT)}") + path.write_text(new_text, encoding="utf-8") + + +def stamp(plan: ReleasePlan) -> None: + BAML_VERSION_RS.parent.mkdir(parents=True, exist_ok=True) + BAML_VERSION_RS.write_text( + "\n".join( + [ + "//! Stamped BAML language product version.", + "", + f'pub const CANONICAL_VERSION: &str = "{plan.canonical_version}";', + f'pub const PYPI_VERSION: &str = "{plan.pypi_version}";', + f'pub const CHANNEL: &str = "{plan.channel}";', + f'pub const STABLE_VERSION: &str = "{plan.canary_version}";', + "", + ] + ), + encoding="utf-8", + ) + replace_regex(PYPROJECT, r'^version = ".*"$', f'version = "{plan.pypi_version}"') + replace_regex(VSIX_PACKAGE, r'^ "version": ".*",$', f' "version": "{plan.canonical_version}",') + if NODE_PACKAGE.exists(): + replace_regex(NODE_PACKAGE, r'^ "version": ".*",$', f' "version": "{plan.canonical_version}",') + + text = PY_INIT.read_text(encoding="utf-8") + version_line = f'__version__ = "{plan.canonical_version}"\n' + if re.search(r'^__version__ = ".*"$', text, flags=re.MULTILINE): + text = re.sub(r'^__version__ = ".*"$', version_line.rstrip(), text, count=1, flags=re.MULTILINE) + else: + marker = "atexit.register(flush_events)\n" + text = text.replace(marker, marker + "\n" + version_line, 1) + PY_INIT.write_text(text, encoding="utf-8") - previous = previous_release_toml() - if previous is None: - print(f"ok: base_version {current}") - return - previous_name, previous_version = parse_release_toml_content(previous) - if previous_name != "alpha": - fail('previous channel.name was not "alpha"') +def sync_canary() -> None: + stamp(make_plan("canary")) - if current < previous_version: - fail(f"base_version must not decrease: {current} < {previous_version}") - print(f"ok: base_version {current}") +def check_release() -> None: + current = load_release() + previous = previous_release_toml() + if previous is not None: + previous_version = parse_previous_release_content(previous) + if current < previous_version: + fail(f"canary_version must not decrease: {current} < {previous_version}") + + if current.next_patch() <= current: + fail(f"nightly base {current.next_patch()} must be greater than canary {current}") + + expected = make_plan("canary") + mismatches: list[str] = [] + checks = { + "baml_version::CANONICAL_VERSION": expected.canonical_version in BAML_VERSION_RS.read_text(encoding="utf-8"), + "baml_version::PYPI_VERSION": expected.pypi_version in BAML_VERSION_RS.read_text(encoding="utf-8"), + "python pyproject version": f'version = "{expected.pypi_version}"' in PYPROJECT.read_text(encoding="utf-8"), + "python __version__": f'__version__ = "{expected.canonical_version}"' in PY_INIT.read_text(encoding="utf-8"), + "vsix package version": f'"version": "{expected.canonical_version}"' in VSIX_PACKAGE.read_text(encoding="utf-8"), + } + if NODE_PACKAGE.exists(): + checks["node package version"] = f'"version": "{expected.canonical_version}"' in NODE_PACKAGE.read_text(encoding="utf-8") + + for name, ok in checks.items(): + if not ok: + mismatches.append(name) + + forbidden = command_output( + [ + "rg", + "-n", + "BAML_RELEASE_VERSION|BAML_WRAPPER_VERSION|env!\\(\"CARGO_PKG_VERSION\"\\)", + "baml_language/crates", + "baml_language/sdks", + "scripts", + ".github/workflows", + ] + ) + if forbidden: + allowed = [] + for line in forbidden.splitlines(): + if "tools_stow" in line or "crates/baml/src" in line: + continue + if "scripts/baml-language-version:" in line: + continue + allowed.append(line) + if allowed: + mismatches.append("deprecated version plumbing:\n" + "\n".join(allowed)) + + wrapper_toml = WRAPPER_CARGO_TOML.read_text(encoding="utf-8") + if "version.workspace" in wrapper_toml: + mismatches.append("wrapper crate must use a literal [package].version") + if not re.search(r'^version = "(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)"$', wrapper_toml, flags=re.MULTILINE): + mismatches.append("wrapper crate version must be literal MAJOR.MINOR.PATCH SemVer") + if mismatches: + fail("version check failed:\n" + "\n".join(f"- {item}" for item in mismatches)) -def alpha_number(value: str) -> int: - if not re.fullmatch(r"(0|[1-9]\d*)", value): - raise argparse.ArgumentTypeError("alpha number must be a non-negative integer") - return int(value) + print(f"ok: canary_version {current}") def main() -> None: @@ -176,6 +423,7 @@ def main() -> None: subparsers = parser.add_subparsers(dest="command", required=True) subparsers.add_parser("show") + set_parser = subparsers.add_parser("set") set_parser.add_argument("version") @@ -185,23 +433,33 @@ def main() -> None: bump_group.add_argument("--minor", action="store_true") bump_group.add_argument("--patch", action="store_true") - subparsers.add_parser("check") compute_parser = subparsers.add_parser("compute") - compute_parser.add_argument("--alpha-number", required=True, type=alpha_number) + compute_parser.add_argument("--channel", choices=["canary", "nightly"], required=True) + compute_parser.add_argument("--pypi", action="store_true") + + plan_parser = subparsers.add_parser("plan") + plan_parser.add_argument("--channel", choices=["canary", "nightly"], required=True) + plan_parser.add_argument("--out", type=Path, required=True) + + stamp_parser = subparsers.add_parser("stamp") + stamp_parser.add_argument("--plan", type=Path, required=True) + + subparsers.add_parser("sync") + subparsers.add_parser("check") args = parser.parse_args() if args.command == "show": - _, version = load_release() - print(version) + print(load_release()) elif args.command == "set": try: version = Version.parse(args.version) except ValueError as exc: fail(str(exc)) write_release(version) + sync_canary() elif args.command == "bump": - _, version = load_release() + version = load_release() if args.major: version = version.bump_major() elif args.minor: @@ -209,11 +467,18 @@ def main() -> None: else: version = version.bump_patch() write_release(version) + sync_canary() + elif args.command == "compute": + canonical = compute_version(args.channel) + print(to_pypi_version(canonical) if args.pypi else canonical) + elif args.command == "plan": + args.out.write_text(make_plan(args.channel).to_json(), encoding="utf-8") + elif args.command == "stamp": + stamp(load_plan(args.plan)) + elif args.command == "sync": + sync_canary() elif args.command == "check": check_release() - elif args.command == "compute": - _, version = load_release() - print(f"{version}-alpha.{args.alpha_number}") if __name__ == "__main__": diff --git a/scripts/baml-package-manager-artifacts b/scripts/baml-package-manager-artifacts new file mode 100755 index 00000000000..47ea0031581 --- /dev/null +++ b/scripts/baml-package-manager-artifacts @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""Generate wrapper-only Homebrew and AUR release files.""" + +from __future__ import annotations + +import argparse +import hashlib +from pathlib import Path + + +TARGETS = { + "mac_arm": "aarch64-apple-darwin", + "mac_intel": "x86_64-apple-darwin", + "linux_arm": "aarch64-unknown-linux-gnu", + "linux_intel": "x86_64-unknown-linux-gnu", +} + + +def sha256(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(1024 * 1024), b""): + h.update(chunk) + return h.hexdigest() + + +def wrapper_archive(wrapper_dir: Path, version: str, target: str) -> Path: + ext = "zip" if target.endswith("windows-msvc") else "tar.gz" + path = wrapper_dir / f"baml-wrapper-{version}-{target}.{ext}" + if not path.exists(): + raise SystemExit(f"missing wrapper archive: {path}") + return path + + +def render_formula(version: str, shas: dict[str, str]) -> str: + return f'''class Baml < Formula + desc "BAML wrapper - toolchain manager and launcher" + homepage "https://github.com/BoundaryML/baml" + version "{version}" + license "Apache-2.0" + + on_macos do + on_arm do + url "https://github.com/BoundaryML/baml/releases/download/baml-wrapper-{version}/baml-wrapper-{version}-aarch64-apple-darwin.tar.gz" + sha256 "{shas["mac_arm"]}" + end + + on_intel do + url "https://github.com/BoundaryML/baml/releases/download/baml-wrapper-{version}/baml-wrapper-{version}-x86_64-apple-darwin.tar.gz" + sha256 "{shas["mac_intel"]}" + end + end + + on_linux do + on_arm do + url "https://github.com/BoundaryML/baml/releases/download/baml-wrapper-{version}/baml-wrapper-{version}-aarch64-unknown-linux-gnu.tar.gz" + sha256 "{shas["linux_arm"]}" + end + + on_intel do + url "https://github.com/BoundaryML/baml/releases/download/baml-wrapper-{version}/baml-wrapper-{version}-x86_64-unknown-linux-gnu.tar.gz" + sha256 "{shas["linux_intel"]}" + end + end + + def install + bin.install "bin/baml" + end + + def caveats + <<~EOS + BAML wrapper installed. + + To install and select the current canary language toolchain: + baml toolchain use canary + + To use nightly: + baml toolchain use nightly + + IDE extension setup is explicit: + baml ide install --cursor + EOS + end +end +''' + + +def aur_srcinfo(pkgbuild: str, pkgname: str, version: str, arch_lines: list[str]) -> str: + lines = [ + f"pkgbase = {pkgname}", + f"\tpkgdesc = BAML wrapper - toolchain manager and launcher", + f"\tpkgver = {version}", + "\tpkgrel = 1", + "\turl = https://github.com/BoundaryML/baml", + "\tarch = x86_64", + "\tarch = aarch64", + "\tlicense = Apache-2.0", + ] + lines.extend(arch_lines) + lines.extend( + [ + "\tprovides = baml", + f"\tconflicts = {'baml-bin' if pkgname == 'baml' else 'baml'}", + "", + f"pkgname = {pkgname}", + "", + ] + ) + return "\n".join(lines) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--wrapper-dir", required=True, type=Path) + parser.add_argument("--version", required=True) + parser.add_argument("--out", required=True, type=Path) + parser.add_argument("--source-sha256", default="SKIP") + args = parser.parse_args() + + args.out.mkdir(parents=True, exist_ok=True) + shas = { + name: sha256(wrapper_archive(args.wrapper_dir, args.version, target)) + for name, target in TARGETS.items() + } + + formula_dir = args.out / "homebrew" / "Formula" + formula_dir.mkdir(parents=True, exist_ok=True) + (formula_dir / "baml.rb").write_text(render_formula(args.version, shas)) + + pkgver = args.version.replace("-", "_") + aur_source = (Path("packaging/aur/baml/PKGBUILD").read_text() + .replace("__VERSION__", pkgver) + .replace("__VERSION_ORIG__", args.version) + .replace("__SHA256__", args.source_sha256)) + aur_bin = (Path("packaging/aur/baml-bin/PKGBUILD").read_text() + .replace("__VERSION__", pkgver) + .replace("__VERSION_ORIG__", args.version) + .replace("__SHA256_X86_64__", shas["linux_intel"]) + .replace("__SHA256_AARCH64__", shas["linux_arm"])) + + aur_dir = args.out / "aur" + for pkgname, text in (("baml", aur_source), ("baml-bin", aur_bin)): + pkg_dir = aur_dir / pkgname + pkg_dir.mkdir(parents=True, exist_ok=True) + (pkg_dir / "PKGBUILD").write_text(text) + + (aur_dir / "baml-bin" / ".SRCINFO").write_text( + aur_srcinfo( + aur_bin, + "baml-bin", + pkgver, + [ + f"\tsource_x86_64 = https://github.com/BoundaryML/baml/releases/download/baml-wrapper-{args.version}/baml-wrapper-{args.version}-x86_64-unknown-linux-gnu.tar.gz", + f"\tsha256sums_x86_64 = {shas['linux_intel']}", + f"\tsource_aarch64 = https://github.com/BoundaryML/baml/releases/download/baml-wrapper-{args.version}/baml-wrapper-{args.version}-aarch64-unknown-linux-gnu.tar.gz", + f"\tsha256sums_aarch64 = {shas['linux_arm']}", + ], + ) + ) + (aur_dir / "baml" / ".SRCINFO").write_text( + aur_srcinfo( + aur_source, + "baml", + pkgver, + [ + "\tmakedepends = rust", + "\tmakedepends = cargo", + "\tmakedepends = cmake", + "\tmakedepends = gcc", + f"\tsource = baml-{pkgver}.tar.gz::https://github.com/BoundaryML/baml/archive/refs/tags/baml-wrapper-{args.version}.tar.gz", + f"\tsha256sums = {args.source_sha256}", + ], + ) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/baml-release-manifests b/scripts/baml-release-manifests new file mode 100755 index 00000000000..b1be2935696 --- /dev/null +++ b/scripts/baml-release-manifests @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Generate BAML toolchain and wrapper JSON manifests from release artifacts.""" + +from __future__ import annotations + +import argparse +import datetime as dt +import hashlib +import json +from pathlib import Path + + +TARGETS = [ + "aarch64-apple-darwin", + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "aarch64-unknown-linux-gnu", + "x86_64-unknown-linux-musl", + "aarch64-unknown-linux-musl", + "x86_64-pc-windows-msvc", + "aarch64-pc-windows-msvc", +] + + +def sha256(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(1024 * 1024), b""): + h.update(chunk) + return h.hexdigest() + + +def artifact_url(tag: str, path: Path) -> str: + return f"https://github.com/BoundaryML/baml/releases/download/{tag}/{path.name}" + + +def collect_artifacts(root: Path, product: str, version: str) -> dict[str, dict[str, str]]: + artifacts = {} + prefix = f"{product}-{version}-" + for path in sorted(root.iterdir()): + if path.suffix == ".sha256" or not path.name.startswith(prefix): + continue + name = path.name.removeprefix(prefix) + if name.endswith(".tar.gz"): + target = name[:-7] + elif name.endswith(".zip"): + target = name[:-4] + else: + continue + artifacts[target] = { + "url": artifact_url(f"{product}-{version}", path), + "sha256": sha256(path), + } + missing = sorted(set(TARGETS) - set(artifacts)) + extra = sorted(set(artifacts) - set(TARGETS)) + if missing or extra: + raise SystemExit(f"{product} artifact target mismatch missing={missing} extra={extra}") + return artifacts + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--toolchain-dir", required=True, type=Path) + parser.add_argument("--wrapper-dir", required=True, type=Path) + parser.add_argument("--vsix-dir", required=True, type=Path) + parser.add_argument("--out", required=True, type=Path) + parser.add_argument("--channel", required=True, choices=["canary", "nightly"]) + parser.add_argument("--version", required=True) + parser.add_argument("--pypi-version", required=True) + parser.add_argument("--wrapper-version", required=True) + parser.add_argument("--wrapper-changed", action="store_true") + args = parser.parse_args() + + args.out.mkdir(parents=True, exist_ok=True) + (args.out / "version").mkdir(parents=True, exist_ok=True) + released_at = dt.datetime.now(dt.UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + vsix = next(args.vsix_dir.glob("*.vsix")) + toolchain = { + "schema": 1, + "version": args.version, + "channel": args.channel, + "released_at": released_at, + "artifacts": collect_artifacts(args.toolchain_dir, "baml-language", args.version), + "vsix": { + "url": artifact_url(f"baml-language-{args.version}", vsix), + "sha256": sha256(vsix), + }, + "baml_core_pypi": { + "version": args.pypi_version, + }, + } + text = json.dumps(toolchain, indent=2, sort_keys=True) + "\n" + (args.out / "version" / f"{args.version}.json").write_text(text) + (args.out / f"{args.channel}.json").write_text(text) + + if args.wrapper_changed: + wrapper = { + "schema": 1, + "version": args.wrapper_version, + "released_at": released_at, + "artifacts": collect_artifacts(args.wrapper_dir, "baml-wrapper", args.wrapper_version), + } + (args.out / "wrapper.json").write_text(json.dumps(wrapper, indent=2, sort_keys=True) + "\n") + + +if __name__ == "__main__": + main() diff --git a/scripts/baml-wrapper-version b/scripts/baml-wrapper-version new file mode 100755 index 00000000000..71206709c41 --- /dev/null +++ b/scripts/baml-wrapper-version @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""Manage the BAML wrapper package version.""" + +from __future__ import annotations + +import argparse +import re +import sys +from dataclasses import dataclass +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +WRAPPER_CARGO_TOML = ROOT / "baml_language" / "crates" / "baml" / "Cargo.toml" +SEMVER_RE = re.compile(r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$") + + +@dataclass(frozen=True) +class Version: + major: int + minor: int + patch: int + + @classmethod + def parse(cls, value: str) -> "Version": + match = SEMVER_RE.fullmatch(value) + if not match: + raise ValueError(f"invalid wrapper version {value!r}; expected MAJOR.MINOR.PATCH") + return cls(*(int(part) for part in match.groups())) + + def __str__(self) -> str: + return f"{self.major}.{self.minor}.{self.patch}" + + def bump_major(self) -> "Version": + return Version(self.major + 1, 0, 0) + + def bump_minor(self) -> "Version": + return Version(self.major, self.minor + 1, 0) + + def bump_patch(self) -> "Version": + return Version(self.major, self.minor, self.patch + 1) + + +def fail(message: str) -> None: + print(f"error: {message}", file=sys.stderr) + raise SystemExit(1) + + +def read_version() -> Version: + text = WRAPPER_CARGO_TOML.read_text(encoding="utf-8") + match = re.search(r'^version = "([^"]+)"$', text, flags=re.MULTILINE) + if not match: + fail("baml wrapper Cargo.toml must contain literal [package].version") + try: + return Version.parse(match.group(1)) + except ValueError as exc: + fail(str(exc)) + + +def write_version(version: Version) -> None: + text = WRAPPER_CARGO_TOML.read_text(encoding="utf-8") + new_text, count = re.subn( + r'^version = "[^"]+"$', + f'version = "{version}"', + text, + count=1, + flags=re.MULTILINE, + ) + if count != 1: + fail("failed to update wrapper version") + WRAPPER_CARGO_TOML.write_text(new_text, encoding="utf-8") + + +def check() -> None: + text = WRAPPER_CARGO_TOML.read_text(encoding="utf-8") + if "version.workspace" in text: + fail("wrapper crate must not inherit version.workspace") + version = read_version() + print(f"ok: wrapper_version {version}") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + subparsers.add_parser("show") + subparsers.add_parser("check") + + bump = subparsers.add_parser("bump") + group = bump.add_mutually_exclusive_group(required=True) + group.add_argument("--major", action="store_true") + group.add_argument("--minor", action="store_true") + group.add_argument("--patch", action="store_true") + + args = parser.parse_args() + if args.command == "show": + print(read_version()) + elif args.command == "check": + check() + elif args.command == "bump": + version = read_version() + if args.major: + version = version.bump_major() + elif args.minor: + version = version.bump_minor() + else: + version = version.bump_patch() + write_version(version) + + +if __name__ == "__main__": + main() diff --git a/scripts/install.ps1 b/scripts/install.ps1 new file mode 100644 index 00000000000..b23088d46c2 --- /dev/null +++ b/scripts/install.ps1 @@ -0,0 +1,107 @@ +param( + [string]$Channel = "canary", + [string]$Version = "", + [switch]$WrapperOnly, + [switch]$NoModifyPath, + [switch]$Yes, + [switch]$Help +) + +if ($Help) { + Write-Host "Usage: install.ps1 [-Channel canary|nightly] [-Version VERSION] [-WrapperOnly] [-NoModifyPath] [-Yes]" + exit 0 +} + +if ($Channel -ne "canary" -and $Channel -ne "nightly") { + Write-Error "Channel must be canary or nightly" + exit 1 +} + +$ManifestBaseUrl = if ($env:BAML_MANIFEST_BASE_URL) { $env:BAML_MANIFEST_BASE_URL.TrimEnd("/") } else { "https://pkg.boundaryml.com/manifest/v1" } +$BamlHome = if ($env:BAML_HOME) { $env:BAML_HOME } else { Join-Path $env:USERPROFILE ".baml" } +$BinDir = Join-Path $BamlHome "bin" +$WrapperPath = Join-Path $BinDir "baml.exe" + +switch ($env:PROCESSOR_ARCHITECTURE) { + "AMD64" { $Target = "x86_64-pc-windows-msvc" } + "ARM64" { $Target = "aarch64-pc-windows-msvc" } + default { + Write-Error "Unsupported Windows architecture $env:PROCESSOR_ARCHITECTURE" + exit 2 + } +} + +$Temp = Join-Path ([System.IO.Path]::GetTempPath()) ("baml-install-" + [System.Guid]::NewGuid()) +New-Item -ItemType Directory -Path $Temp | Out-Null +try { + $ManifestPath = Join-Path $Temp "wrapper.json" + try { + Invoke-WebRequest -Uri "$ManifestBaseUrl/wrapper.json" -OutFile $ManifestPath -UseBasicParsing + } catch { + Write-Error "Failed to download wrapper manifest: $_" + exit 3 + } + + $Manifest = Get-Content $ManifestPath -Raw | ConvertFrom-Json + $Artifact = $Manifest.artifacts.$Target + if (-not $Artifact) { + Write-Error "Wrapper manifest has no artifact for $Target" + exit 5 + } + + $Archive = Join-Path $Temp "wrapper.zip" + try { + Invoke-WebRequest -Uri $Artifact.url -OutFile $Archive -UseBasicParsing + } catch { + Write-Error "Failed to download $($Artifact.url): $_" + exit 3 + } + + $Actual = (Get-FileHash -Algorithm SHA256 -Path $Archive).Hash.ToLowerInvariant() + if ($Actual -ne $Artifact.sha256.ToLowerInvariant()) { + Write-Error "Checksum verification failed for wrapper archive" + exit 4 + } + + $Extract = Join-Path $Temp "extract" + Expand-Archive -Path $Archive -DestinationPath $Extract -Force + $ExtractedWrapper = Join-Path $Extract "bin\baml.exe" + if (-not (Test-Path $ExtractedWrapper)) { + Write-Error "Wrapper archive must contain bin/baml.exe" + exit 5 + } + if ((Test-Path (Join-Path $Extract "bin\baml-cli.exe")) -or (Test-Path (Join-Path $Extract "bin\baml-pack-host.exe")) -or (Test-Path (Join-Path $Extract "assets\baml-vscode.vsix"))) { + Write-Error "Wrapper archive layout is not safe" + exit 5 + } + + New-Item -ItemType Directory -Path $BinDir -Force | Out-Null + $TmpWrapper = Join-Path $BinDir ".baml.exe.tmp" + Copy-Item $ExtractedWrapper $TmpWrapper -Force + Move-Item $TmpWrapper $WrapperPath -Force + + if (-not $NoModifyPath -and $Yes) { + $UserPath = [Environment]::GetEnvironmentVariable("Path", "User") + $Parts = @($UserPath -split ';' | Where-Object { $_ }) + if ($Parts -notcontains $BinDir) { + [Environment]::SetEnvironmentVariable("Path", (($Parts + $BinDir) -join ';'), "User") + Write-Host "Updated user PATH. Restart your terminal to pick up baml.exe." + } + } + + if (-not $WrapperOnly) { + $Selector = if ($Version) { $Version } else { $Channel } + & $WrapperPath toolchain use $Selector + if ($LASTEXITCODE -ne 0) { + Write-Error "Wrapper installed, but toolchain bootstrap failed. Retry: $WrapperPath toolchain use $Selector" + exit 7 + } + } + + Write-Host "BAML installed at $WrapperPath" + if ($NoModifyPath -or -not $Yes) { + Write-Host "Add $BinDir to your user PATH to run baml.exe from new terminals." + } +} finally { + Remove-Item -Path $Temp -Recurse -Force -ErrorAction SilentlyContinue +} diff --git a/scripts/install.sh b/scripts/install.sh new file mode 100755 index 00000000000..24a7d2f82cd --- /dev/null +++ b/scripts/install.sh @@ -0,0 +1,154 @@ +#!/bin/sh +set -eu + +MANIFEST_BASE_URL="${BAML_MANIFEST_BASE_URL:-https://pkg.boundaryml.com/manifest/v1}" +BAML_HOME="${BAML_HOME:-$HOME/.baml}" +CHANNEL="canary" +VERSION="" +WRAPPER_ONLY=0 +NO_MODIFY_PATH=0 +YES=0 + +usage() { + cat <<'USAGE' +Usage: install.sh [--channel canary|nightly] [--version VERSION] [--wrapper-only] [--no-modify-path] [--yes] +USAGE +} + +while [ "$#" -gt 0 ]; do + case "$1" in + --channel) CHANNEL="${2:?missing channel}"; shift 2 ;; + --version) VERSION="${2:?missing version}"; shift 2 ;; + --wrapper-only) WRAPPER_ONLY=1; shift ;; + --no-modify-path) NO_MODIFY_PATH=1; shift ;; + --yes) YES=1; shift ;; + --help|-h) usage; exit 0 ;; + *) echo "error: unknown argument $1" >&2; usage >&2; exit 1 ;; + esac +done + +case "$CHANNEL" in + canary|nightly) ;; + *) echo "error: --channel must be canary or nightly" >&2; exit 1 ;; +esac + +machine="$(uname -m)" +system="$(uname -s)" +case "$system:$machine" in + Darwin:arm64) target="aarch64-apple-darwin" ;; + Darwin:x86_64) target="x86_64-apple-darwin" ;; + Linux:aarch64|Linux:arm64) target="aarch64-unknown-linux-gnu" ;; + Linux:x86_64) target="x86_64-unknown-linux-gnu" ;; + *) echo "error: unsupported platform $system $machine" >&2; exit 2 ;; +esac + +tmp="${TMPDIR:-/tmp}/baml-install.$$" +mkdir -p "$tmp" +cleanup() { rm -rf "$tmp"; } +trap cleanup EXIT INT TERM + +manifest="$tmp/wrapper.json" +if ! curl -fsSL "$MANIFEST_BASE_URL/wrapper.json" -o "$manifest"; then + echo "error: failed to download wrapper manifest" >&2 + exit 3 +fi + +json="$(tr -d '\n' < "$manifest")" +block="$(printf '%s' "$json" | sed -n "s/.*\"$target\"[[:space:]]*:[[:space:]]*{\([^}]*\)}.*/\1/p")" +url="$(printf '%s' "$block" | sed -n 's/.*"url"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p')" +sha256="$(printf '%s' "$block" | sed -n 's/.*"sha256"[[:space:]]*:[[:space:]]*"\([0-9a-fA-F]*\)".*/\1/p' | tr 'A-F' 'a-f')" + +if [ -z "$url" ] || [ -z "$sha256" ]; then + echo "error: wrapper manifest has no artifact for $target" >&2 + exit 5 +fi + +archive="$tmp/wrapper.archive" +if ! curl -fsSL "$url" -o "$archive"; then + echo "error: failed to download $url" >&2 + exit 3 +fi + +actual="$(if command -v sha256sum >/dev/null 2>&1; then sha256sum "$archive" | awk '{print $1}'; else shasum -a 256 "$archive" | awk '{print $1}'; fi)" +if [ "$actual" != "$sha256" ]; then + echo "error: checksum verification failed for wrapper archive" >&2 + exit 4 +fi + +entries="$tmp/entries.txt" +case "$archive" in + *.zip) unzip -Z1 "$archive" > "$entries" ;; + *) tar -tzf "$archive" > "$entries" ;; +esac + +if grep -Eq '(^/|(^|/)\.\.(/|$)|baml-cli|baml-pack-host|baml-vscode\.vsix)' "$entries"; then + echo "error: wrapper archive layout is not safe" >&2 + exit 5 +fi +if ! grep -qx 'bin/baml' "$entries"; then + echo "error: wrapper archive must contain bin/baml" >&2 + exit 5 +fi + +extract="$tmp/extract" +mkdir -p "$extract" +case "$archive" in + *.zip) unzip -q "$archive" -d "$extract" ;; + *) tar -xzf "$archive" -C "$extract" ;; +esac + +mkdir -p "$BAML_HOME/bin" +install_tmp="$BAML_HOME/bin/.baml.tmp" +cp "$extract/bin/baml" "$install_tmp" +chmod 755 "$install_tmp" +mv "$install_tmp" "$BAML_HOME/bin/baml" + +cat > "$BAML_HOME/env" <<'ENV' +export BAML_HOME="${BAML_HOME:-$HOME/.baml}" +case ":$PATH:" in + *":$BAML_HOME/bin:"*) ;; + *) export PATH="$BAML_HOME/bin:$PATH" ;; +esac +ENV + +if [ "$NO_MODIFY_PATH" -eq 0 ] && [ "$YES" -eq 1 ]; then + line='. "$HOME/.baml/env"' + shell_name="$(basename "${SHELL:-}")" + case "$shell_name" in + zsh) profiles="$HOME/.zshrc" ;; + bash) profiles="$HOME/.bashrc" ;; + *) profiles="" ;; + esac + for profile in $profiles; do + touch "$profile" + if ! grep -Fqx "$line" "$profile"; then + printf '\n%s\n' "$line" >> "$profile" || exit 6 + fi + done +fi + +if [ "$WRAPPER_ONLY" -eq 0 ]; then + selector="$CHANNEL" + if [ -n "$VERSION" ]; then + selector="$VERSION" + fi + if ! "$BAML_HOME/bin/baml" toolchain use "$selector"; then + echo "error: wrapper installed, but toolchain bootstrap failed" >&2 + echo "retry: $BAML_HOME/bin/baml toolchain use $selector" >&2 + exit 7 + fi +fi + +echo "BAML installed at $BAML_HOME/bin/baml" +if [ "$NO_MODIFY_PATH" -eq 1 ] || [ "$YES" -eq 0 ]; then + cat <<'NEXT' + +Add BAML to your PATH by adding this to your shell profile: + + . "$HOME/.baml/env" + +Or run for this shell session: + + export PATH="$HOME/.baml/bin:$PATH" +NEXT +fi diff --git a/tools/pkg_boundaryml_com/data/index.html b/tools/pkg_boundaryml_com/data/index.html index f291fa2acbb..0b9b05f5280 100644 --- a/tools/pkg_boundaryml_com/data/index.html +++ b/tools/pkg_boundaryml_com/data/index.html @@ -7,6 +7,8 @@

pkg.boundaryml.com

-

BAML release artifacts.

+

BAML wrapper installers and language toolchain manifests.

+ 
curl -fsSL https://pkg.boundaryml.com/install.sh | sh -s
+

Homebrew and AUR packages install the wrapper only. Run baml toolchain use canary after package-manager install.

diff --git a/typescript2/README.md b/typescript2/README.md index e75c86ecef2..389eb99272a 100644 --- a/typescript2/README.md +++ b/typescript2/README.md @@ -71,44 +71,36 @@ What `pnpm vscode:package` does: pnpm build:wasm ``` - The current VSIX does not bundle this WASM output; it ships the native `baml-cli` and the webview talks to that CLI over WebSocket. Building WASM here makes the local packaging command catch browser/WASM regressions too. + The VSIX is platform-neutral. It does not bundle `baml-cli`; the extension launches `baml lsp` through the configured `baml` wrapper or `baml` on PATH. -3. Builds the release language-server CLI: - - ```bash - cargo build -p baml_cli --manifest-path ../baml_language/Cargo.toml --release - ``` - -4. Builds the packaged React webview: +3. Builds the packaged React webview: ```bash pnpm --filter app-vscode-webview build ``` -5. Builds the VS Code extension host bundle: +4. Builds the VS Code extension host bundle: ```bash pnpm --filter app-vscode-ext build ``` -6. Stages runtime assets into the extension package: +5. Stages runtime assets into the extension package: ```bash - rm -rf app-vscode-ext/dist/playground app-vscode-ext/dist/baml-cli - mkdir -p app-vscode-ext/dist/playground app-vscode-ext/dist/baml-cli + rm -rf app-vscode-ext/dist/playground + mkdir -p app-vscode-ext/dist/playground cp -R app-vscode-webview/dist/. app-vscode-ext/dist/playground/ - cp ../baml_language/target/release/baml-cli app-vscode-ext/dist/baml-cli/baml-cli - chmod 755 app-vscode-ext/dist/baml-cli/baml-cli ``` -7. Packages the VSIX: +6. Packages the VSIX: ```bash cd app-vscode-ext pnpm dlx @vscode/vsce package --no-dependencies --allow-missing-repository --skip-license ``` -The packaged extension includes `dist/extension.js`, the built webview under `dist/playground`, and the bundled `baml-cli` under `dist/baml-cli`. The bundled CLI is platform-specific, so build the VSIX on the same platform/architecture as the machine that will install it. +The packaged extension includes `dist/extension.js` and the built webview under `dist/playground`. It launches `baml lsp` through the wrapper, so the same VSIX can work across supported platforms and project toolchain pins. ### Standalone Web App diff --git a/typescript2/app-vscode-ext/package.json b/typescript2/app-vscode-ext/package.json index a10027d71e8..58166319c30 100644 --- a/typescript2/app-vscode-ext/package.json +++ b/typescript2/app-vscode-ext/package.json @@ -1,8 +1,8 @@ { - "name": "app-vscode-ext", + "name": "baml-language", "displayName": "BAML", "description": "BAML language support — syntax highlighting, hover, completions, diagnostics, go-to-definition", - "version": "0.1.0", + "version": "0.11.0", "publisher": "Boundary", "private": true, "type": "commonjs", @@ -80,7 +80,7 @@ "baml.cliPath": { "type": ["string", "null"], "default": null, - "description": "Path to baml-cli binary. If not set, uses the bundled CLI when available, then falls back to 'baml-cli' on PATH.", + "description": "Path to the baml wrapper. If not set, uses 'baml' on PATH.", "scope": "machine-overridable" }, "baml.trace.server": { diff --git a/typescript2/app-vscode-ext/src/compat.ts b/typescript2/app-vscode-ext/src/compat.ts new file mode 100644 index 00000000000..7dcf3ad381e --- /dev/null +++ b/typescript2/app-vscode-ext/src/compat.ts @@ -0,0 +1,22 @@ +export const BAML_LSP_PROTOCOL_MIN = 1; +export const BAML_LSP_PROTOCOL_MAX = 1; +export const BAML_PLAYGROUND_PROTOCOL_MIN = 1; +export const BAML_PLAYGROUND_PROTOCOL_MAX = 1; + +export type BamlProtocolRange = { + min: number; + max: number; +}; + +export type BamlServerMetadata = { + toolchainVersion?: string; + lspProtocol?: number; + minSupportedClientLspProtocol?: number; + playgroundProtocol?: number; + minSupportedClientPlaygroundProtocol?: number; + capabilities?: string[]; +}; + +export function isProtocolCompatible(serverProtocol: number, serverMinClientProtocol: number, clientRange: BamlProtocolRange): boolean { + return serverProtocol >= clientRange.min && serverProtocol <= clientRange.max && clientRange.max >= serverMinClientProtocol; +} diff --git a/typescript2/app-vscode-ext/src/extension.ts b/typescript2/app-vscode-ext/src/extension.ts index b8fa13b257c..e36936462f0 100644 --- a/typescript2/app-vscode-ext/src/extension.ts +++ b/typescript2/app-vscode-ext/src/extension.ts @@ -1,5 +1,6 @@ import * as vscode from 'vscode'; import * as fs from 'fs'; +import * as path from 'path'; import { LanguageClient, type LanguageClientOptions, @@ -7,14 +8,25 @@ import { State, } from 'vscode-languageclient/node'; import { WebviewPanel } from './panels/WebviewPanel'; - -let client: LanguageClient | undefined; +import { + BAML_LSP_PROTOCOL_MAX, + BAML_LSP_PROTOCOL_MIN, + BAML_PLAYGROUND_PROTOCOL_MAX, + BAML_PLAYGROUND_PROTOCOL_MIN, + isProtocolCompatible, + type BamlServerMetadata, +} from './compat'; + +const clients = new Map(); let knownProjects: string[] = []; let currentServerState: 'starting' | 'running' | 'stopped' | 'error' = 'starting'; let statusBarItem: vscode.StatusBarItem | undefined; +let extensionContext: vscode.ExtensionContext | undefined; +let playgroundDir: string | undefined; +let wrapperPath = 'baml'; function getExtVersion(): string { - return vscode.extensions.getExtension('Boundary.app-vscode-ext')?.packageJSON?.version ?? '?'; + return vscode.extensions.getExtension('Boundary.baml-language')?.packageJSON?.version ?? '?'; } /** Short display name: last path component (e.g. "/Users/x/repos/myapp/baml_src" → "myapp/baml_src") */ @@ -24,7 +36,7 @@ function projectLabel(fullPath: string): string { } function buildStatusTooltip(serverState: 'starting' | 'running' | 'stopped' | 'error'): vscode.MarkdownString { - const serverVersion = client?.initializeResult?.serverInfo?.version ?? '—'; + const serverVersion = activeClient()?.initializeResult?.serverInfo?.version ?? '—'; const md = new vscode.MarkdownString(undefined, true); md.isTrusted = true; @@ -81,45 +93,63 @@ function refreshTooltip() { } } -function getBundledCliPath(context: vscode.ExtensionContext): string | undefined { - const binaryName = process.platform === 'win32' ? 'baml-cli.exe' : 'baml-cli'; - const bundledPath = vscode.Uri.joinPath(context.extensionUri, 'dist', 'baml-cli', binaryName).fsPath; +function getPlaygroundDir(context: vscode.ExtensionContext): string | undefined { + const playgroundDir = vscode.Uri.joinPath(context.extensionUri, 'dist', 'playground').fsPath; + return fs.existsSync(playgroundDir) ? playgroundDir : undefined; +} - if (!fs.existsSync(bundledPath)) { - return undefined; +function activeDocumentUri(): vscode.Uri | undefined { + const editor = vscode.window.activeTextEditor; + if (editor?.document.languageId === 'baml' && editor.document.uri.scheme === 'file') { + return editor.document.uri; } + return vscode.workspace.textDocuments.find((doc) => doc.languageId === 'baml' && doc.uri.scheme === 'file')?.uri; +} - if (process.platform !== 'win32') { - try { - fs.chmodSync(bundledPath, 0o755); - } catch { - // Best effort: VS Code usually preserves the executable bit from the VSIX. - } +function activeClient(): LanguageClient | undefined { + const uri = activeDocumentUri(); + if (!uri) { + return clients.values().next().value; } - - return bundledPath; + const root = findBamlProjectRoot(uri); + return clients.get(root); } -function getPlaygroundDir(context: vscode.ExtensionContext): string | undefined { - const playgroundDir = vscode.Uri.joinPath(context.extensionUri, 'dist', 'playground').fsPath; - return fs.existsSync(playgroundDir) ? playgroundDir : undefined; -} +function findBamlProjectRoot(uri: vscode.Uri): string { + let dir = fs.statSync(uri.fsPath).isDirectory() ? uri.fsPath : path.dirname(uri.fsPath); + const workspaceFolder = vscode.workspace.getWorkspaceFolder(uri); + const workspaceRoot = workspaceFolder?.uri.fsPath; -export async function activate(context: vscode.ExtensionContext) { - const config = vscode.workspace.getConfiguration('baml'); - const playgroundDir = getPlaygroundDir(context); + while (true) { + if (fs.existsSync(path.join(dir, 'baml.toml'))) { + return dir; + } + if (workspaceRoot && dir === workspaceRoot) { + return workspaceRoot; + } + const parent = path.dirname(dir); + if (parent === dir) { + break; + } + dir = parent; + } - // Priority: BAML_CLI_PATH env var (for debug) -> setting -> bundled CLI -> PATH lookup. - const cliPath = - process.env.BAML_CLI_PATH ?? - config.get('cliPath') ?? - getBundledCliPath(context) ?? - 'baml-cli'; + return workspaceRoot ?? path.dirname(uri.fsPath); +} +async function ensureClient(projectRoot: string): Promise { + const existing = clients.get(projectRoot); + if (existing) { + if (existing.state === State.Stopped) { + await existing.start(); + } + return existing; + } const serverOptions: ServerOptions = { - command: cliPath, + command: wrapperPath, args: ['lsp'], options: { + cwd: projectRoot, env: { ...process.env, ...(playgroundDir ? { BAML_PLAYGROUND_DIR: playgroundDir } : {}), @@ -132,21 +162,35 @@ export async function activate(context: vscode.ExtensionContext) { synchronize: { fileEvents: vscode.workspace.createFileSystemWatcher('**/*.baml'), }, + initializationOptions: { + bamlClient: { + kind: 'vscode', + extensionVersion: getExtVersion(), + projectRoot, + supportedLspProtocol: { min: BAML_LSP_PROTOCOL_MIN, max: BAML_LSP_PROTOCOL_MAX }, + supportedPlaygroundProtocol: { min: BAML_PLAYGROUND_PROTOCOL_MIN, max: BAML_PLAYGROUND_PROTOCOL_MAX }, + capabilities: ['openPlayground.v1', 'listProjects.v1', 'playgroundWebSocket.v1'], + }, + }, }; - client = new LanguageClient( - 'baml', - 'BAML Language Server', + const client = new LanguageClient( + `baml:${projectRoot}`, + `BAML Language Server (${projectLabel(projectRoot)})`, serverOptions, clientOptions, ); + clients.set(projectRoot, client); + wireClient(client); + await client.start(); + return client; +} - statusBarItem = vscode.window.createStatusBarItem(vscode.StatusBarAlignment.Left, 0); - statusBarItem.text = '$(loading~spin) 🐑 BAML'; - statusBarItem.tooltip = buildStatusTooltip('starting'); - statusBarItem.show(); - context.subscriptions.push(statusBarItem); - +function wireClient(client: LanguageClient) { + if (!extensionContext) { + return; + } + const context = extensionContext; client.onDidChangeState((e) => { switch (e.newState) { case State.Starting: @@ -154,6 +198,7 @@ export async function activate(context: vscode.ExtensionContext) { break; case State.Running: updateStatusBar('running'); + validateServerCompatibility(client); break; case State.Stopped: knownProjects = []; @@ -162,16 +207,77 @@ export async function activate(context: vscode.ExtensionContext) { } }); + client.onNotification( + 'baml/openPlayground', + async (params: { port: number; projectPath: string; functionName?: string }) => { + await WebviewPanel.render(context.extensionUri, params.port, { + project: params.projectPath, + ...(params.functionName ? { functionName: params.functionName } : {}), + }); + }, + ); + + client.onNotification( + 'baml/listProjects', + (params: { projects: string[] }) => { + knownProjects = params.projects ?? []; + refreshTooltip(); + }, + ); +} + +function validateServerCompatibility(client: LanguageClient) { + const metadata = client.initializeResult?.capabilities?.experimental?.baml as BamlServerMetadata | undefined; + if (!metadata?.lspProtocol || !metadata.minSupportedClientLspProtocol) { + return; + } + if (!isProtocolCompatible(metadata.lspProtocol, metadata.minSupportedClientLspProtocol, { + min: BAML_LSP_PROTOCOL_MIN, + max: BAML_LSP_PROTOCOL_MAX, + })) { + vscode.window.showWarningMessage('BAML language server protocol is incompatible with this extension. Update the BAML extension or the active BAML toolchain.'); + } +} + +export async function activate(context: vscode.ExtensionContext) { + extensionContext = context; + const config = vscode.workspace.getConfiguration('baml'); + playgroundDir = getPlaygroundDir(context); + wrapperPath = process.env.BAML_CLI_PATH ?? config.get('cliPath') ?? 'baml'; + + statusBarItem = vscode.window.createStatusBarItem(vscode.StatusBarAlignment.Left, 0); + statusBarItem.text = '$(loading~spin) 🐑 BAML'; + statusBarItem.tooltip = buildStatusTooltip('starting'); + statusBarItem.show(); + context.subscriptions.push(statusBarItem); + + const startForUri = async (uri: vscode.Uri | undefined) => { + if (uri) { + await ensureClient(findBamlProjectRoot(uri)); + } + }; + context.subscriptions.push(vscode.window.onDidChangeActiveTextEditor((editor) => { + if (editor?.document.languageId === 'baml') { + void startForUri(editor.document.uri); + } + })); + context.subscriptions.push(vscode.workspace.onDidOpenTextDocument((document) => { + if (document.languageId === 'baml' && document.uri.scheme === 'file') { + void startForUri(document.uri); + } + })); + // ── Commands ──────────────────────────────────────────────────────── context.subscriptions.push( vscode.commands.registerCommand('baml.openLogs', () => { - client?.outputChannel.show(); + activeClient()?.outputChannel.show(); }), ); context.subscriptions.push( vscode.commands.registerCommand('baml.restartLanguageServer', async () => { + const client = activeClient(); if (client) { await client.restart(); vscode.window.showInformationMessage('BAML Language Server restarted.'); @@ -181,6 +287,7 @@ export async function activate(context: vscode.ExtensionContext) { context.subscriptions.push( vscode.commands.registerCommand('baml.stopLanguageServer', async () => { + const client = activeClient(); if (client) { await client.stop(); vscode.window.showInformationMessage('BAML Language Server stopped.'); @@ -190,6 +297,7 @@ export async function activate(context: vscode.ExtensionContext) { context.subscriptions.push( vscode.commands.registerCommand('baml.startLanguageServer', async () => { + const client = activeClient(); if (client) { await client.start(); vscode.window.showInformationMessage('BAML Language Server started.'); @@ -202,6 +310,7 @@ export async function activate(context: vscode.ExtensionContext) { // NativePlaygroundSender can decide how to open it (and attach the port). context.subscriptions.push( vscode.commands.registerCommand('baml.openPlayground', async (projectPath?: string) => { + const client = projectPath ? clients.get(projectPath) : activeClient(); if (!client || client.state !== State.Running) { vscode.window.showWarningMessage('BAML Language Server is not running.'); return; @@ -217,32 +326,11 @@ export async function activate(context: vscode.ExtensionContext) { }), ); - await client.start(); - - // The LSP sends `baml/openPlayground` when the user clicks a code lens - // or invokes the manual command above. The notification carries the port. - client.onNotification( - 'baml/openPlayground', - async (params: { port: number; projectPath: string; functionName?: string }) => { - await WebviewPanel.render(context.extensionUri, params.port, { - project: params.projectPath, - ...(params.functionName ? { functionName: params.functionName } : {}), - }); - }, - ); - - // Track discovered projects so the status bar tooltip can show per-project links. - client.onNotification( - 'baml/listProjects', - (params: { projects: string[] }) => { - knownProjects = params.projects ?? []; - refreshTooltip(); - }, - ); + await startForUri(activeDocumentUri()); } export async function deactivate() { - if (client) { + for (const client of clients.values()) { await client.stop(); } } diff --git a/typescript2/package.json b/typescript2/package.json index 913111ca705..1e6757e110b 100644 --- a/typescript2/package.json +++ b/typescript2/package.json @@ -8,13 +8,13 @@ "dev:wasm": "BRIDGE_WASM_FORCE_RERUN=1 npx nodemon --watch ../baml_language --ignore '**/target/**' --ext rs,toml --exec \"pnpm build:wasm\"", "dev:webview": "pnpm --filter app-vscode-webview dev", "dev:cargo": "cargo watch -C ../baml_language --ignore '**/target/**' --skip-local-deps --watch-when-idle -w crates -x 'build -p baml_cli'", - "dev:vscode": "turbo run build --filter=app-vscode-webview && turbo run dev --filter=app-vscode-ext --filter=app-vscode-webview --filter=@b/pkg-playground '//#dev:cargo'", + "dev:vscode": "turbo run build --filter=app-vscode-webview && turbo run dev --filter=baml-language --filter=app-vscode-webview --filter=@b/pkg-playground '//#dev:cargo'", "dev:promptfiddle": "turbo run dev --filter app-promptfiddle --filter @b/pkg-playground '//#dev:wasm'", "build": "turbo run build", "vscode:package": "pnpm run vscode:package:clean && pnpm run vscode:package:build && pnpm run vscode:package:stage && pnpm run vscode:package:vsix", "vscode:package:clean": "rm -rf app-vscode-webview/dist app-vscode-ext/dist app-vscode-ext/*.vsix", - "vscode:package:build": "pnpm run build:wasm && cargo build -p baml_cli --manifest-path ../baml_language/Cargo.toml --release && pnpm --filter app-vscode-webview build && pnpm --filter app-vscode-ext build", - "vscode:package:stage": "rm -rf app-vscode-ext/dist/playground app-vscode-ext/dist/baml-cli && mkdir -p app-vscode-ext/dist/playground app-vscode-ext/dist/baml-cli && cp -R app-vscode-webview/dist/. app-vscode-ext/dist/playground/ && cp ../baml_language/target/release/baml-cli app-vscode-ext/dist/baml-cli/baml-cli && chmod 755 app-vscode-ext/dist/baml-cli/baml-cli", + "vscode:package:build": "pnpm run build:wasm && pnpm --filter app-vscode-webview build && pnpm --filter baml-language build", + "vscode:package:stage": "rm -rf app-vscode-ext/dist/playground && mkdir -p app-vscode-ext/dist/playground && cp -R app-vscode-webview/dist/. app-vscode-ext/dist/playground/", "vscode:package:vsix": "cd app-vscode-ext && pnpm dlx @vscode/vsce package --no-dependencies --allow-missing-repository --skip-license", "typecheck": "turbo run typecheck", "test": "turbo run test", diff --git a/typescript2/pkg-playground/src/ports/WebSocketRuntimePort.ts b/typescript2/pkg-playground/src/ports/WebSocketRuntimePort.ts index f80f3747245..b72ccb89a3e 100644 --- a/typescript2/pkg-playground/src/ports/WebSocketRuntimePort.ts +++ b/typescript2/pkg-playground/src/ports/WebSocketRuntimePort.ts @@ -17,9 +17,11 @@ import { decodeCallResult, RuntimeEvent } from '@b/pkg-proto'; import { truncateMessage, normalizeLogLevel } from '../shared/log-decorations'; import { formatValue } from '../shared/format-value'; import { deserializeRuntimeEvent } from '../shared/deserialize-event'; +import { isPlaygroundProtocolCompatible } from '../protocol'; /** Server → Client message shapes (must match playground_ws.rs WsOutMessage) */ type WsOutMessage = + | { type: 'hello'; toolchainVersion: string; playgroundProtocol: number; minClientPlaygroundProtocol: number; capabilities: string[] } | { type: 'ready' } | { type: 'playgroundNotification'; notification: PlaygroundNotification } | { type: 'callFunctionResult'; id: number; result: string } @@ -64,6 +66,7 @@ export class WebSocketRuntimePort implements RuntimePort { private reconnectTimer: ReturnType | null = null; private decorationsByLine = new Map(); private textEncoder = new TextEncoder(); + private playgroundCompatible = true; constructor(url: string) { this.url = url; @@ -255,7 +258,19 @@ export class WebSocketRuntimePort implements RuntimePort { private fromServer(raw: WsOutMessage): WorkerOutMessage | null { switch (raw.type) { + case 'hello': + this.playgroundCompatible = isPlaygroundProtocolCompatible(raw.playgroundProtocol, raw.minClientPlaygroundProtocol); + if (!this.playgroundCompatible) { + return { + type: 'runtimeEventError', + error: `BAML playground protocol ${raw.playgroundProtocol} from toolchain ${raw.toolchainVersion} is incompatible with this extension.`, + }; + } + return null; case 'ready': + if (!this.playgroundCompatible) { + return null; + } return { type: 'ready' }; case 'playgroundNotification': return { type: 'playgroundNotification', notification: raw.notification }; diff --git a/typescript2/pkg-playground/src/protocol.ts b/typescript2/pkg-playground/src/protocol.ts new file mode 100644 index 00000000000..5cf9032b8a1 --- /dev/null +++ b/typescript2/pkg-playground/src/protocol.ts @@ -0,0 +1,8 @@ +export const BAML_PLAYGROUND_PROTOCOL_MIN = 1; +export const BAML_PLAYGROUND_PROTOCOL_MAX = 1; + +export function isPlaygroundProtocolCompatible(serverProtocol: number, minClientProtocol: number): boolean { + return serverProtocol >= BAML_PLAYGROUND_PROTOCOL_MIN + && serverProtocol <= BAML_PLAYGROUND_PROTOCOL_MAX + && BAML_PLAYGROUND_PROTOCOL_MAX >= minClientProtocol; +} From f3bc96fab88b362ef65b95a37b9a40a8df73c55c Mon Sep 17 00:00:00 2001 From: rossirpaulo Date: Tue, 2 Jun 2026 11:33:57 -0500 Subject: [PATCH 2/7] Remove stale BAML language release checks --- .github/workflows/ci.yaml | 36 +++++++++++++++++-- .github/workflows/release-baml-language.yml | 2 +- TASK/docs/release-maintainer.md | 4 +++ .../lib/pkg-boundaryml-com-stack.ts | 4 +-- 4 files changed, 41 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 370c1044dd6..fae6b4eb73e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -74,10 +74,17 @@ jobs: if git diff --quiet "${MERGE_BASE}...HEAD" -- \ ':baml_language/**' \ ':scripts/baml-language-version' \ + ':scripts/baml-wrapper-version' \ + ':scripts/baml-release-manifests' \ + ':scripts/baml-package-manager-artifacts' \ ':scripts/install.sh' \ ':scripts/install.ps1' \ + ':packaging/aur/**' \ + ':tools/pkg_boundaryml_com/**' \ ':.github/workflows/ci.yaml' \ ':.github/workflows/release-baml-language.yml' \ + ':.github/workflows/publish-python-pypi.yml' \ + ':.github/workflows/build-python-sdk.reusable.yaml' \ ':.github/workflows/cargo-tests.reusable.yaml' \ ':.github/workflows/size-gate.reusable.yaml' \ ':.github/workflows/wasm-pack-tests.reusable.yaml' \ @@ -306,8 +313,33 @@ jobs: - name: "Validate baml_language release metadata" run: scripts/baml-language-version check - - name: "Validate alpha version computation" - run: test "$(scripts/baml-language-version compute --alpha-number 1842)" = "$(scripts/baml-language-version show)-alpha.1842" + - name: "Validate canary/nightly version computation" + env: + BAML_LANGUAGE_VERSION_DATE: "20260522" + run: | + set -euo pipefail + canary="$(scripts/baml-language-version show)" + test "$(scripts/baml-language-version compute --channel canary)" = "$canary" + test "$(scripts/baml-language-version compute --channel canary --pypi)" = "$canary" + + nightly="$(scripts/baml-language-version compute --channel nightly)" + pypi="$(scripts/baml-language-version compute --channel nightly --pypi)" + python3 - "$canary" "$nightly" "$pypi" <<'PY' + import re + import sys + + canary, nightly, pypi = sys.argv[1:] + major, minor, patch = [int(part) for part in canary.split(".")] + base = f"{major}.{minor}.{patch + 1}" + match = re.fullmatch(rf"{re.escape(base)}-nightly\.20260522\.([a-z])", nightly) + if not match: + raise SystemExit(f"unexpected nightly version: {nightly}") + + index = ord(match.group(1)) - ord("a") + expected_pypi = f"{base}.dev20260522{index:02d}" + if pypi != expected_pypi: + raise SystemExit(f"unexpected nightly PyPI version: {pypi} != {expected_pypi}") + PY # Call reusable workflows cargo-tests: diff --git a/.github/workflows/release-baml-language.yml b/.github/workflows/release-baml-language.yml index 179e7dffdba..87d25613864 100644 --- a/.github/workflows/release-baml-language.yml +++ b/.github/workflows/release-baml-language.yml @@ -1,6 +1,6 @@ name: BAML Language Release -# Channel-aware replacement for the alpha language release workflow. +# Channel-aware replacement for the legacy BAML language release workflow. # Release workflows run packaging smoke tests only; the full suite runs in CI. on: diff --git a/TASK/docs/release-maintainer.md b/TASK/docs/release-maintainer.md index af5e905cbf6..d522cbdc2cc 100644 --- a/TASK/docs/release-maintainer.md +++ b/TASK/docs/release-maintainer.md @@ -64,3 +64,7 @@ Homebrew and AUR publish only wrapper packages when `wrapper_changed == true`. T ## Rollback Rollback is a code revert plus mutable pointer repair. Leave immutable artifacts in place and move channel pointers to a fixed version. + +## Legacy Alpha Consumers + +`tools/baml-bench` still resolves `baml-language-*-alpha.*` GitHub pre-releases and downloads a matching release binary. That is separate service logic, not part of the new release graph. Before deleting the old alpha release assets entirely, either migrate that service to `manifest/v1/{canary,nightly}.json` or explicitly retire the benchmark builder path. diff --git a/tools/pkg_boundaryml_com/lib/pkg-boundaryml-com-stack.ts b/tools/pkg_boundaryml_com/lib/pkg-boundaryml-com-stack.ts index 6e8bda69132..28a6d6cc760 100644 --- a/tools/pkg_boundaryml_com/lib/pkg-boundaryml-com-stack.ts +++ b/tools/pkg_boundaryml_com/lib/pkg-boundaryml-com-stack.ts @@ -31,13 +31,13 @@ export class PkgBoundarymlComStack extends cdk.Stack { this.githubReleaseRole = new iam.Role(this, 'GitHubReleaseRole', { roleName: 'pkg-boundaryml-com-github-release', - description: `Assumed by GitHub Actions in ${GITHUB_REPO} (tag pushes only) to publish to the pkg.boundaryml.com bucket`, + description: `Assumed by GitHub Actions in ${GITHUB_REPO} from refs/heads/canary to publish to the pkg.boundaryml.com bucket`, assumedBy: new iam.OpenIdConnectPrincipal(githubProvider, { StringEquals: { 'token.actions.githubusercontent.com:aud': 'sts.amazonaws.com', }, StringLike: { - 'token.actions.githubusercontent.com:sub': `repo:${GITHUB_REPO}:ref:refs/tags/*`, + 'token.actions.githubusercontent.com:sub': `repo:${GITHUB_REPO}:ref:refs/heads/canary`, }, }), maxSessionDuration: cdk.Duration.hours(1), From 12463db4a787f1ec199039d8ccc6726ce19e7929 Mon Sep 17 00:00:00 2001 From: rossirpaulo Date: Wed, 3 Jun 2026 00:12:07 -0500 Subject: [PATCH 3/7] coderabbit: fixes --- .github/workflows/publish-python-pypi.yml | 1 - .github/workflows/release-baml-language.yml | 15 ++++ TASK/USER-PATHS.md | 2 +- baml_language/crates/baml/src/main.rs | 86 ++++++++++++++++--- .../crates/baml_cli/src/ide_command.rs | 5 +- baml_language/crates/baml_release/src/lib.rs | 28 ++++-- .../src/bex_lsp/multi_project/request.rs | 54 ++++++++++-- .../bex_project/src/bex_lsp/protocol.rs | 4 + scripts/baml-language-version | 7 +- scripts/baml-package-manager-artifacts | 8 +- scripts/baml-release-manifests | 4 +- scripts/baml-wrapper-version | 5 +- scripts/install.sh | 13 ++- typescript2/README.md | 2 +- typescript2/app-vscode-ext/src/extension.ts | 27 ++++-- 15 files changed, 212 insertions(+), 49 deletions(-) diff --git a/.github/workflows/publish-python-pypi.yml b/.github/workflows/publish-python-pypi.yml index 98fdbadb4d3..0cfe5d83edd 100644 --- a/.github/workflows/publish-python-pypi.yml +++ b/.github/workflows/publish-python-pypi.yml @@ -32,7 +32,6 @@ concurrency: permissions: contents: read - id-token: write jobs: # Version comes from the stamped checkout prepared by the release graph. diff --git a/.github/workflows/release-baml-language.yml b/.github/workflows/release-baml-language.yml index 87d25613864..ac3a02030a5 100644 --- a/.github/workflows/release-baml-language.yml +++ b/.github/workflows/release-baml-language.yml @@ -67,6 +67,7 @@ jobs: with: ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.sha }} fetch-depth: 2 + persist-credentials: false - name: Compute release plan id: plan @@ -158,6 +159,8 @@ jobs: os: windows-latest steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/download-artifact@v4 with: name: release-plan @@ -203,6 +206,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/download-artifact@v4 with: name: release-plan @@ -253,6 +258,8 @@ jobs: os: windows-latest steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Build wrapper run: cargo build --manifest-path baml_language/Cargo.toml --release --bin baml --target "${{ matrix.target }}" - name: Archive wrapper @@ -362,6 +369,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/download-artifact@v4 with: name: release-plan @@ -465,6 +474,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/download-artifact@v4 with: pattern: wrapper-* @@ -500,6 +511,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/download-artifact@v4 with: pattern: wrapper-* @@ -550,6 +563,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/download-artifact@v4 with: pattern: toolchain-* diff --git a/TASK/USER-PATHS.md b/TASK/USER-PATHS.md index 7bebfa5298c..aea94b640ba 100644 --- a/TASK/USER-PATHS.md +++ b/TASK/USER-PATHS.md @@ -230,7 +230,7 @@ PyPI cannot. Python uses PEP 440, so nightly releases are translated: | `0.11.1-nightly.20260522.a` | `0.11.1.dev2026052200` | | `0.11.1-nightly.20260522.b` | `0.11.1.dev2026052201` | -Python users still see the canonical BAML version from `baml_core.__version__` and `baml_core.get_version()`. PyPI stores the PEP 440 form because it must. All other foresseable SDKs don't have any issues with this system. +Python users still see the canonical BAML version from `baml_core.__version__` and `baml_core.get_version()`. PyPI stores the PEP 440 form because it must. All other foreseeable SDKs don't have any issues with this system. ### Deprecated Version Plumbing diff --git a/baml_language/crates/baml/src/main.rs b/baml_language/crates/baml/src/main.rs index 2c6ecb5cdfc..f85a45fb9db 100644 --- a/baml_language/crates/baml/src/main.rs +++ b/baml_language/crates/baml/src/main.rs @@ -16,6 +16,7 @@ use serde::{Deserialize, Serialize}; const CONFIG_FILE: &str = "config.toml"; const STATE_FILE: &str = "state.toml"; const CHANNEL_CACHE_TTL: Duration = Duration::from_secs(24 * 60 * 60); +const HTTP_TIMEOUT: Duration = Duration::from_secs(30); #[derive(Debug, Default, Serialize, Deserialize)] struct Config { @@ -94,7 +95,12 @@ fn main() { fn run() -> Result { let mut args: Vec = env::args().skip(1).collect(); - if args.iter().any(|arg| arg == "--version" || arg == "-V") { + #[cfg(windows)] + if args.first().map(String::as_str) == Some("--replace") { + args.remove(0); + return replace_running_exe(args).map(|()| 0); + } + if matches!(args.first().map(String::as_str), Some("--version" | "-V")) { println!("baml {}", env!("CARGO_PKG_VERSION")); return Ok(0); } @@ -290,14 +296,14 @@ fn project_toolchain_selector() -> Result> { let candidate = dir.join("baml.toml"); if candidate.exists() { let text = fs::read_to_string(&candidate)?; - if let Ok(config) = toml::from_str::(&text) { - if let Some(toolchain) = config.toolchain { - if let Some(version) = toolchain.version { - return Ok(Some((candidate, version))); - } - if let Some(channel) = toolchain.channel { - return Ok(Some((candidate, channel))); - } + let config = toml::from_str::(&text) + .with_context(|| format!("failed to parse {}", candidate.display()))?; + if let Some(toolchain) = config.toolchain { + if let Some(version) = toolchain.version { + return Ok(Some((candidate, version))); + } + if let Some(channel) = toolchain.channel { + return Ok(Some((candidate, channel))); } } } @@ -431,7 +437,10 @@ fn fetch_manifest(selector: &str, override_url: Option<&str>) -> Result text, None => { fetched_remote = true; - reqwest::blocking::get(&url) + let client = http_client()?; + client + .get(&url) + .send() .with_context(|| format!("failed to fetch {url}"))? .error_for_status() .with_context(|| format!("failed to fetch {url}"))? @@ -621,6 +630,17 @@ fn self_update() -> Result<()> { use std::os::unix::fs::PermissionsExt; fs::set_permissions(&tmp, fs::Permissions::from_mode(0o755))?; } + #[cfg(windows)] + { + std::process::Command::new(¤t) + .arg("--replace") + .arg(&tmp) + .arg(¤t) + .spawn() + .with_context(|| format!("failed to spawn updater {}", current.display()))?; + println!("updating BAML wrapper to {}", manifest.version); + return Ok(()); + } fs::rename(tmp, current)?; println!("updated BAML wrapper to {}", manifest.version); Ok(()) @@ -630,7 +650,10 @@ fn fetch_wrapper_manifest() -> Result { let base = baml_release::manifest_base_url(); let url = format!("{base}/wrapper.json"); let cache_path = manifest_cache_dir(&base).join("wrapper.json"); - let text = reqwest::blocking::get(&url) + let client = http_client()?; + let text = client + .get(&url) + .send() .with_context(|| format!("failed to fetch {url}"))? .error_for_status() .with_context(|| format!("failed to fetch {url}"))? @@ -643,6 +666,47 @@ fn fetch_wrapper_manifest() -> Result { Ok(manifest) } +fn http_client() -> Result { + reqwest::blocking::Client::builder() + .timeout(HTTP_TIMEOUT) + .build() + .context("failed to build HTTP client") +} + +#[cfg(windows)] +fn replace_running_exe(args: Vec) -> Result<()> { + if args.len() != 2 { + anyhow::bail!("usage: baml --replace "); + } + let tmp = PathBuf::from(&args[0]); + let current = PathBuf::from(&args[1]); + let mut last_error = None; + for _ in 0..60 { + std::thread::sleep(Duration::from_millis(250)); + match fs::remove_file(¤t) { + Ok(()) => match fs::rename(&tmp, ¤t) { + Ok(()) => return Ok(()), + Err(err) => last_error = Some(err), + }, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + match fs::rename(&tmp, ¤t) { + Ok(()) => return Ok(()), + Err(err) => last_error = Some(err), + } + } + Err(err) => last_error = Some(err), + } + } + Err(anyhow!( + "failed to replace {} with {}: {}", + current.display(), + tmp.display(), + last_error + .map(|err| err.to_string()) + .unwrap_or_else(|| "timed out".to_string()) + )) +} + fn write_text_atomic(path: &Path, text: &str) -> Result<()> { if let Some(parent) = path.parent() { fs::create_dir_all(parent)?; diff --git a/baml_language/crates/baml_cli/src/ide_command.rs b/baml_language/crates/baml_cli/src/ide_command.rs index e368d524762..3eeecee7e1b 100644 --- a/baml_language/crates/baml_cli/src/ide_command.rs +++ b/baml_language/crates/baml_cli/src/ide_command.rs @@ -27,7 +27,7 @@ pub(crate) enum IdeCommand { pub(crate) struct IdeInstallArgs { #[arg(long)] pub cursor: bool, - #[arg(long)] + #[arg(long, conflicts_with = "cursor")] pub code: bool, } @@ -66,9 +66,6 @@ impl IdeInstallArgs { } fn resolve_editor(&self) -> Result { - if self.cursor && self.code { - return Err(anyhow!("choose only one of --cursor or --code")); - } if self.cursor { return Ok(OsString::from("cursor")); } diff --git a/baml_language/crates/baml_release/src/lib.rs b/baml_language/crates/baml_release/src/lib.rs index 846109c43e9..850d848d86d 100644 --- a/baml_language/crates/baml_release/src/lib.rs +++ b/baml_language/crates/baml_release/src/lib.rs @@ -54,6 +54,11 @@ impl Product { pub enum FetchError { #[error("network error fetching {url}: {source}")] Network { url: String, source: reqwest::Error }, + #[error("HTTP {status} fetching {url}")] + HttpStatus { + url: String, + status: reqwest::StatusCode, + }, #[error("manifest 404 for version {version} (not released yet?)")] ManifestNotFound { version: String }, #[error("manifest schema {got} not supported (max {max}); run `baml self-update`")] @@ -152,7 +157,6 @@ impl Fetcher { pub fn install_to_toolchain_root(&self, root: &Path) -> Result { fs::create_dir_all(root)?; let _lock = InstallLock::acquire(root.join(format!(".{}.lock", self.spec.version)))?; - let archive = self.fetch_archive()?; let install_dir = root.join(&self.spec.version); if fs::read_to_string(install_dir.join("VERSION")) .map(|version| version.trim() == self.spec.version) @@ -161,6 +165,7 @@ impl Fetcher { return Ok(install_dir); } + let archive = self.fetch_archive()?; let tmp_dir = root.join(format!(".{}.tmp", self.spec.version)); if tmp_dir.exists() { fs::remove_dir_all(&tmp_dir)?; @@ -380,6 +385,12 @@ fn download_bytes(url: &str) -> Result, FetchError> { Ok(response) if response.status().is_server_error() || response.status().is_redirection() => { + if response.status().is_redirection() { + return Err(FetchError::HttpStatus { + url: url.to_string(), + status: response.status(), + }); + } last_error = response.error_for_status().err(); } Ok(response) => { @@ -399,10 +410,17 @@ fn download_bytes(url: &str) -> Result, FetchError> { thread::sleep(Duration::from_secs(2)); } } - Err(FetchError::Network { - url: url.to_string(), - source: last_error.expect("retry loop should capture a network error"), - }) + if let Some(source) = last_error { + Err(FetchError::Network { + url: url.to_string(), + source, + }) + } else { + Err(FetchError::HttpStatus { + url: url.to_string(), + status: reqwest::StatusCode::INTERNAL_SERVER_ERROR, + }) + } } pub fn extract_binary_from_archive( diff --git a/baml_language/crates/bex_project/src/bex_lsp/multi_project/request.rs b/baml_language/crates/bex_project/src/bex_lsp/multi_project/request.rs index 1950a975155..99ad7ef15ad 100644 --- a/baml_language/crates/bex_project/src/bex_lsp/multi_project/request.rs +++ b/baml_language/crates/bex_project/src/bex_lsp/multi_project/request.rs @@ -91,6 +91,16 @@ pub(super) fn server_capabilities() -> ServerCapabilities { } } +fn initialize_result() -> lsp_types::InitializeResult { + lsp_types::InitializeResult { + capabilities: server_capabilities(), + server_info: Some(lsp_types::ServerInfo { + name: "baml-lsp".to_string(), + version: Some(baml_version::CANONICAL_VERSION.to_string()), + }), + } +} + impl BexLspRequest for BexMulitProject { fn request_sender( &self, @@ -145,13 +155,7 @@ impl BexLspRequest for BexMulitProject { *self.workspace_roots.lock().unwrap() = roots; - Ok(lsp_types::InitializeResult { - capabilities: server_capabilities(), - server_info: Some(lsp_types::ServerInfo { - name: "baml-lsp".to_string(), - version: Some(baml_version::CANONICAL_VERSION.to_string()), - }), - }) + Ok(initialize_result()) } fn on_request_text_document_code_lens( @@ -894,3 +898,39 @@ impl BexMulitProject { Ok(op(lsp_db, source_file, project, offset)) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn initialize_metadata_matches_protocol_contract() { + let result = initialize_result(); + let experimental = result + .capabilities + .experimental + .expect("initialize result should advertise experimental metadata"); + + assert_eq!( + experimental, + serde_json::json!({ + "baml": { + "toolchainVersion": baml_version::CANONICAL_VERSION, + "lspProtocol": 1, + "minSupportedClientLspProtocol": 1, + "playgroundProtocol": 1, + "minSupportedClientPlaygroundProtocol": 1, + "capabilities": [ + "openPlayground.v1", + "listProjects.v1", + "playgroundWebSocket.v1", + ], + } + }) + ); + assert_eq!( + result.server_info.and_then(|info| info.version), + Some(baml_version::CANONICAL_VERSION.to_string()) + ); + } +} diff --git a/baml_language/crates/bex_project/src/bex_lsp/protocol.rs b/baml_language/crates/bex_project/src/bex_lsp/protocol.rs index f9c1779e467..1eca18adba4 100644 --- a/baml_language/crates/bex_project/src/bex_lsp/protocol.rs +++ b/baml_language/crates/bex_project/src/bex_lsp/protocol.rs @@ -1,3 +1,7 @@ +// Keep these protocol constants in sync with the VS Code extension: +// - typescript2/app-vscode-ext/src/compat.ts mirrors the BAML_* protocol ranges. +// - typescript2/app-vscode-ext/src/extension.ts mirrors CAPABILITIES in initializationOptions. +// TODO: generate both sides from one source if these start changing often. pub(super) const BAML_LSP_PROTOCOL_VERSION: u32 = 1; pub(super) const BAML_PLAYGROUND_PROTOCOL_VERSION: u32 = 1; pub(super) const MIN_SUPPORTED_VSCODE_LSP_PROTOCOL: u32 = 1; diff --git a/scripts/baml-language-version b/scripts/baml-language-version index 2c5010d9135..b8c6964ddb8 100755 --- a/scripts/baml-language-version +++ b/scripts/baml-language-version @@ -349,7 +349,12 @@ def stamp(plan: ReleasePlan) -> None: text = re.sub(r'^__version__ = ".*"$', version_line.rstrip(), text, count=1, flags=re.MULTILINE) else: marker = "atexit.register(flush_events)\n" - text = text.replace(marker, marker + "\n" + version_line, 1) + if marker in text: + text = text.replace(marker, marker + "\n" + version_line, 1) + else: + text = text.rstrip() + "\n\n" + version_line + if '__version__ = "' not in text: + fail(f"failed to stamp __version__ in {PY_INIT.relative_to(ROOT)}") PY_INIT.write_text(text, encoding="utf-8") diff --git a/scripts/baml-package-manager-artifacts b/scripts/baml-package-manager-artifacts index 47ea0031581..8ffc2c89689 100755 --- a/scripts/baml-package-manager-artifacts +++ b/scripts/baml-package-manager-artifacts @@ -85,10 +85,10 @@ end ''' -def aur_srcinfo(pkgbuild: str, pkgname: str, version: str, arch_lines: list[str]) -> str: +def aur_srcinfo(pkgname: str, pkgdesc: str, version: str, arch_lines: list[str]) -> str: lines = [ f"pkgbase = {pkgname}", - f"\tpkgdesc = BAML wrapper - toolchain manager and launcher", + f"\tpkgdesc = {pkgdesc}", f"\tpkgver = {version}", "\tpkgrel = 1", "\turl = https://github.com/BoundaryML/baml", @@ -146,8 +146,8 @@ def main() -> None: (aur_dir / "baml-bin" / ".SRCINFO").write_text( aur_srcinfo( - aur_bin, "baml-bin", + "BAML wrapper - toolchain manager and launcher (prebuilt binary)", pkgver, [ f"\tsource_x86_64 = https://github.com/BoundaryML/baml/releases/download/baml-wrapper-{args.version}/baml-wrapper-{args.version}-x86_64-unknown-linux-gnu.tar.gz", @@ -159,8 +159,8 @@ def main() -> None: ) (aur_dir / "baml" / ".SRCINFO").write_text( aur_srcinfo( - aur_source, "baml", + "BAML wrapper - toolchain manager and launcher (built from source)", pkgver, [ "\tmakedepends = rust", diff --git a/scripts/baml-release-manifests b/scripts/baml-release-manifests index b1be2935696..cc031d49b79 100755 --- a/scripts/baml-release-manifests +++ b/scripts/baml-release-manifests @@ -75,7 +75,9 @@ def main() -> None: (args.out / "version").mkdir(parents=True, exist_ok=True) released_at = dt.datetime.now(dt.UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - vsix = next(args.vsix_dir.glob("*.vsix")) + vsix = next(args.vsix_dir.glob("*.vsix"), None) + if vsix is None: + raise SystemExit(f"no .vsix files found in --vsix-dir {args.vsix_dir}") toolchain = { "schema": 1, "version": args.version, diff --git a/scripts/baml-wrapper-version b/scripts/baml-wrapper-version index 71206709c41..0d65674f027 100755 --- a/scripts/baml-wrapper-version +++ b/scripts/baml-wrapper-version @@ -48,7 +48,10 @@ def fail(message: str) -> None: def read_version() -> Version: text = WRAPPER_CARGO_TOML.read_text(encoding="utf-8") - match = re.search(r'^version = "([^"]+)"$', text, flags=re.MULTILINE) + section = re.search(r"(?ms)^\[package\]\s*(.*?)(?:^\[|\Z)", text) + if not section: + fail("baml wrapper Cargo.toml must contain a [package] table") + match = re.search(r'^version = "([^"]+)"$', section.group(1), flags=re.MULTILINE) if not match: fail("baml wrapper Cargo.toml must contain literal [package].version") try: diff --git a/scripts/install.sh b/scripts/install.sh index 24a7d2f82cd..b9f5e60f793 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -42,8 +42,11 @@ case "$system:$machine" in *) echo "error: unsupported platform $system $machine" >&2; exit 2 ;; esac -tmp="${TMPDIR:-/tmp}/baml-install.$$" -mkdir -p "$tmp" +tmp="$(mktemp -d "${TMPDIR:-/tmp}/baml-install.XXXXXX")" +if [ -z "$tmp" ] || [ ! -d "$tmp" ]; then + echo "error: failed to create temporary directory" >&2 + exit 3 +fi cleanup() { rm -rf "$tmp"; } trap cleanup EXIT INT TERM @@ -75,11 +78,13 @@ if [ "$actual" != "$sha256" ]; then exit 4 fi +raw_entries="$tmp/entries.raw.txt" entries="$tmp/entries.txt" case "$archive" in - *.zip) unzip -Z1 "$archive" > "$entries" ;; - *) tar -tzf "$archive" > "$entries" ;; + *.zip) unzip -Z1 "$archive" > "$raw_entries" ;; + *) tar -tzf "$archive" > "$raw_entries" ;; esac +sed 's#^\./##' "$raw_entries" > "$entries" if grep -Eq '(^/|(^|/)\.\.(/|$)|baml-cli|baml-pack-host|baml-vscode\.vsix)' "$entries"; then echo "error: wrapper archive layout is not safe" >&2 diff --git a/typescript2/README.md b/typescript2/README.md index 389eb99272a..ba112082bfe 100644 --- a/typescript2/README.md +++ b/typescript2/README.md @@ -82,7 +82,7 @@ What `pnpm vscode:package` does: 4. Builds the VS Code extension host bundle: ```bash - pnpm --filter app-vscode-ext build + pnpm --filter baml-language build ``` 5. Stages runtime assets into the extension package: diff --git a/typescript2/app-vscode-ext/src/extension.ts b/typescript2/app-vscode-ext/src/extension.ts index e36936462f0..d199ed3f2cb 100644 --- a/typescript2/app-vscode-ext/src/extension.ts +++ b/typescript2/app-vscode-ext/src/extension.ts @@ -18,7 +18,7 @@ import { } from './compat'; const clients = new Map(); -let knownProjects: string[] = []; +const knownProjects = new Map(); let currentServerState: 'starting' | 'running' | 'stopped' | 'error' = 'starting'; let statusBarItem: vscode.StatusBarItem | undefined; let extensionContext: vscode.ExtensionContext | undefined; @@ -46,8 +46,9 @@ function buildStatusTooltip(serverState: 'starting' | 'running' | 'stopped' | 'e md.appendMarkdown(`---\n\n`); md.appendMarkdown(`[$(output) Open Logs](command:baml.openLogs)\n\n`); - if (knownProjects.length > 0) { - for (const project of knownProjects) { + const projects = Array.from(new Set(Array.from(knownProjects.values()).flat())).sort(); + if (projects.length > 0) { + for (const project of projects) { const encoded = encodeURIComponent(JSON.stringify(project)); md.appendMarkdown(`[$(play) Open Playground — ${projectLabel(project)}](command:baml.openPlayground?${encoded})\n\n`); } @@ -137,6 +138,14 @@ function findBamlProjectRoot(uri: vscode.Uri): string { return workspaceRoot ?? path.dirname(uri.fsPath); } +function projectKeyForPath(projectPath: string): string { + try { + return findBamlProjectRoot(vscode.Uri.file(projectPath)); + } catch { + return projectPath; + } +} + async function ensureClient(projectRoot: string): Promise { const existing = clients.get(projectRoot); if (existing) { @@ -181,12 +190,12 @@ async function ensureClient(projectRoot: string): Promise { clientOptions, ); clients.set(projectRoot, client); - wireClient(client); + wireClient(projectRoot, client); await client.start(); return client; } -function wireClient(client: LanguageClient) { +function wireClient(projectRoot: string, client: LanguageClient) { if (!extensionContext) { return; } @@ -201,7 +210,7 @@ function wireClient(client: LanguageClient) { validateServerCompatibility(client); break; case State.Stopped: - knownProjects = []; + knownProjects.delete(projectRoot); updateStatusBar('stopped'); break; } @@ -220,7 +229,7 @@ function wireClient(client: LanguageClient) { client.onNotification( 'baml/listProjects', (params: { projects: string[] }) => { - knownProjects = params.projects ?? []; + knownProjects.set(projectRoot, params.projects ?? []); refreshTooltip(); }, ); @@ -310,7 +319,9 @@ export async function activate(context: vscode.ExtensionContext) { // NativePlaygroundSender can decide how to open it (and attach the port). context.subscriptions.push( vscode.commands.registerCommand('baml.openPlayground', async (projectPath?: string) => { - const client = projectPath ? clients.get(projectPath) : activeClient(); + const client = projectPath + ? clients.get(projectKeyForPath(projectPath)) ?? activeClient() + : activeClient(); if (!client || client.state !== State.Running) { vscode.window.showWarningMessage('BAML Language Server is not running.'); return; From 98356e91921095ef4b075ad6b6b16f6f032d0340 Mon Sep 17 00:00:00 2001 From: rossirpaulo Date: Wed, 3 Jun 2026 01:41:44 -0500 Subject: [PATCH 4/7] coderabbit: address workflow review comments --- .github/actions/setup-node/action.yml | 8 -------- .github/workflows/build-nodejs-sdk.reusable.yaml | 12 ++++++++---- .github/workflows/publish-python-pypi.yml | 1 - TASK/MERGE-TO-CANARY.md | 4 ++-- 4 files changed, 10 insertions(+), 15 deletions(-) diff --git a/.github/actions/setup-node/action.yml b/.github/actions/setup-node/action.yml index 3113fa81ecf..ca209e6c822 100644 --- a/.github/actions/setup-node/action.yml +++ b/.github/actions/setup-node/action.yml @@ -6,10 +6,6 @@ inputs: description: 'Node.js version to install' required: false default: '20' - options: - - 20 - - 24 - - latest pnpm-version: description: 'pnpm version to install' required: false @@ -35,10 +31,6 @@ inputs: required: false # Default to v4 for backwards compatibility. default: 'v4' - type: choice - options: - - v4 - - v6 runs: using: 'composite' diff --git a/.github/workflows/build-nodejs-sdk.reusable.yaml b/.github/workflows/build-nodejs-sdk.reusable.yaml index aa9cfd7e6a2..2e2408c742a 100644 --- a/.github/workflows/build-nodejs-sdk.reusable.yaml +++ b/.github/workflows/build-nodejs-sdk.reusable.yaml @@ -44,8 +44,9 @@ jobs: _: - target: aarch64-unknown-linux-gnu host: ubuntu-24.04-arm - # Target glibc 2.17. This is the oldest glibc we can practically support. - # Note that ubuntu-22.04-arm has glibc 2.35, but our users need older glibc. + # Target glibc 2.28 for aarch64 using the available rust-cross image. + # This is newer than the ideal glibc floor; our users still need the oldest + # practical compatible baseline we can support for this target. # See https://github.com/BoundaryML/baml/issues/2736 container: ghcr.io/rust-cross/manylinux_2_28-cross:aarch64 before: | @@ -91,12 +92,15 @@ jobs: persist-credentials: false - name: Stamp release plan - if: ${{ inputs.release_plan_json != '' }} env: RELEASE_PLAN_JSON: ${{ inputs.release_plan_json }} run: | set -euo pipefail - printf '%s\n' "$RELEASE_PLAN_JSON" > release-plan.json + if [[ -n "${RELEASE_PLAN_JSON:-}" ]]; then + printf '%s\n' "$RELEASE_PLAN_JSON" > release-plan.json + else + scripts/baml-language-version plan --channel canary --out release-plan.json + fi scripts/baml-language-version stamp --plan release-plan.json - name: Check GLIBC version diff --git a/.github/workflows/publish-python-pypi.yml b/.github/workflows/publish-python-pypi.yml index 1ce232ab813..0cfe5d83edd 100644 --- a/.github/workflows/publish-python-pypi.yml +++ b/.github/workflows/publish-python-pypi.yml @@ -45,7 +45,6 @@ jobs: runs-on: ubuntu-latest needs: - build-python-sdk - - build-nodejs-sdk steps: - run: echo "::do-nothing::" >/dev/null diff --git a/TASK/MERGE-TO-CANARY.md b/TASK/MERGE-TO-CANARY.md index 825d5f9a7ea..ad7c87f7f3f 100644 --- a/TASK/MERGE-TO-CANARY.md +++ b/TASK/MERGE-TO-CANARY.md @@ -2,9 +2,9 @@ This file records how to merge `paulo/baml-language-release-pipeline` with the newer `canary` branch. It is intentionally written as merge guidance, not as a new product spec. `TASK/TASK.md` remains the source of truth for the target release architecture. -## Current Branch Relationship +## Pre-Merge Branch Snapshot -At the time this was last audited: +This snapshot captured the state before the canary merge was performed. Do not treat these SHAs as current branch metadata; keep them only as historical context for why the merge had the conflict shape described below. - branch: `paulo/baml-language-release-pipeline` - merge base: `1742580cac5049c3e6c6eca8ba48f601a0b9bda1` From 51e11d0e41a543d8df17081b7d267c02471188c7 Mon Sep 17 00:00:00 2001 From: rossirpaulo Date: Wed, 3 Jun 2026 02:20:45 -0500 Subject: [PATCH 5/7] test: ignore diagnostic trailing whitespace --- .../baml_lsp2_actions_tests/src/runner.rs | 36 ++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/baml_language/crates/baml_lsp2_actions_tests/src/runner.rs b/baml_language/crates/baml_lsp2_actions_tests/src/runner.rs index 97b81b9a2a1..7bde5fa90af 100644 --- a/baml_language/crates/baml_lsp2_actions_tests/src/runner.rs +++ b/baml_language/crates/baml_lsp2_actions_tests/src/runner.rs @@ -199,11 +199,21 @@ pub fn run_test(parsed: &ParsedTestFile) -> TestResult { }; // Compare against expectations - let diagnostics_passed = parsed.expected_diagnostics == actual_diagnostics; - let hovers_passed = parsed.expected_hovers == actual_hovers; + let diagnostics_passed = expectation_text_matches( + Some(&parsed.expected_diagnostics), + Some(&actual_diagnostics), + ); + let hovers_passed = + expectation_text_matches(parsed.expected_hovers.as_deref(), actual_hovers.as_deref()); let completions_passed = completion_result.as_ref().map(|r| r.passed).unwrap_or(true); - let inlay_hints_passed = parsed.expected_inlay_hints == actual_inlay_hints; - let semantic_tokens_passed = parsed.expected_semantic_tokens == actual_semantic_tokens; + let inlay_hints_passed = expectation_text_matches( + parsed.expected_inlay_hints.as_deref(), + actual_inlay_hints.as_deref(), + ); + let semantic_tokens_passed = expectation_text_matches( + parsed.expected_semantic_tokens.as_deref(), + actual_semantic_tokens.as_deref(), + ); let passed = diagnostics_passed && hovers_passed @@ -274,6 +284,7 @@ fn format_cursor_hover_results(results: &[CursorHoverResult]) -> String { fn format_as_comment(text: &str) -> String { text.lines() .map(|line| { + let line = line.trim_end(); if line.is_empty() { "//".to_string() } else { @@ -284,6 +295,23 @@ fn format_as_comment(text: &str) -> String { .join("\n") } +fn expectation_text_matches(expected: Option<&str>, actual: Option<&str>) -> bool { + match (expected, actual) { + (Some(expected), Some(actual)) => { + normalize_line_endings(expected) == normalize_line_endings(actual) + } + (None, None) => true, + _ => false, + } +} + +fn normalize_line_endings(text: &str) -> String { + text.lines() + .map(str::trim_end) + .collect::>() + .join("\n") +} + /// Convert a byte offset to a 1-indexed (line, column) pair. fn offset_to_line_col(content: &str, offset: u32) -> (usize, usize) { let offset = offset as usize; From 79128b7d5140887fe50eda6cad867fb899a5d762 Mon Sep 17 00:00:00 2001 From: rossirpaulo Date: Wed, 3 Jun 2026 09:49:14 -0500 Subject: [PATCH 6/7] Fix BAML release pipeline blockers --- .../workflows/build-nodejs-sdk.reusable.yaml | 6 + .../workflows/build-python-sdk.reusable.yaml | 7 ++ .github/workflows/publish-python-pypi.yml | 63 +++++++++++ .github/workflows/release-baml-language.yml | 31 +++++- TASK/OPEN-QUESTIONS.md | 60 +++++----- TASK/docs/toolchain-system.md | 4 +- baml_language/Cargo.lock | 1 + baml_language/crates/baml/Cargo.toml | 3 + baml_language/crates/baml/src/main.rs | 104 ++++++++++++++++-- .../lib/pkg-boundaryml-com-stack.ts | 1 + 10 files changed, 234 insertions(+), 46 deletions(-) diff --git a/.github/workflows/build-nodejs-sdk.reusable.yaml b/.github/workflows/build-nodejs-sdk.reusable.yaml index 2e2408c742a..3ebbdd3d182 100644 --- a/.github/workflows/build-nodejs-sdk.reusable.yaml +++ b/.github/workflows/build-nodejs-sdk.reusable.yaml @@ -17,6 +17,11 @@ on: type: string default: "" required: false + source_sha: + description: "Exact repository commit to build" + type: string + default: "" + required: false push: branches: # Mirror the Python reusable workflow's rehearsal-branch trigger so the @@ -89,6 +94,7 @@ jobs: steps: - uses: actions/checkout@v6 with: + ref: ${{ inputs.source_sha || github.sha }} persist-credentials: false - name: Stamp release plan diff --git a/.github/workflows/build-python-sdk.reusable.yaml b/.github/workflows/build-python-sdk.reusable.yaml index 45a281ce69c..73f1ec57cee 100644 --- a/.github/workflows/build-python-sdk.reusable.yaml +++ b/.github/workflows/build-python-sdk.reusable.yaml @@ -8,6 +8,11 @@ on: type: string default: "" required: false + source_sha: + description: "Exact repository commit to build" + type: string + default: "" + required: false push: branches: - sam/publish-python @@ -74,6 +79,8 @@ jobs: } - uses: actions/checkout@v6 + with: + ref: ${{ inputs.source_sha || github.sha }} - name: Stamp release plan if: ${{ inputs.release_plan_json != '' }} diff --git a/.github/workflows/publish-python-pypi.yml b/.github/workflows/publish-python-pypi.yml index 0cfe5d83edd..d3be60cb4ef 100644 --- a/.github/workflows/publish-python-pypi.yml +++ b/.github/workflows/publish-python-pypi.yml @@ -18,6 +18,11 @@ on: type: string default: "" required: false + source_sha: + description: "Exact repository commit to build" + type: string + default: "" + required: false workflow_dispatch: inputs: publish: @@ -39,6 +44,7 @@ jobs: uses: ./.github/workflows/build-python-sdk.reusable.yaml with: release_plan_json: ${{ inputs.release_plan_json }} + source_sha: ${{ inputs.source_sha }} all-builds: name: Assert all builds passed @@ -72,7 +78,63 @@ jobs: exit 1 fi echo "Found $wheel_count wheels" + - name: Check existing PyPI release + id: pypi + env: + RELEASE_PLAN_JSON: ${{ inputs.release_plan_json }} + run: | + set -euo pipefail + python3 - <<'PY' >> "$GITHUB_OUTPUT" + import json + import os + import re + import sys + import urllib.error + import urllib.request + from pathlib import Path + + plan_json = os.environ.get("RELEASE_PLAN_JSON", "").strip() + if plan_json: + version = json.loads(plan_json)["pypi_version"] + else: + wheels = sorted(Path("dist").glob("baml_core-*.whl")) + if not wheels: + print("::error::No baml_core wheels found in dist", file=sys.stderr) + raise SystemExit(1) + match = re.match(r"^baml_core-(.+?)-", wheels[0].name) + if not match: + print(f"::error::Could not infer PyPI version from {wheels[0].name}", file=sys.stderr) + raise SystemExit(1) + version = match.group(1) + + local_files = sorted(path.name for path in Path("dist").glob("*.whl")) + url = f"https://pypi.org/pypi/baml-core/{version}/json" + try: + with urllib.request.urlopen(url, timeout=20) as response: + data = json.load(response) + except urllib.error.HTTPError as exc: + if exc.code == 404: + print(f"pypi_version={version}") + print("release_exists=false") + print("all_files_exist=false") + print(f"PyPI release {version} does not exist yet", file=sys.stderr) + raise SystemExit(0) + raise + + remote_files = {item["filename"] for item in data.get("urls", [])} + missing = [name for name in local_files if name not in remote_files] + print(f"pypi_version={version}") + print("release_exists=true") + print(f"all_files_exist={'false' if missing else 'true'}") + if missing: + print(f"PyPI release {version} exists but is missing {len(missing)} file(s); publish will use skip-existing.", file=sys.stderr) + for name in missing: + print(f"missing: {name}", file=sys.stderr) + else: + print(f"PyPI release {version} already contains every built wheel; skipping publish.", file=sys.stderr) + PY - name: Publish package to PyPI + if: ${{ steps.pypi.outputs.all_files_exist != 'true' }} # OIDC trusted publisher binding configured at # https://pypi.org/manage/project/baml-core/settings/publishing/ # — workflow filename and environment must match this file. @@ -80,3 +142,4 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 with: packages-dir: dist/ + skip-existing: true diff --git a/.github/workflows/release-baml-language.yml b/.github/workflows/release-baml-language.yml index ac3a02030a5..65f295c4023 100644 --- a/.github/workflows/release-baml-language.yml +++ b/.github/workflows/release-baml-language.yml @@ -58,6 +58,7 @@ jobs: pypi_version: ${{ steps.plan.outputs.pypi_version }} git_tag: ${{ steps.plan.outputs.git_tag }} release_plan_json: ${{ steps.plan.outputs.release_plan_json }} + source_sha: ${{ steps.plan.outputs.source_sha }} dry_run: ${{ steps.plan.outputs.dry_run }} wrapper_version: ${{ steps.wrapper.outputs.wrapper_version }} wrapper_changed: ${{ steps.wrapper.outputs.wrapper_changed }} @@ -93,6 +94,7 @@ jobs: fi fi scripts/baml-language-version plan --channel "$channel" --out release-plan.json + source_sha="$(git rev-parse HEAD)" version="$(jq -r .canonical_version release-plan.json)" pypi_version="$(jq -r .pypi_version release-plan.json)" git_tag="$(jq -r .git_tag release-plan.json)" @@ -104,6 +106,7 @@ jobs: echo "release_plan_json<> "$GITHUB_OUTPUT" @@ -160,6 +163,7 @@ jobs: steps: - uses: actions/checkout@v4 with: + ref: ${{ needs.plan.outputs.source_sha }} persist-credentials: false - uses: actions/download-artifact@v4 with: @@ -207,6 +211,7 @@ jobs: steps: - uses: actions/checkout@v4 with: + ref: ${{ needs.plan.outputs.source_sha }} persist-credentials: false - uses: actions/download-artifact@v4 with: @@ -259,6 +264,7 @@ jobs: steps: - uses: actions/checkout@v4 with: + ref: ${{ needs.plan.outputs.source_sha }} persist-credentials: false - name: Build wrapper run: cargo build --manifest-path baml_language/Cargo.toml --release --bin baml --target "${{ matrix.target }}" @@ -299,6 +305,7 @@ jobs: with: publish: true release_plan_json: ${{ needs.plan.outputs.release_plan_json }} + source_sha: ${{ needs.plan.outputs.source_sha }} permissions: id-token: write contents: read @@ -370,6 +377,7 @@ jobs: steps: - uses: actions/checkout@v4 with: + ref: ${{ needs.plan.outputs.source_sha }} persist-credentials: false - uses: actions/download-artifact@v4 with: @@ -417,20 +425,34 @@ jobs: local source="$1" local dest="$2" local tmp + local err tmp="$(mktemp)" - if aws s3 cp "s3://$PKG_BUCKET/$dest" "$tmp" >/dev/null 2>&1; then + err="$(mktemp)" + if aws s3api head-object --bucket "$PKG_BUCKET" --key "$dest" >/dev/null 2>"$err"; then + if ! aws s3 cp "s3://$PKG_BUCKET/$dest" "$tmp" >/dev/null 2>>"$err"; then + echo "::error::immutable object exists but could not be read: $dest" + cat "$err" + rm -f "$tmp" "$err" + exit 1 + fi if cmp -s "$source" "$tmp"; then echo "immutable object already exists and matches: $dest" else echo "::error::immutable object exists with different content: $dest" + rm -f "$tmp" "$err" exit 1 fi - else + elif grep -Eq '(\(404\)|Not Found|NoSuchKey)' "$err"; then aws s3 cp "$source" "s3://$PKG_BUCKET/$dest" \ --content-type application/json \ --cache-control "public, max-age=86400, immutable" + else + echo "::error::failed to inspect immutable object before upload: $dest" + cat "$err" + rm -f "$tmp" "$err" + exit 1 fi - rm -f "$tmp" + rm -f "$tmp" "$err" } version="${{ needs.plan.outputs.version }}" @@ -475,6 +497,7 @@ jobs: steps: - uses: actions/checkout@v4 with: + ref: ${{ needs.plan.outputs.source_sha }} persist-credentials: false - uses: actions/download-artifact@v4 with: @@ -512,6 +535,7 @@ jobs: steps: - uses: actions/checkout@v4 with: + ref: ${{ needs.plan.outputs.source_sha }} persist-credentials: false - uses: actions/download-artifact@v4 with: @@ -564,6 +588,7 @@ jobs: steps: - uses: actions/checkout@v4 with: + ref: ${{ needs.plan.outputs.source_sha }} persist-credentials: false - uses: actions/download-artifact@v4 with: diff --git a/TASK/OPEN-QUESTIONS.md b/TASK/OPEN-QUESTIONS.md index 9d5400a1ae6..450d4f83f30 100644 --- a/TASK/OPEN-QUESTIONS.md +++ b/TASK/OPEN-QUESTIONS.md @@ -1,6 +1,6 @@ **Highest Priority** -1. **VSIX multi-project/toolchain model is underspecified** - [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:603), [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:1000) +1. **VSIX multi-project/toolchain model is underspecified** + [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:603), [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:1000) The doc says the VSIX launches `baml lsp` and should support projects/toolchains without reinstalling the VSIX, but it does not define whether the extension starts one LSP per workspace folder, one per `baml.toml` project root, or one global LSP. Since the wrapper resolves toolchain at process start from cwd, a single LSP launched from “first workspace folder” cannot safely support multiple open projects with different `[toolchain]` pins. This needs a concrete process model. **Answer:** Start one lazy VS Code `LanguageClient` / LSP process per BAML project root, not one global LSP and not one process per file. A BAML project root is the nearest ancestor of the active/open `.baml` file containing `baml.toml`; if none exists, fall back to the containing VS Code workspace folder; if the file is outside any workspace folder, use the file's directory as an ad-hoc root or run in limited-support mode. @@ -11,8 +11,8 @@ Playground ownership is per LSP process. Each project-root LSP owns its own playground server/port, and `baml.openPlayground` uses the client associated with the active editor's project root. Restart only the affected project-root client when that project's `[toolchain]` changes, the configured `baml` executable path changes, compatibility metadata changes after a wrapper/toolchain update, or that LSP crashes. Restart all clients only for truly global setting changes. -2. **Package-manager bootstrap may be impossible or inconsistent as written** - [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:96), [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:936) +2. **Package-manager bootstrap may be impossible or inconsistent as written** + [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:96), [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:936) It says package-manager artifacts install only the wrapper, but first-time `brew install baml` should leave a working canary toolchain. The doc does not say whether Homebrew post-install runs `baml toolchain install canary`, whether that is acceptable for Homebrew tap policy, and what AUR does since package scripts run as root and cannot safely write a user’s `~/.baml`. This needs per-package-manager mechanics. **Answer:** Package-manager installs must remain wrapper-only in v1. `brew install baml`, AUR `baml`, and AUR `baml-bin` install only the `baml` wrapper and must not download, install, select, or update a BAML language toolchain during package install, reinstall, or upgrade. @@ -43,8 +43,8 @@ This decision depends on point 3: `baml toolchain use ` should be the one-command user setup path. If the selected concrete toolchain is missing, `use` should install it and then mark it active. `install` should download a toolchain without making it active. -3. **`baml toolchain use` vs `install` semantics are still ambiguous** - [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:274), [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:322) +3. **`baml toolchain use` vs `install` semantics are still ambiguous** + [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:274), [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:322) The first-run error suggests `baml toolchain use canary`, but package-manager bootstrap uses both `install` and `use`. Network policy says `use` may fetch channel metadata, but does not clearly say whether `use` also installs the selected toolchain if missing. This is a core UX behavior and should be exact. **Answer:** `baml toolchain use ` is the one-command setup path. It resolves the selector, ensures the selected concrete toolchain is installed, and then records that selector as the user's default. If the concrete toolchain is missing, `use` downloads/verifies/installs it before marking it active. @@ -71,8 +71,8 @@ First-run no-toolchain error should also recommend `baml toolchain use canary` as the primary fix. This command both installs and selects the canary toolchain. -4. **Wrapper state schema is not concrete enough** - [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:322), [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:344) +4. **Wrapper state schema is not concrete enough** + [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:322), [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:344) The text refers to “locally recorded concrete version” for channels, but `config.toml` only shows `[default] channel = "canary"`. Where is `canary -> 0.11.0` stored? In manifest cache? In config? In a separate channel state file? This matters for offline behavior and atomic updates. **Answer:** Use separate files for user intent, wrapper-owned machine state, remote metadata cache, and installed toolchain inventory. Do not make the manifest cache authoritative for normal command resolution. @@ -154,8 +154,8 @@ Remote manifests are only inputs to explicit toolchain-management commands. -5. **Dry-run release testing lacks a wrapper override mechanism** - [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:1156), [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:1172) +5. **Dry-run release testing lacks a wrapper override mechanism** + [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:1156), [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:1172) The doc says dry-run may write to a `dryrun/` prefix and merge criteria include `baml toolchain install ` against a dry-run manifest. But the wrapper only knows `https://pkg.boundaryml.com/manifest/v1/...`. We need an explicit env var or flag like `BAML_MANIFEST_BASE_URL` for tests and mirrors. **Answer:** A dry run means running the release graph as realistically as possible without publishing to production release paths consumed by users. Production users read: @@ -236,8 +236,8 @@ - Cache entries are namespaced by manifest base URL so dry-run manifests cannot poison the production cache. Production may use `manifest-cache/prod/`; overrides use `manifest-cache/override//`. - Channel state written under an override records the manifest base URL/hash in `state.toml`. A later command using a different manifest base must not silently treat that channel state as valid; it should either use a temporary `BAML_HOME` in CI or print a clear diagnostic telling the user to run `baml toolchain use ` under the current source. -6. **Release concurrency is not specified** - [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:178), [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:529) +6. **Release concurrency is not specified** + [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:178), [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:529) Nightly suffix selection reads existing GitHub releases and picks the next letter. Without a workflow concurrency group or locking rule, two successful `canary` runs close together can compute the same letter. The PyPI collision check catches some fallout, but the release graph should prevent the race. **Answer:** GitHub merge queue is the first line of defense: all normal writes to `canary` should go through the queue, and the release workflow should publish only from actual `push` events / successful CI on `refs/heads/canary`, not from PR or `merge_group` branches. However, merge queue is not the release lock. Manual dispatches, retries, admin bypasses, and slow publish jobs can still overlap. @@ -270,8 +270,8 @@ With this rule, merge queue serializes branch state, and the release workflow serializes publishing state. -7. **Rollback is too optimistic** - [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:1190) +7. **Rollback is too optimistic** + [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:1190) “Rollback is a code rollback” restores workflow files, but it does not address already-published mutable pointers like `canary.json`, `nightly.json`, `wrapper.json`, install scripts, or a bad GitHub release. The rollback playbook needs explicit behavior for channel pointers and whether to advance to a fixed version, repoint, or leave immutable versions alone. **Answer:** Rollback is code revert plus mutable pointer repair. `git revert ` restores workflow files, but it does not undo external release state. The rollback playbook must distinguish immutable release artifacts from mutable pointers. @@ -349,8 +349,8 @@ - Short cache TTL on mutable pointers so pointer repair reaches wrappers quickly. **Medium Priority** -8. **Wrapper release version source is vague** - [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:83), [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:963) +8. **Wrapper release version source is vague** + [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:83), [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:963) It mentions wrapper crate version / `BAML_WRAPPER_VERSION` and a wrapper-version-changed check, but not the authoritative file or comparison rule. Is it `baml_language/crates/baml/Cargo.toml`, a constant, or workflow metadata? **Answer:** The authoritative wrapper version is the literal package version in `baml_language/crates/baml/Cargo.toml`, once the wrapper crate exists. Remove `BAML_WRAPPER_VERSION` from the plan; do not introduce an environment-variable version authority for the wrapper. @@ -426,8 +426,8 @@ Language/toolchain version source: baml_language/release.toml + release-plan.json stamping ``` -9. **Direct `baml-cli` warning detection needs a mechanism** - [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:228) +9. **Direct `baml-cli` warning detection needs a mechanism** + [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:228) It says `baml-cli` should warn when invoked outside the wrapper. How does it know? The wrapper likely needs to set an env var such as `BAML_WRAPPER_EXEC=1`, and direct invocation warns when absent. **Answer:** The wrapper sets an advisory environment variable before execing the selected internal toolchain binary: @@ -490,8 +490,8 @@ - Machine-readable stdout tests assert the warning never appears on stdout. - Optional mismatch test: create a temporary project with `baml.toml [toolchain]` selecting a different version and invoke `baml-cli` directly; assert the mismatch warning appears without any network access. -10. **Installer path/profile behavior is too light** - [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:491) +10. **Installer path/profile behavior is too light** + [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:491) Flags are listed, but not which shell profiles are edited, how PATH modification is skipped/detected, whether writes are atomic, how Windows PATH mutation works, or what exit codes mean. This is less architectural, but installers are sharp edges. **Answer:** Curl/PowerShell installers are user-scoped BAML installers. They may install/update the wrapper under `BAML_HOME`, optionally bootstrap a requested toolchain, and optionally add `BAML_HOME/bin` to the user's PATH. They must not use `sudo`, write system directories, mutate Homebrew/AUR/package-manager paths, install IDE extensions automatically, or write outside `BAML_HOME` except for user profile/PATH configuration. @@ -661,8 +661,8 @@ - If checksum or archive validation fails, do not replace the existing wrapper. - Re-running the installer is idempotent: it refreshes the wrapper from `wrapper.json`, repairs PATH/env file entries if requested, and only bootstraps the requested toolchain when `--wrapper-only` is not passed. -11. **`baml ide install` fallback is risky as stated** - [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:970) +11. **`baml ide install` fallback is risky as stated** + [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:970) “Fall back to dropping into the editor’s extensions directory and unzipping” needs exact per-editor directories, extension IDs, version replacement behavior, and whether that is actually supported. I’d either specify it tightly or remove fallback in v1. **Answer:** Remove the manual extension-directory unzip fallback from v1. `baml ide install` installs only through supported editor CLIs. If no supported editor CLI is available, print the VSIX path and the exact manual command the user can run. @@ -727,8 +727,8 @@ - No supported CLI detected prints manual install commands and VSIX path. - No test or implementation path manually unzips into editor extension directories. -12. **Homebrew/AUR publishing details are abstract** - [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:168), [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:936) +12. **Homebrew/AUR publishing details are abstract** + [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:168), [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:936) The desired package shape is clear, but credentials, target repos, dispatch vs direct commit, AUR update mechanism, and source/archive URL conventions are not as concrete as the GitHub/S3/PyPI parts. **Answer:** Homebrew and AUR publishing are wrapper-release-only jobs. They never run for `baml-toolchain` nightly/canary releases, never track BAML language versions, never bootstrap toolchains during package install/upgrade, and never write user-scoped `~/.baml` state. @@ -815,8 +815,8 @@ - AUR install hooks do not run `baml toolchain use` or `baml toolchain install`. - Caveats/install messages include `baml toolchain use canary`. -13. **Manifest target completeness has a subtle policy gap** - [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:358) +13. **Manifest target completeness has a subtle policy gap** + [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:358) It defines required and best-effort targets, but not whether the publish workflow may publish a manifest missing a Tier 1 target due to a failed matrix job. I think Tier 1 failures should block publish, while best-effort failures can be omitted only with an explicit gate. **Answer:** Do not support optional/best-effort targets in v1. If a target is in the supported release matrix, it is required. Any target build, archive-layout, checksum, or smoke failure blocks the entire release. Maintainers can rerun the failed matrix job or rerun the workflow to recover. @@ -848,8 +848,8 @@ This keeps the release model simple: if a BAML version exists, all supported targets for that release exist. **Lower Priority** -14. **Docs phase is intentionally broad** - [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:1106) +14. **Docs phase is intentionally broad** + [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:1106) “New page: install matrix...” is fine for a milestone, but not enough if this is meant to hand directly to a docs implementer. It needs page paths, audience, and required examples. **Answer:** These docs refer to new docs that must be created as part of the feature branch. There is no established public `baml_language` documentation home yet, so v1 should create in-repo implementation/user-flow docs under `TASK/docs/`. Public product-docs migration is a later product/docs decision and should not block the release-infra implementation. @@ -919,8 +919,8 @@ These docs are part of the handoff and release-infra branch. They can be moved or rewritten for the public docs site later. -15. **Marketplace/deprecation story is deferred but product-impacting** - [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:1136) +15. **Marketplace/deprecation story is deferred but product-impacting** + [TASK.md](/Users/rossir/dev/baml-canary/TASK/TASK.md:1136) It is correctly marked as product decision, but if implementation depends on extension ID, Marketplace publisher, or coexistence behavior, that decision may need to move earlier. **Answer:** Marketplace publishing and old-extension deprecation remain deferred to Phase 4, but the new VSIX identity is not deferred. V1 ships a stable, distinct, toolchain-bundled VSIX installed explicitly by `baml ide install`; it is not published to the VS Code Marketplace in v1. diff --git a/TASK/docs/toolchain-system.md b/TASK/docs/toolchain-system.md index f60edc4168d..c57ff536555 100644 --- a/TASK/docs/toolchain-system.md +++ b/TASK/docs/toolchain-system.md @@ -45,13 +45,13 @@ Precedence: Normal commands never hit the network and never auto-install. Toolchain commands are the allowlisted network surface. -`toolchain use` installs if missing and selects the default. `toolchain install` downloads without selecting. `toolchain update` advances channel selectors only. `toolchain list` is local-only. +`toolchain use` installs if missing and selects the default. `toolchain install` downloads without selecting. `toolchain update` advances channel selectors only and is the explicit freshness check. `toolchain list` is local-only. ## Manifests Toolchain and wrapper manifests use schema `1`, HTTPS artifact URLs, lowercase SHA-256 checksums, and exact target sets. `BAML_MANIFEST_BASE_URL` can point wrapper validation at dry-run or mirror manifests and is not persisted. -Manifest cache entries are namespaced by manifest base URL: production uses `manifest-cache/prod`, while overrides use `manifest-cache/override/`. Channel pointers have a 24-hour cache TTL for toolchain-management commands. Normal pass-through commands never read remote metadata; they use `config.toml`, `state.toml`, and installed `VERSION` files only. Channel state records the manifest base URL that produced it, so dry-run or mirror state is not silently reused under production. +Manifest cache entries are namespaced by manifest base URL: production uses `manifest-cache/prod`, while overrides use `manifest-cache/override/`. Channel `toolchain use` has a 24-hour cache TTL so selecting an already-known channel does not block on the network unnecessarily. `toolchain update` bypasses that TTL and checks the remote channel pointer before mutating state. Normal pass-through commands never read remote metadata; they use `config.toml`, `state.toml`, and installed `VERSION` files only. Channel state records the manifest base URL that produced it, so dry-run or mirror state is not silently reused under production. ## IDE And Compatibility diff --git a/baml_language/Cargo.lock b/baml_language/Cargo.lock index 85d3f300da5..a5961846e3d 100644 --- a/baml_language/Cargo.lock +++ b/baml_language/Cargo.lock @@ -650,6 +650,7 @@ dependencies = [ "reqwest", "serde", "serde_json", + "tempfile", "toml 0.8.23", ] diff --git a/baml_language/crates/baml/Cargo.toml b/baml_language/crates/baml/Cargo.toml index 6c1bd633277..3589b5b11ba 100644 --- a/baml_language/crates/baml/Cargo.toml +++ b/baml_language/crates/baml/Cargo.toml @@ -21,6 +21,9 @@ serde = { workspace = true, features = [ "derive" ] } serde_json = { workspace = true } toml = { workspace = true } +[dev-dependencies] +tempfile = { workspace = true } + [[bin]] name = "baml" path = "src/main.rs" diff --git a/baml_language/crates/baml/src/main.rs b/baml_language/crates/baml/src/main.rs index f85a45fb9db..0d1149b8dce 100644 --- a/baml_language/crates/baml/src/main.rs +++ b/baml_language/crates/baml/src/main.rs @@ -82,6 +82,12 @@ struct ResolvedSelector { source: SelectorSource, } +#[derive(Clone, Copy)] +enum FetchPolicy { + CacheAllowed, + ForceRemote, +} + fn main() { let exit = match run() { Ok(code) => code, @@ -406,7 +412,11 @@ fn manifest_base_url(override_url: Option<&str>) -> String { .unwrap_or_else(baml_release::manifest_base_url) } -fn fetch_manifest(selector: &str, override_url: Option<&str>) -> Result { +fn fetch_manifest( + selector: &str, + override_url: Option<&str>, + policy: FetchPolicy, +) -> Result { let base = manifest_base_url(override_url); let url = if is_channel(selector) { format!("{base}/{selector}.json") @@ -420,13 +430,7 @@ fn fetch_manifest(selector: &str, override_url: Option<&str>) -> Result Deserialize<'de>>(text: &str) -> Result { serde_json::from_str(text).context("invalid JSON manifest") } +fn should_use_manifest_cache(selector: &str, cache_path: &Path, policy: FetchPolicy) -> bool { + match policy { + FetchPolicy::ForceRemote => false, + FetchPolicy::CacheAllowed => { + !is_channel(selector) + || cache_path + .metadata() + .and_then(|metadata| metadata.modified()) + .ok() + .and_then(|modified| SystemTime::now().duration_since(modified).ok()) + .is_some_and(|age| age <= CHANNEL_CACHE_TTL) + } + } +} + fn install_toolchain( selector: &str, activate_channel: bool, override_url: Option<&str>, force: bool, ) -> Result<()> { - let manifest = fetch_manifest(selector, override_url)?; + install_toolchain_with_policy( + selector, + activate_channel, + override_url, + force, + FetchPolicy::CacheAllowed, + ) +} + +fn install_toolchain_with_policy( + selector: &str, + activate_channel: bool, + override_url: Option<&str>, + force: bool, + policy: FetchPolicy, +) -> Result<()> { + let manifest = fetch_manifest(selector, override_url, policy)?; let target = baml_release::release_host_target_triple()?; let artifact = manifest.artifact_for_target(target)?.clone(); install_manifest_artifact(&manifest.version, target, artifact, force)?; @@ -530,7 +565,7 @@ fn use_toolchain(selector: &str, override_url: Option<&str>) -> Result<()> { } else { let target = baml_release::release_host_target_triple()?; if !toolchain_cli_path(selector).exists() { - let manifest = fetch_manifest(selector, override_url)?; + let manifest = fetch_manifest(selector, override_url, FetchPolicy::CacheAllowed)?; let artifact = manifest.artifact_for_target(target)?.clone(); install_manifest_artifact(&manifest.version, target, artifact, false)?; } @@ -552,7 +587,19 @@ fn update_toolchain(override_url: Option<&str>) -> Result<()> { ); return Ok(()); } - install_toolchain(&config.default.selector, true, override_url, false) + install_toolchain_with_policy( + &config.default.selector, + true, + override_url, + false, + FetchPolicy::ForceRemote, + ) + .with_context(|| { + format!( + "failed to refresh {} from the remote manifest; the active installed toolchain was left unchanged", + config.default.selector + ) + }) } fn installed_toolchains() -> Vec { @@ -724,3 +771,38 @@ fn is_managed_install(path: &Path) -> bool { || text.starts_with("/usr/bin/") || text.starts_with("/opt/") } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cache_allowed_uses_fresh_channel_cache() { + let tmp = tempfile::NamedTempFile::new().unwrap(); + assert!(should_use_manifest_cache( + "canary", + tmp.path(), + FetchPolicy::CacheAllowed + )); + } + + #[test] + fn force_remote_ignores_fresh_channel_cache() { + let tmp = tempfile::NamedTempFile::new().unwrap(); + assert!(!should_use_manifest_cache( + "nightly", + tmp.path(), + FetchPolicy::ForceRemote + )); + } + + #[test] + fn exact_version_cache_is_immutable_and_allowed() { + let missing = Path::new("/tmp/definitely-missing-baml-manifest.json"); + assert!(should_use_manifest_cache( + "0.11.0", + missing, + FetchPolicy::CacheAllowed + )); + } +} diff --git a/tools/pkg_boundaryml_com/lib/pkg-boundaryml-com-stack.ts b/tools/pkg_boundaryml_com/lib/pkg-boundaryml-com-stack.ts index 28a6d6cc760..f99ca503a51 100644 --- a/tools/pkg_boundaryml_com/lib/pkg-boundaryml-com-stack.ts +++ b/tools/pkg_boundaryml_com/lib/pkg-boundaryml-com-stack.ts @@ -43,6 +43,7 @@ export class PkgBoundarymlComStack extends cdk.Stack { maxSessionDuration: cdk.Duration.hours(1), }); + this.bucket.grantRead(this.githubReleaseRole); this.bucket.grantPut(this.githubReleaseRole); this.bucket.grantPutAcl(this.githubReleaseRole); this.githubReleaseRole.addToPolicy( From 4b0d3ae8056e3cc5e9bd8afc96979da0df3078ef Mon Sep 17 00:00:00 2001 From: rossirpaulo Date: Wed, 3 Jun 2026 10:11:54 -0500 Subject: [PATCH 7/7] Address Python release workflow review --- .../workflows/build-python-sdk.reusable.yaml | 8 ++++-- .github/workflows/publish-python-pypi.yml | 26 +++++++++++++++---- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-python-sdk.reusable.yaml b/.github/workflows/build-python-sdk.reusable.yaml index 73f1ec57cee..1fed0c619b6 100644 --- a/.github/workflows/build-python-sdk.reusable.yaml +++ b/.github/workflows/build-python-sdk.reusable.yaml @@ -81,14 +81,18 @@ jobs: - uses: actions/checkout@v6 with: ref: ${{ inputs.source_sha || github.sha }} + persist-credentials: false - name: Stamp release plan - if: ${{ inputs.release_plan_json != '' }} env: RELEASE_PLAN_JSON: ${{ inputs.release_plan_json }} run: | set -euo pipefail - printf '%s\n' "$RELEASE_PLAN_JSON" > release-plan.json + if [[ -n "${RELEASE_PLAN_JSON:-}" ]]; then + printf '%s\n' "$RELEASE_PLAN_JSON" > release-plan.json + else + scripts/baml-language-version plan --channel canary --out release-plan.json + fi scripts/baml-language-version stamp --plan release-plan.json # NOTE: we deliberately do NOT use ./.github/actions/setup-mise here. diff --git a/.github/workflows/publish-python-pypi.yml b/.github/workflows/publish-python-pypi.yml index d3be60cb4ef..449debe4f22 100644 --- a/.github/workflows/publish-python-pypi.yml +++ b/.github/workflows/publish-python-pypi.yml @@ -94,20 +94,36 @@ jobs: from pathlib import Path plan_json = os.environ.get("RELEASE_PLAN_JSON", "").strip() + wheels = sorted(Path("dist").glob("baml_core-*.whl")) + if not wheels: + print("::error::No baml_core wheels found in dist", file=sys.stderr) + raise SystemExit(1) + if plan_json: version = json.loads(plan_json)["pypi_version"] else: - wheels = sorted(Path("dist").glob("baml_core-*.whl")) - if not wheels: - print("::error::No baml_core wheels found in dist", file=sys.stderr) - raise SystemExit(1) match = re.match(r"^baml_core-(.+?)-", wheels[0].name) if not match: print(f"::error::Could not infer PyPI version from {wheels[0].name}", file=sys.stderr) raise SystemExit(1) version = match.group(1) - local_files = sorted(path.name for path in Path("dist").glob("*.whl")) + local_versions = set() + for path in wheels: + match = re.match(r"^baml_core-(.+?)-", path.name) + if not match: + print(f"::error::Could not infer PyPI version from {path.name}", file=sys.stderr) + raise SystemExit(1) + local_versions.add(match.group(1)) + + if local_versions != {version}: + print( + f"::error::Built wheels do not match PyPI version {version}: {sorted(local_versions)}", + file=sys.stderr, + ) + raise SystemExit(1) + + local_files = sorted(path.name for path in wheels) url = f"https://pypi.org/pypi/baml-core/{version}/json" try: with urllib.request.urlopen(url, timeout=20) as response: