Build and validate maintained macOS ARM64 preview tester kits

.github/workflows/sqlsim-macos-arm64-private-preview.yml

@@ -1,14 +1,33 @@
-name: sqlsim macOS ARM64 Private Preview
+name: sqlsimtools macOS ARM64 Private Preview
 
 on:
   workflow_dispatch:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - '.github/workflows/sqlsim-macos-arm64-private-preview.yml'
+      - 'sqlsim/**'
+      - 'sqlpysim/**'
+      - 'sqlsimstudio/**'
+      - 'release/**'
+      - '.github/skills/sqlsim/**'
+      - '.github/skills/sqlpysim/**'
+      - '.github/skills/sqlsimstudio/**'
+      - '.github/agents/sqlsimtools-workload-builder.agent.md'
   push:
     branches:
       - main
     paths:
       - '.github/workflows/sqlsim-macos-arm64-private-preview.yml'
       - 'sqlsim/**'
+      - 'sqlpysim/**'
+      - 'sqlsimstudio/**'
       - 'release/**'
+      - '.github/skills/sqlsim/**'
+      - '.github/skills/sqlpysim/**'
+      - '.github/skills/sqlsimstudio/**'
+      - '.github/agents/sqlsimtools-workload-builder.agent.md'
 
 permissions:
   contents: read
@@ -26,33 +45,143 @@ jobs:
           sw_vers
 
       - name: Install build prerequisites
-        run: brew install cmake unixodbc
+        run: brew install cmake unixodbc openssl pandoc coreutils jq
+
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: '3.11'
+
+      - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0
+        with:
+          dotnet-version: '10.0.x'
+
+      - name: Install Python driver
+        working-directory: sqlpysim
+        run: python -m pip install -r requirements.txt
 
       - name: Build and test sqlsim
         working-directory: sqlsim
         run: |
           bash ./build-macos.sh --clean
           bash ./smoke-test-macos.sh --skip-build
 
-      - name: Stage archive
+      - name: Stage sqlsim
         working-directory: sqlsim
         run: |
           bash ./publish-macos.sh --skip-build --release-dir '${{ runner.temp }}/stage/sqlsim'
 
-      - name: Validate staged package
+      - name: Stage sqlpysim
+        working-directory: sqlpysim
+        shell: pwsh
+        run: |
+          ./publish.ps1 -ReleaseDir '${{ runner.temp }}/stage/sqlpysim'
+          if ($LASTEXITCODE -ne 0) { throw 'sqlpysim publishing failed.' }
+
+      - name: Build Studio with both engines
+        working-directory: sqlsimstudio
+        shell: pwsh
+        run: |
+          ./build.ps1 -Runtime osx-arm64
+          if ($LASTEXITCODE -ne 0) { throw 'Studio publishing failed.' }
+
+      - name: Create and extract candidate archives
+        env:
+          CANDIDATE_SHA: ${{ github.sha }}
+          CANDIDATE_RUN: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          mkdir -p "$RUNNER_TEMP/artifacts" "$RUNNER_TEMP/verify"
+          printf 'Commit: %s\nBuild: %s\nPlatform: macOS 15+ ARM64\nValidation: offline only; live SQL testing required\n' \
+            "$CANDIDATE_SHA" "$CANDIDATE_RUN" > "$RUNNER_TEMP/artifacts/BUILD-INFO.txt"
+          for tool in sqlsim sqlpysim sqlsimstudio; do
+            stage="$RUNNER_TEMP/stage/$tool"
+            if [[ "$tool" == "sqlsimstudio" ]]; then
+              stage="$GITHUB_WORKSPACE/sqlsimstudio/build/osx-arm64"
+            fi
+            cp "$RUNNER_TEMP/artifacts/BUILD-INFO.txt" "$stage/BUILD-INFO.txt"
+            archive="$RUNNER_TEMP/artifacts/$tool-macos-arm64-private-preview.tar.gz"
+            tar -C "$stage" -czf "$archive" .
+            mkdir -p "$RUNNER_TEMP/verify/$tool"
+            tar -C "$RUNNER_TEMP/verify/$tool" -xzf "$archive"
+          done
+
+      - name: Validate extracted CLI packages
         shell: pwsh
         run: |
           ./release/validate-package-layout.ps1 `
-            -InstallRoot '${{ runner.temp }}/stage/sqlsim' `
+            -InstallRoot '${{ runner.temp }}/verify/sqlsim' `
             -Tool sqlsim `
             -SqlsimExecutable 'sqlsim/arm64/sqlsim'
+          if ($LASTEXITCODE -ne 0) { throw 'sqlsim archive validation failed.' }
+          ./release/validate-package-layout.ps1 `
+            -InstallRoot '${{ runner.temp }}/verify/sqlpysim' `
+            -Tool sqlpysim
+          if ($LASTEXITCODE -ne 0) { throw 'sqlpysim archive validation failed.' }
+
+      - name: Validate extracted Studio package
+        shell: pwsh
+        run: |
+          $studio = '${{ runner.temp }}/verify/sqlsimstudio'
+          foreach ($path in @(
+            'sqlsimstudio', 'sqlsim/sqlsim', 'sqlpysim/sqlpysim.py',
+            'sqlpysim/requirements.txt', 'LICENSE', 'THIRD-PARTY-NOTICES.md',
+            'sqlsimstudio.md', 'sqlsimstudio.html', 'CONFIGURATION.md',
+            'MACOS-PRIVATE-PREVIEW.md', 'examples',
+            '.github/skills/sqlsimstudio/SKILL.md', 'BUILD-INFO.txt'
+          )) {
+            if (-not (Test-Path -LiteralPath (Join-Path $studio $path))) {
+              throw "Studio archive is missing $path"
+            }
+          }
+          & lipo (Join-Path $studio 'sqlsimstudio') -verify_arch arm64
+          if ($LASTEXITCODE -ne 0) { throw 'Studio is not an ARM64 executable.' }
+          & lipo (Join-Path $studio 'sqlsim/sqlsim') -verify_arch arm64
+          if ($LASTEXITCODE -ne 0) { throw 'Bundled sqlsim is not an ARM64 executable.' }
+          & (Join-Path $studio 'sqlsim/sqlsim') --version
+          if ($LASTEXITCODE -ne 0) { throw 'Bundled sqlsim could not start.' }
+          & python (Join-Path $studio 'sqlpysim/sqlpysim.py') --version
+          if ($LASTEXITCODE -ne 0) { throw 'Bundled sqlpysim could not start.' }
+
+      - name: Assemble self-contained tester kit
+        shell: pwsh
+        run: |
+          ./release/build-macos-tester-kit.ps1 `
+            -SqlsimRoot '${{ runner.temp }}/verify/sqlsim' `
+            -SqlpysimRoot '${{ runner.temp }}/verify/sqlpysim' `
+            -StudioRoot '${{ runner.temp }}/verify/sqlsimstudio' `
+            -OutputDirectory '${{ runner.temp }}/tester-kit'
 
-      - name: Create archive
+      - name: Archive and extract tester kit
         run: |
-          tar -C '${{ runner.temp }}/stage/sqlsim' -czf '${{ runner.temp }}/sqlsim-macos-arm64-private-preview.tar.gz' .
+          tar -C "$RUNNER_TEMP/tester-kit" -czf "$RUNNER_TEMP/artifacts/sqlsimtools-macos-arm64-tester-kit.tar.gz" .
+          mkdir -p "$RUNNER_TEMP/verify/tester-kit"
+          tar -C "$RUNNER_TEMP/verify/tester-kit" -xzf "$RUNNER_TEMP/artifacts/sqlsimtools-macos-arm64-tester-kit.tar.gz"
+
+      - name: Validate extracted tester kit and runner safety gates
+        timeout-minutes: 15
+        shell: pwsh
+        run: |
+          ./scripts/check-suite-parity.ps1 -Quiet
+          if ($LASTEXITCODE -ne 0) { throw 'Shared suite inventories diverged.' }
+          ./release/test-macos-tester-kit.ps1 `
+            -KitRoot '${{ runner.temp }}/verify/tester-kit' `
+            -ValidationDirectory '${{ runner.temp }}/tester-kit-validation'
+          if ($LASTEXITCODE -ne 0) { throw 'Tester kit validation failed.' }
+
+      - name: Checksum all candidate archives
+        run: |
+          cd "$RUNNER_TEMP/artifacts"
+          shasum -a 256 ./*.tar.gz > SHA256SUMS
+          shasum -a 256 -c SHA256SUMS
+
+      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        if: always()
+        with:
+          name: macos-tester-kit-offline-results
+          path: ${{ runner.temp }}/tester-kit-validation/offline-results/
+          if-no-files-found: ignore
 
       - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
         with:
-          name: sqlsim-macos-arm64-private-preview
-          path: ${{ runner.temp }}/sqlsim-macos-arm64-private-preview.tar.gz
+          name: sqlsimtools-macos-arm64-private-preview
+          path: ${{ runner.temp }}/artifacts/
           if-no-files-found: error

BACKLOG.md

@@ -23,6 +23,13 @@ Azure SQL Managed Instance, and SQL database in Microsoft Fabric.
 
 ### Testing and validation
 
+- **Maintain macOS as a tested platform, not just a volunteer preview.**
+  Follow [MACOS-SUPPORT-PLAN.md](MACOS-SUPPORT-PLAN.md): continuously validate
+  extracted packages and the shared tester kit on Apple Silicon; evaluate
+  maintainer-owned live-testing backends before selecting or provisioning one;
+  and complete clean-install, browser, signing/notarization, and release gates
+  before promoting each tool. Tester feedback supplements rather than replaces
+  a repeatable maintainer-owned validation baseline.
 - **Close the sqlpysim workload and live-telemetry gaps.** Agreed as the next
   piece of work, in this order, because the three share plumbing and together
   restore what Studio currently disables for the Python engine:

BUILDING.md

@@ -86,6 +86,48 @@ Studio builds are self-contained and include the matching native sqlsim binary.
 See [sqlsimstudio/README.md](sqlsimstudio/README.md) for the supported platform
 matrix and package contents.
 
+### macOS ARM64 Private Preview candidates
+
+The [macOS preview workflow](.github/workflows/sqlsim-macos-arm64-private-preview.yml)
+builds all three tools on a macOS 15 Apple Silicon runner using their existing
+build and publish scripts. To build an invited-tester candidate without
+creating a tag or public release:
+
+```powershell
+gh workflow run sqlsim-macos-arm64-private-preview.yml --repo microsoft/sqlsimtools --ref main
+```
+
+The `sqlsimtools-macos-arm64-private-preview` Actions artifact contains:
+
+| Archive | Intended use |
+| --- | --- |
+| `sqlsim-macos-arm64-private-preview.tar.gz` | Complete native CLI overlay, including its macOS preview guide |
+| `sqlpysim-macos-arm64-private-preview.tar.gz` | Complete Python source overlay, including its macOS preview guide |
+| `sqlsimstudio-macos-arm64-private-preview.tar.gz` | Self-contained Studio with both engines and its macOS preview guide |
+| `sqlsimtools-macos-arm64-tester-kit.tar.gz` | Self-contained installation of all tools plus shared offline, smoke, and full suites, fixtures, and a JSON-reporting runner |
+
+The artifact also contains `SHA256SUMS` and `BUILD-INFO.txt`; each archive
+includes the same build receipt. Distribute packages from the same successful
+run to the invited cohort. Downloading Actions artifacts requires signing in
+to GitHub, and the artifacts expire according to repository retention settings.
+Private Preview describes support status, not access control: artifacts in
+this public repository are not confidential.
+
+The workflow extracts each archive before validation, runs CLI help/version
+and layout checks, verifies Studio and its native engine are ARM64, and runs
+Studio's offline startup smoke test. This does not validate live SQL
+connectivity, workload execution, Finder launch, signing, or notarization.
+Testers should follow the packaged preview guides and report sanitized results
+in their existing request issues. macOS candidates are not public release assets.
+
+The tester kit extracts into a new folder and does not require a checkout.
+Its [README](release/macos-tester-README.md) documents dependencies, offline
+checks, explicit live/load-test consent, and reports. CI runs the offline tier
+from the extracted kit, not from source-tree paths. Relevant pull requests also
+run the macOS workflow. The long-term live-testing backend is deliberately
+undecided; see [MACOS-SUPPORT-PLAN.md](MACOS-SUPPORT-PLAN.md) for the maintained
+platform plan and promotion gates.
+
 ## Prepare for source testing
 
 After building sqlsim and preparing sqlpysim, run the shared test preflight

MACOS-SUPPORT-PLAN.md

@@ -0,0 +1,133 @@
+# Maintained macOS ARM64 support plan
+
+## Objective and current status
+
+macOS must become a maintained build, packaging, and validation target, not a
+platform supported only by a one-time volunteer campaign. The initial scope is
+Apple Silicon with macOS 15 or later; Intel Macs are not in scope.
+
+All three tools remain **Private Preview**. Building a candidate or collecting
+positive feedback does not promote support status. The maintainers own the
+gates below; volunteers provide additional real-machine, browser, installation,
+and SQL-environment coverage.
+
+The current implementation provides:
+
+- A native Apple Silicon build of sqlsim and a source package of sqlpysim.
+- A self-contained ARM64 Studio with both engines.
+- Extracted-package layout/architecture/entry-point checks.
+- A self-contained tester kit assembled from the same candidate packages and
+  the canonical suites, with explicit live/load-test consent and JSON reports.
+- A macOS workflow for relevant PRs, pushes to main, and manual dispatch.
+- Automated offline execution of the extracted tester kit before upload.
+
+These checks do not establish live database correctness, signing/notarization,
+Finder launch, or browser-specific behavior.
+
+## Stage 1: continuously maintain offline validation
+
+The maintainer acceptance gate for changes affecting macOS is a successful
+[macOS workflow](.github/workflows/sqlsim-macos-arm64-private-preview.yml).
+Do not bypass it because another platform passes.
+
+On every relevant PR and main update:
+
+1. Build native ARM64 sqlsim and Studio with the existing per-tool scripts.
+2. Publish the three packages and extract them into clean directories.
+3. Validate architecture, manifests, licenses, agent guidance, help/version,
+   and startup.
+4. Assemble the tester kit from those exact extracted packages and canonical
+   suites; archive it, extract it again, and run its offline tier.
+5. Check suite syntax, CLI inventory parity, and the runner's refusal of
+   incomplete credentials or unacknowledged live/load testing.
+6. Preserve reports, build receipts, and checksums. Never upload credentials.
+
+Use one maintained Unix CLI suite for Linux and macOS; fix portability in the
+source suite rather than editing generated kits. Keep platform-specific
+assertions explicit and preserve the existing Windows/Bash test inventories.
+Keep Studio's PowerShell suites shared across platforms.
+
+Repository branch-protection settings are not changed by this work. A
+maintainer should decide whether to require the macOS check once its PR-triggered
+workflow has a stable baseline and its check name is available.
+
+## Stage 2: establish maintainer-owned live automation
+
+**The live-testing backend has not been selected or provisioned.** Evaluate
+these options before implementing live CI:
+
+| Option | Advantages | Costs and constraints |
+| --- | --- | --- |
+| GitHub-hosted Apple Silicon + dedicated Azure SQL test database | No Mac hardware ownership; isolated disposable data; direct Azure SQL/authentication coverage | Cloud cost, network/firewall setup, least-privilege identity, token renewal; does not cover every SQL Server/MI/Fabric feature |
+| GitHub-hosted Apple Silicon + existing reachable SQL Server test instance | Reuses SQL Server fixtures; covers engine-specific features | Secure network access from ephemeral runners, isolated database, identity and availability ownership |
+| Maintainer-controlled/self-hosted Apple Silicon runner | Real local installation/browser testing; access to a private test network | Hardware/cloud-Mac cost, patching, runner isolation, credential protection; never execute untrusted fork PR code on a privileged runner |
+
+Record the selected backend, owner, budget, network design, credential
+mechanism, and data-reset policy before enabling it. Do not widen database
+firewalls or expose a local SQL instance merely to make CI pass.
+
+The eventual live workflow must:
+
+- Be opt-in initially and run only trusted, reviewed commits.
+- Use a protected GitHub environment and scoped credentials, preferably a
+  short-lived identity where the selected backend supports it.
+- Run the same packaged tester kit, never a separate CI-only test inventory.
+- Run smoke first, then full suites against disposable test data with explicit
+  concurrency limits and separate approval for scale tests.
+- Fail on test failures and distinguish product defects, harness failures,
+  permission/feature skips, and infrastructure failures.
+- Produce sanitized, commit-linked reports without credential files or raw
+  customer data; clean up test objects and credentials even on cancellation.
+
+Required initial live coverage is SQL authentication and Azure SQL access-token
+authentication for both engines and Studio, on the appropriate selected
+targets. If the first backend cannot exercise both, leave the missing coverage
+open and plan a second target. Kerberos, managed identity, service principal,
+MI, and Fabric must be tracked individually rather than inferred from another
+authentication mode or product.
+
+Until this infrastructure exists, maintainers must explicitly record the live
+validation gap. Volunteer results can inform fixes but are not a substitute
+for repeatable release validation.
+
+## Stage 3: installation, browser, and distribution readiness
+
+Before broader availability:
+
+- Decide and document Developer ID signing and notarization for native
+  executables and Studio; arrange certificate ownership/renewal and protected
+  signing credentials.
+- Validate download/extraction/quarantine behavior on a clean Mac without
+  disabling Gatekeeper.
+- Exercise Studio launch, reconnect, shutdown, both engines, monitoring,
+  cancellation, results, and A/B comparison in real macOS browsers. WebKit
+  emulation on Windows is not Safari-on-macOS evidence.
+- Decide whether to add a `.app` launcher or retain the documented terminal
+  launch experience; package that choice consistently.
+- Document ODBC, Python/OpenSSL, minimum macOS, and architecture requirements.
+- Keep the system-ICU exception for macOS explicit and test culture-sensitive
+  comparisons rather than promising identical collation across platforms.
+- Define reproducible candidate labels, artifact retention, update/rollback,
+  and the eventual versioned release-asset path.
+
+## Promotion and ongoing maintenance
+
+Promotion is an explicit maintainer decision, separately for each tool. Require:
+
+1. Green native build, packaging, and extracted-kit offline gates.
+2. Recorded maintainer-controlled smoke/full live results on the release
+   candidate, with SQL/authentication coverage and skips listed.
+3. Clean-machine installation and actual macOS browser evidence where relevant.
+4. Signing/notarization and distribution decisions completed and documented.
+5. No unresolved installation, credential-handling, data-integrity,
+   cancellation, or stability blockers.
+6. Updated platform matrices, manuals, release notes, skills, and release
+   packaging/validation for the promoted tools.
+
+After promotion, keep macOS on relevant PR/main validation and the release
+pipeline, revalidate driver/.NET/Python changes, and run the selected live
+gate before every release. Evaluate newer macOS/Python versions explicitly;
+do not silently raise the current minimum or imply an untested version works.
+Choose a periodic live/clean-install cadence when the backend and its budget
+are approved. Volunteers remain valuable for diverse environments, but the
+maintainer owns release confidence and the regression baseline.

release/README.md

@@ -31,6 +31,9 @@ as evidence that a release artifact is correct.
 | --- | --- |
 | `publish.ps1` | Runs the sqlsim and sqlpysim publishers for local staging |
 | `build-sqlsimtools-bundle.ps1` | Assembles the Windows sqlsimtools bundle containing Studio and both engines |
+| `build-macos-tester-kit.ps1` | Assembles a self-contained macOS test installation from validated candidate packages and canonical suites |
+| `run-macos-tests.ps1` | Tester-kit runner; defaults to offline checks, gates live/load testing, and writes JSON plus logs |
+| `macos-tester-README.md` | Instructions shipped as the tester kit's README |
 | `validate-package-layout.ps1` | Validates an extracted tool package and its documented entry points |
 | `check-release-version.ps1` | Confirms version parity across every shipped tool |
 | `stage-release-assets.ps1` | Stages the five versioned public release assets; the README links to the latest release page |