From b0ad636db07b870a8a63dd7c0992b0f2e2859ba2 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:50:58 +0100 Subject: [PATCH 1/3] Release workflow builds macOS binaries on the rented Mac until 21 November GitHub's free 7 GB macOS runner swaps on the shipped-profile compile and needed close to its 360-minute limit. The rented Mac (12 cores, 16 GB, warm checkouts) is registered as an org runner under the rented-mac label; Ufuk approved release builds there until the rental ends. Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- .github/workflows/release.yml | 26 +++++++++++-------- .../land-release-macos-rented-mac.json | 8 ++++++ scripts/lib/test_workflow_gates.py | 8 +++--- 3 files changed, 28 insertions(+), 14 deletions(-) create mode 100644 benchmarks/aft-search/slice-descriptors/land-release-macos-rented-mac.json diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7604be6293..2ec2ae2e53 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -224,12 +224,14 @@ jobs: # --------------------------------------------------------------------------- build-darwin-arm64: name: Build macOS ARM64 - # The shipped-profile library compile peaks near 12 GB and swaps on this - # free 7 GB runner (0.63.1's ARM64 build ran past a shorter limit twice), - # so the limit leaves only the tag canceller's 10 minutes below GitHub's - # 360-minute maximum. - runs-on: macos-26 - timeout-minutes: 350 + # The macOS release jobs run on CortexKit's rented Mac (12 cores, 16 GB, + # one job at a time, checkouts kept warm between jobs) until the rental + # ends on 21 November 2026; then they move back to `macos-26`. GitHub's + # free 7 GB runner swaps on the shipped-profile library compile, which + # peaks near 12 GB, and needed close to its 360-minute maximum. The rented + # Mac has no Rust or Bun on its PATH, so every job installs its toolchains. + runs-on: [self-hosted, rented-mac] + timeout-minutes: 150 outputs: rust_version: ${{ steps.bundle.outputs.rust_version }} permissions: @@ -306,8 +308,8 @@ jobs: build-darwin-x64: name: Build macOS x64 # Same runner and limit as the ARM64 build; see the note there. - runs-on: macos-26 - timeout-minutes: 350 + runs-on: [self-hosted, rented-mac] + timeout-minutes: 150 env: SCCACHE_GHA_ENABLED: "true" RUSTC_WRAPPER: sccache @@ -590,9 +592,11 @@ jobs: e2e-darwin-arm64: name: Shipped E2E (macOS ARM64) needs: build-darwin-arm64 - runs-on: macos-26 - # v0.62.0/v0.63.3/v0.63.4: 2.4/3.0/3.4 min; 10 min covers cold setup. - timeout-minutes: 10 + # The rented Mac, like the macOS builds; see the note on build-darwin-arm64. + runs-on: [self-hosted, rented-mac] + # v0.62.0/v0.63.3/v0.63.4: 2.4/3.0/3.4 min on GitHub's runner. 20 min + # covers a first cold toolchain install on the rented Mac. + timeout-minutes: 20 env: CARGO_INCREMENTAL: "0" steps: diff --git a/benchmarks/aft-search/slice-descriptors/land-release-macos-rented-mac.json b/benchmarks/aft-search/slice-descriptors/land-release-macos-rented-mac.json new file mode 100644 index 0000000000..ebd574711e --- /dev/null +++ b/benchmarks/aft-search/slice-descriptors/land-release-macos-rented-mac.json @@ -0,0 +1,8 @@ +{ + "targeted_mechanism": "none", + "kind": "harness", + "slice_class": "non_ranking", + "fixtures": [ + "harness-goldens" + ] +} diff --git a/scripts/lib/test_workflow_gates.py b/scripts/lib/test_workflow_gates.py index df0a6eb7f2..59af060df3 100644 --- a/scripts/lib/test_workflow_gates.py +++ b/scripts/lib/test_workflow_gates.py @@ -741,7 +741,9 @@ def test_libtest_codegen_is_split_without_rebuilding_dependencies(self) -> None: def test_macos_jobs_use_free_github_runners(self) -> None: # AFT's repository is public, where GitHub's macOS runners are free; - # no job may name a paid runner provider. + # no job may name a paid runner provider. The macOS release jobs run + # on CortexKit's rented Mac until its rental ends on 21 November 2026 + # (see the note on build-darwin-arm64 in release.yml). suite = workflow_gates.parse_workflow(str(REPO_ROOT / ".github/workflows/_unit-suite.yml")) release = workflow_gates.parse_workflow(str(REPO_ROOT / ".github/workflows/release.yml")) jobs = suite["jobs"] @@ -749,8 +751,8 @@ def test_macos_jobs_use_free_github_runners(self) -> None: self.assertEqual(producer["name"], "Cargo libtest archive (macOS)") for name in ("unit-macos-libtest-archive", "unit-macos-libtest-shard", "unit-macos-bin-archive", "unit-macos-nextest-archive", "unit-macos-inventory-proof"): self.assertEqual(jobs[name]["runs-on"], "macos-26", name) - for name in ("build-darwin-arm64", "build-darwin-x64"): - self.assertEqual(release["jobs"][name]["runs-on"], "macos-26", name) + for name in ("build-darwin-arm64", "build-darwin-x64", "e2e-darwin-arm64"): + self.assertEqual(release["jobs"][name]["runs-on"], ["self-hosted", "rented-mac"], name) for workflow in (suite, release): self.assertEqual( [name for name, job in workflow["jobs"].items() if "blacksmith" in str(job.get("runs-on", "")).lower()], From 5a2f8c4efcee2b44b40730c27ebeabe8b9ddfe76 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 9 Nov 2026 19:07:13 +0100 Subject: [PATCH 2/3] Shipped macOS E2E runs on a Mac without Homebrew or Node The rented release Mac has neither. The E2E job now sets up Node for the mock model server, and run.sh falls back to a small Perl stand-in for GNU timeout when Homebrew is missing. Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- .github/workflows/release.yml | 5 ++++ tests/macos-e2e/run.sh | 56 +++++++++++++++++++++++++++-------- 2 files changed, 48 insertions(+), 13 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2ec2ae2e53..4cccc96798 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -607,6 +607,11 @@ jobs: - uses: oven-sh/setup-bun@v2 with: bun-version: latest + # The rented Mac has no Node; the E2E suite installs its mock model + # server with npm, into setup-node's own writable prefix. + - uses: actions/setup-node@v5 + with: + node-version: "24" - name: Sync versions from tag if: startsWith(github.ref, 'refs/tags/v') run: node scripts/version-sync.mjs --from-tag diff --git a/tests/macos-e2e/run.sh b/tests/macos-e2e/run.sh index 415d768df9..f17fcb6793 100755 --- a/tests/macos-e2e/run.sh +++ b/tests/macos-e2e/run.sh @@ -67,20 +67,50 @@ echo " Plugin dist: $AFT_PLUGIN_DIST" # `timeout` is GNU coreutils — it ships with Linux but NOT with macOS by # default, and a vanilla macOS GH Actions runner without coreutils returns # exit 127 ("command not found") the moment the harness tries to invoke it. -# Install coreutils via Homebrew (pre-installed on macOS-latest images) and -# put its `gnubin` directory ahead of $PATH so `timeout` resolves to the GNU -# version. We do NOT use `gtimeout` because the harness is shared with Linux -# and Linux has no `gtimeout` — keeping the binary name `timeout` on both -# platforms means the shared script needs zero platform branches. -echo "── Installing GNU coreutils (for \`timeout\`) ──" -brew install coreutils >/dev/null -COREUTILS_GNUBIN="$(brew --prefix coreutils)/libexec/gnubin" -if [ ! -x "$COREUTILS_GNUBIN/timeout" ]; then - echo "Failed to locate GNU timeout under $COREUTILS_GNUBIN" >&2 - exit 2 +# With Homebrew (GitHub's macOS images), install coreutils and put its +# `gnubin` directory ahead of $PATH so `timeout` resolves to the GNU version. +# Without Homebrew (the self-hosted release Mac), put a small Perl stand-in +# named `timeout` on $PATH; it covers the one form the harness uses, +# `timeout --signal=KILL `, and exits 124 on expiry like +# GNU timeout. We do NOT use `gtimeout` because the harness is shared with +# Linux and Linux has no `gtimeout` — keeping the binary name `timeout` on +# both platforms means the shared script needs zero platform branches. +echo "── Providing \`timeout\` ──" +if command -v brew >/dev/null 2>&1; then + brew install coreutils >/dev/null + COREUTILS_GNUBIN="$(brew --prefix coreutils)/libexec/gnubin" + if [ ! -x "$COREUTILS_GNUBIN/timeout" ]; then + echo "Failed to locate GNU timeout under $COREUTILS_GNUBIN" >&2 + exit 2 + fi + export PATH="$COREUTILS_GNUBIN:$PATH" +else + TIMEOUT_SHIM_DIR="$(mktemp -d)" + cat > "$TIMEOUT_SHIM_DIR/timeout" <<'PERL' +#!/usr/bin/perl +# timeout [--signal=SIG] SECONDS COMMAND [ARGS...] +use strict; +use warnings; +my $signal = 'TERM'; +if (@ARGV && $ARGV[0] =~ /^--signal=(\w+)$/) { $signal = $1; shift @ARGV; } +my $secs = shift @ARGV; +die "timeout: invalid duration\n" unless defined $secs && $secs =~ /^(\d+)s?$/; +$secs = $1; +my $pid = fork(); +die "timeout: fork failed: $!\n" unless defined $pid; +if ($pid == 0) { exec { $ARGV[0] } @ARGV or exit 127; } +my $timed_out = 0; +local $SIG{ALRM} = sub { $timed_out = 1; kill $signal, $pid; }; +alarm $secs; +waitpid($pid, 0); +alarm 0; +exit 124 if $timed_out; +exit($? & 127 ? 128 + ($? & 127) : $? >> 8); +PERL + chmod +x "$TIMEOUT_SHIM_DIR/timeout" + export PATH="$TIMEOUT_SHIM_DIR:$PATH" fi -export PATH="$COREUTILS_GNUBIN:$PATH" -echo " GNU timeout: $(command -v timeout)" +echo " timeout: $(command -v timeout)" # ---- Install OpenCode ------------------------------------------------------ # Single source of truth: .github/opencode-version.txt. All three E2E From 44c8a6c92dc02b5361baec94c5bbec01ec6f510c Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:21:51 +0100 Subject: [PATCH 3/3] Keep the macOS end-to-end release job on GitHub's runner On the rented Mac, downloading the 135 MB tested bundle from GitHub's artifact storage took 14 minutes (about 160 KB/s), so the job hit its limit before the suite ran. The two macOS builds stay on the rented Mac. Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- .github/workflows/release.yml | 16 ++++----- scripts/lib/test_workflow_gates.py | 3 +- tests/macos-e2e/run.sh | 56 +++++++----------------------- 3 files changed, 21 insertions(+), 54 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4cccc96798..2b952a976a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -592,11 +592,12 @@ jobs: e2e-darwin-arm64: name: Shipped E2E (macOS ARM64) needs: build-darwin-arm64 - # The rented Mac, like the macOS builds; see the note on build-darwin-arm64. - runs-on: [self-hosted, rented-mac] - # v0.62.0/v0.63.3/v0.63.4: 2.4/3.0/3.4 min on GitHub's runner. 20 min - # covers a first cold toolchain install on the rented Mac. - timeout-minutes: 20 + # Stays on GitHub's runner: the rented Mac downloads the 135 MB tested + # bundle at about 160 KB/s (14 minutes), while the suite itself takes a + # few minutes here. + runs-on: macos-26 + # v0.62.0/v0.63.3/v0.63.4: 2.4/3.0/3.4 min; 10 min covers cold setup. + timeout-minutes: 10 env: CARGO_INCREMENTAL: "0" steps: @@ -607,11 +608,6 @@ jobs: - uses: oven-sh/setup-bun@v2 with: bun-version: latest - # The rented Mac has no Node; the E2E suite installs its mock model - # server with npm, into setup-node's own writable prefix. - - uses: actions/setup-node@v5 - with: - node-version: "24" - name: Sync versions from tag if: startsWith(github.ref, 'refs/tags/v') run: node scripts/version-sync.mjs --from-tag diff --git a/scripts/lib/test_workflow_gates.py b/scripts/lib/test_workflow_gates.py index 59af060df3..afcc024700 100644 --- a/scripts/lib/test_workflow_gates.py +++ b/scripts/lib/test_workflow_gates.py @@ -751,8 +751,9 @@ def test_macos_jobs_use_free_github_runners(self) -> None: self.assertEqual(producer["name"], "Cargo libtest archive (macOS)") for name in ("unit-macos-libtest-archive", "unit-macos-libtest-shard", "unit-macos-bin-archive", "unit-macos-nextest-archive", "unit-macos-inventory-proof"): self.assertEqual(jobs[name]["runs-on"], "macos-26", name) - for name in ("build-darwin-arm64", "build-darwin-x64", "e2e-darwin-arm64"): + for name in ("build-darwin-arm64", "build-darwin-x64"): self.assertEqual(release["jobs"][name]["runs-on"], ["self-hosted", "rented-mac"], name) + self.assertEqual(release["jobs"]["e2e-darwin-arm64"]["runs-on"], "macos-26") for workflow in (suite, release): self.assertEqual( [name for name, job in workflow["jobs"].items() if "blacksmith" in str(job.get("runs-on", "")).lower()], diff --git a/tests/macos-e2e/run.sh b/tests/macos-e2e/run.sh index f17fcb6793..415d768df9 100755 --- a/tests/macos-e2e/run.sh +++ b/tests/macos-e2e/run.sh @@ -67,50 +67,20 @@ echo " Plugin dist: $AFT_PLUGIN_DIST" # `timeout` is GNU coreutils — it ships with Linux but NOT with macOS by # default, and a vanilla macOS GH Actions runner without coreutils returns # exit 127 ("command not found") the moment the harness tries to invoke it. -# With Homebrew (GitHub's macOS images), install coreutils and put its -# `gnubin` directory ahead of $PATH so `timeout` resolves to the GNU version. -# Without Homebrew (the self-hosted release Mac), put a small Perl stand-in -# named `timeout` on $PATH; it covers the one form the harness uses, -# `timeout --signal=KILL `, and exits 124 on expiry like -# GNU timeout. We do NOT use `gtimeout` because the harness is shared with -# Linux and Linux has no `gtimeout` — keeping the binary name `timeout` on -# both platforms means the shared script needs zero platform branches. -echo "── Providing \`timeout\` ──" -if command -v brew >/dev/null 2>&1; then - brew install coreutils >/dev/null - COREUTILS_GNUBIN="$(brew --prefix coreutils)/libexec/gnubin" - if [ ! -x "$COREUTILS_GNUBIN/timeout" ]; then - echo "Failed to locate GNU timeout under $COREUTILS_GNUBIN" >&2 - exit 2 - fi - export PATH="$COREUTILS_GNUBIN:$PATH" -else - TIMEOUT_SHIM_DIR="$(mktemp -d)" - cat > "$TIMEOUT_SHIM_DIR/timeout" <<'PERL' -#!/usr/bin/perl -# timeout [--signal=SIG] SECONDS COMMAND [ARGS...] -use strict; -use warnings; -my $signal = 'TERM'; -if (@ARGV && $ARGV[0] =~ /^--signal=(\w+)$/) { $signal = $1; shift @ARGV; } -my $secs = shift @ARGV; -die "timeout: invalid duration\n" unless defined $secs && $secs =~ /^(\d+)s?$/; -$secs = $1; -my $pid = fork(); -die "timeout: fork failed: $!\n" unless defined $pid; -if ($pid == 0) { exec { $ARGV[0] } @ARGV or exit 127; } -my $timed_out = 0; -local $SIG{ALRM} = sub { $timed_out = 1; kill $signal, $pid; }; -alarm $secs; -waitpid($pid, 0); -alarm 0; -exit 124 if $timed_out; -exit($? & 127 ? 128 + ($? & 127) : $? >> 8); -PERL - chmod +x "$TIMEOUT_SHIM_DIR/timeout" - export PATH="$TIMEOUT_SHIM_DIR:$PATH" +# Install coreutils via Homebrew (pre-installed on macOS-latest images) and +# put its `gnubin` directory ahead of $PATH so `timeout` resolves to the GNU +# version. We do NOT use `gtimeout` because the harness is shared with Linux +# and Linux has no `gtimeout` — keeping the binary name `timeout` on both +# platforms means the shared script needs zero platform branches. +echo "── Installing GNU coreutils (for \`timeout\`) ──" +brew install coreutils >/dev/null +COREUTILS_GNUBIN="$(brew --prefix coreutils)/libexec/gnubin" +if [ ! -x "$COREUTILS_GNUBIN/timeout" ]; then + echo "Failed to locate GNU timeout under $COREUTILS_GNUBIN" >&2 + exit 2 fi -echo " timeout: $(command -v timeout)" +export PATH="$COREUTILS_GNUBIN:$PATH" +echo " GNU timeout: $(command -v timeout)" # ---- Install OpenCode ------------------------------------------------------ # Single source of truth: .github/opencode-version.txt. All three E2E