From 4b14ca7c2c825785080ec713b7da6046e05a26e1 Mon Sep 17 00:00:00 2001 From: Ani Date: Wed, 5 Aug 2026 00:50:33 +0530 Subject: [PATCH] feat: Apple Silicon host support + generous runtime startup timeout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apple/Metal hosts: doctor detects vendor=apple via system_profiler+sysctl, budgets 65% of unified memory as the GPU working set (no discrete VRAM), resolves the bare darwin/arm64 runtime (Metal, no backend suffix), and adds 'apple' to the heartbeat gpu_vendor enum (was closed to nvidia/amd/unknown — would have rejected every Apple heartbeat). Verified live on an M4. Runtime startup timeout raised 30s -> 5min: cold-loading a model on Metal (shader compilation + GBs into unified memory) exceeded 30s and the node errored while the model was still loading fine. Crashes still fail fast via process-exit detection. Found on the first real Apple host. --- CHANGELOG.md | 5 + docs/ARCHITECTURE.md | 2 +- docs/DECISIONS.md | 5 + docs/LAUNCH_CHECKLIST.md | 1 + internal/node/hardware/apple_test.go | 236 ++++++++++++++++++ internal/node/hardware/doctor.go | 81 +++++- internal/node/hardware/resources.go | 22 ++ internal/node/hardware/vendor.go | 114 +++++++++ internal/node/models/select_test.go | 30 +++ internal/node/runtime/artifact_select_test.go | 54 ++++ internal/node/runtime/launcher.go | 96 ++++++- internal/node/runtime/launcher_test.go | 185 ++++++++++++++ internal/node/runtime/manifest.go | 7 + .../schemas/node.heartbeat.schema.json | 1 + 14 files changed, 827 insertions(+), 12 deletions(-) create mode 100644 internal/node/hardware/apple_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e0e440..a385f8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## Unreleased +- Raised llama-server startup health timeout to 5 minutes for cold large-model loads while still failing fast when the runtime process exits before readiness. +- Added Apple Silicon host support: `thirdshift doctor` now passes on `darwin/arm64`, detects the GPU via `system_profiler` and unified memory via `sysctl`, and reports vendor `apple`. macOS uses the existing signed `darwin/arm64` llama.cpp build, which has Metal compiled in, so no new runtime artifact was needed. +- Apple Silicon reports a GPU working-set budget of 65% of unified memory in place of discrete VRAM, which feeds the VRAM doctor check and auto model selection. +- Added `apple` to the `gpu_vendor` enum in the `node.heartbeat` schema; without it the coordinator rejected every Apple heartbeat. + - Fixed `thirdshift start` with no `--model` failing on installed binaries: automatic selection listed the catalog directory from disk instead of falling back to the catalog embedded in the binary. - Fixed nodes going permanently offline about an hour after starting: access tokens were fetched once at startup and never renewed, so any reconnect past the token lifetime was rejected with 401 until the process was restarted by hand. The agent now renews proactively near expiry and retries once on a 401. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 0c494ca..d1f2cb5 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -20,7 +20,7 @@ The current repository contains the contracts, database schema, coordinator heal The Milestone 1 node code is split behind interfaces so macOS development can use fakes while Windows-specific host concerns remain build-tagged. - Doctor checks report platform, GPU vendor, VRAM, RAM, disk, and configured outbound HTTPS/WSS reachability. Non-Windows hosts report unsupported host status without crashing. -- GPU vendor detection probes `nvidia-smi` first and falls back to `Win32_VideoController` through PowerShell on Windows, yielding `nvidia`, `amd`, or `unknown`. AMD hosts pass and serve on llama.cpp's Vulkan backend. `AdapterRAM` is a 32-bit field, so VRAM that lands in the wrap band is reported unknown rather than estimated. +- GPU vendor detection probes `nvidia-smi` first, then `system_profiler SPDisplaysDataType` on macOS and `Win32_VideoController` through PowerShell on Windows, yielding `nvidia`, `amd`, `apple`, or `unknown`. AMD hosts pass and serve on llama.cpp's Vulkan backend. `AdapterRAM` is a 32-bit field, so VRAM that lands in the wrap band is reported unknown rather than estimated. - Runtime manifests are signed with Ed25519. The manager verifies the manifest, downloads artifacts to a temporary path, checks SHA-256 and byte length, extracts archives, promotes into a versioned runtime directory, and tracks previous/current installs for rollback. - Artifact keys may carry a compute backend suffix (`windows/amd64/cuda`, `windows/amd64/vulkan`). The node resolves its vendor to a backend and tries the backend key first, falling back to the bare platform key. Each key installs into its own directory, so a CUDA and a Vulkan build of one release never collide. The coordinator records every key's hash for a release and treats runtime-hash validity as set membership, so a Vulkan host is schedulable with no scheduler change. - Model cache downloads by HTTP Range into `.partial`, verifies byte length and SHA-256, atomically promotes to `.gguf`, and applies LRU eviction without removing the active model. diff --git a/docs/DECISIONS.md b/docs/DECISIONS.md index 2012c52..4ebe631 100644 --- a/docs/DECISIONS.md +++ b/docs/DECISIONS.md @@ -117,3 +117,8 @@ | D-109 | The public ticker shows a host once it has earned, or while it is connected | The previous rule — any node seen in the last 24h — kept dead test registrations on the launch page for a day. A machine that registered, never served, and went away is an artifact, not a contributor. Hosts who have earned stay visible while asleep, so a real host's credit does not vanish when they close the lid | | D-110 | The agent renews its own access token: proactively near expiry, and reactively on a 401 | Access tokens live an hour and were fetched once at `thirdshift start`. A node that lost its session after that hour reconnected with a dead token and was rejected forever — a host left running overnight was silently offline by morning, needing a manual restart nobody would know to perform. The agent already holds the node private key, so it can sign its own renewal. Reactive 401 retry covers tokens whose expiry is unknown; proactive renewal avoids the disconnect entirely | | D-111 | Catalog listing falls back to the embedded catalog, not just per-file reads | `ReadCatalogFile` fell back to the embedded catalog but `LoadSelectableManifests` listed the directory directly, so auto model selection failed on every installed binary with "read catalog dir: cannot find the path". The same omission as the protocol schemas in D-0xx: an embedded fallback is only complete when *enumeration* is covered too, not just reads by name | +| D-112 | Apple Silicon Macs are supported hosts, expanding past handoff 5.3 and D-002 alongside the AMD addition | The signed `darwin/arm64` llama.cpp artifact has been the dev and test path since Milestone 1, so this un-gates an already-exercised runtime rather than integrating a new one. Supply remains the binding constraint and Macs are plentiful | +| D-113 | macOS uses the bare `darwin/arm64` artifact key; no `darwin/arm64/metal` key exists | Metal is compiled into the single macOS llama.cpp build, so there is nothing to select between. `BackendForVendor` maps `apple` to an empty backend so darwin never probes a suffixed key that cannot exist, while unknown and AMD vendors keep taking Vulkan on Windows | +| D-114 | Apple Silicon reports a GPU budget of 65 percent of unified memory, not a VRAM figure | Apple Silicon has no discrete VRAM: CPU and GPU share one pool, and Metal's own recommended working set on a 16 GB machine is roughly 70 percent. We sit below that on purpose because a host Mac is somebody's daily-driver laptop, and auto-selection must not pick a model that makes the machine unusable while it serves. On a 16 GB M4 this yields 10649 MB and selects the 7B tier; on 64 GB it yields 42598 MB and reaches the 32B tier. Hosts who want to push further can pin a larger model with `--model` | +| D-115 | `gpu_vendor` gained `apple` in the heartbeat schema enum | The field was additive but its enum was closed to nvidia, amd and unknown. Without this the coordinator's protocol validator rejects every Apple heartbeat and closes the session, so Apple support is impossible without the schema change | +| D-116 | Runtime startup health waits up to 5 minutes but exits fast when the child process dies | Cold GGUF loads can spend minutes compiling GPU shaders and paging several GB into unified memory or VRAM; a long ceiling costs nothing once warm because `/health` returns promptly, while process-exit detection keeps real crashes from waiting out the ceiling | diff --git a/docs/LAUNCH_CHECKLIST.md b/docs/LAUNCH_CHECKLIST.md index 889e897..59e85b7 100644 --- a/docs/LAUNCH_CHECKLIST.md +++ b/docs/LAUNCH_CHECKLIST.md @@ -25,6 +25,7 @@ - [x] Hash verification failure blocks execution. - [x] Scheduler placement is in-process and alpha-scale. - [x] Windows 11 x64 NVIDIA host verified on real hardware (2026-07-31, RTX 3060 Ti). +- [ ] Apple Silicon host verified on real hardware (Metal backend). `thirdshift doctor --json` has been run on an M4 (16 GB) and reports platform pass, vendor apple, and a 10649 MB GPU budget; still unverified end to end: `thirdshift start` auto-selecting and serving a routed request on Metal. - [ ] Windows 11 x64 AMD host verified on real hardware (Vulkan backend). Unverified until a Radeon machine runs `thirdshift doctor` and completes a routed request: vendor detection, Vulkan artifact download, and llama-server startup on Vulkan have only been exercised against fixtures. - [ ] Five stable cafe nodes enrolled. Pending human pilot. - [ ] At least 70% scheduled-hours connected. Pending pilot measurement. diff --git a/internal/node/hardware/apple_test.go b/internal/node/hardware/apple_test.go new file mode 100644 index 0000000..95631b4 --- /dev/null +++ b/internal/node/hardware/apple_test.go @@ -0,0 +1,236 @@ +package hardware + +import ( + "context" + "errors" + "testing" +) + +// Captured verbatim from `system_profiler SPDisplaysDataType -json` on an +// Apple M4 MacBook Pro (10-core GPU, 16 GB unified memory), trimmed to the +// fields this code reads plus enough context to keep the shape realistic. +const spDisplaysM4 = `{ + "SPDisplaysDataType" : [ + { + "_name" : "Apple M4", + "spdisplays_mtlgpufamilysupport" : "spdisplays_metal4", + "spdisplays_vendor" : "sppci_vendor_Apple", + "sppci_bus" : "spdisplays_builtin", + "sppci_cores" : "10", + "sppci_device_type" : "spdisplays_gpu", + "sppci_model" : "Apple M4" + } + ] +}` + +// A larger Apple Silicon machine, for the 32B tier. +const spDisplaysM3Max = `{ + "SPDisplaysDataType" : [ + { + "_name" : "Apple M3 Max", + "spdisplays_vendor" : "sppci_vendor_Apple", + "sppci_bus" : "spdisplays_builtin", + "sppci_cores" : "40", + "sppci_device_type" : "spdisplays_gpu", + "sppci_model" : "Apple M3 Max" + } + ] +}` + +// An Intel Mac with a discrete AMD card: not Apple Silicon. +const spDisplaysIntelAMD = `{ + "SPDisplaysDataType" : [ + { + "_name" : "AMD Radeon Pro 5500M", + "spdisplays_vendor" : "sppci_vendor_amd", + "sppci_device_type" : "spdisplays_gpu", + "sppci_model" : "AMD Radeon Pro 5500M" + } + ] +}` + +const ( + memsize16GB = "17179869184\n" + memsize64GB = "68719476736\n" +) + +type appleRunner struct { + displays string + displaysErr error + memsize string + memsizeErr error + calls []string +} + +func (r *appleRunner) Run(_ context.Context, name string, _ ...string) ([]byte, error) { + r.calls = append(r.calls, name) + switch name { + case "nvidia-smi": + return nil, errors.New("exec: nvidia-smi not found") + case "system_profiler": + return []byte(r.displays), r.displaysErr + case "sysctl": + return []byte(r.memsize), r.memsizeErr + default: + return nil, errors.New("unexpected command " + name) + } +} + +func TestParseAppleDisplaysJSON(t *testing.T) { + gpus, err := ParseAppleDisplaysJSON(spDisplaysM4) + if err != nil { + t.Fatalf("parse: %v", err) + } + if len(gpus) != 1 { + t.Fatalf("gpus = %#v", gpus) + } + if gpus[0].Model != "Apple M4" || gpus[0].Cores != "10" { + t.Fatalf("gpu = %#v", gpus[0]) + } + if _, err := ParseAppleDisplaysJSON(" "); err == nil { + t.Fatal("empty output accepted") + } + if _, err := ParseAppleDisplaysJSON("{}"); err == nil { + t.Fatal("a report with no GPUs was accepted") + } +} + +func TestSelectAppleGPUIgnoresNonApple(t *testing.T) { + apple, err := ParseAppleDisplaysJSON(spDisplaysM4) + if err != nil { + t.Fatalf("parse: %v", err) + } + if gpu, ok := SelectAppleGPU(apple); !ok || gpu.Model != "Apple M4" { + t.Fatalf("apple gpu = %#v ok=%v", gpu, ok) + } + + // An Intel Mac with a discrete AMD card is not an Apple Silicon host. + intel, err := ParseAppleDisplaysJSON(spDisplaysIntelAMD) + if err != nil { + t.Fatalf("parse intel: %v", err) + } + if _, ok := SelectAppleGPU(intel); ok { + t.Fatal("a discrete AMD card on an Intel Mac was treated as Apple Silicon") + } +} + +func TestParseMemsize(t *testing.T) { + got, err := ParseMemsize(memsize16GB) + if err != nil || got != 17179869184 { + t.Fatalf("memsize = (%d, %v)", got, err) + } + for _, bad := range []string{"", " ", "not-a-number", "0", "-1"} { + if _, err := ParseMemsize(bad); err == nil { + t.Fatalf("memsize %q was accepted", bad) + } + } +} + +// The budget is a share of one shared pool, never the whole thing: the machine +// still has to run macOS while it serves. +func TestAppleGPUMemoryBudget(t *testing.T) { + for name, tc := range map[string]struct { + totalBytes int64 + wantMB int64 + }{ + "16GB M4": {17179869184, 10649}, + "64GB": {68719476736, 42598}, + "8GB": {8589934592, 5324}, + "unknown": {0, VRAMUnknown}, + "nonsense": {-5, VRAMUnknown}, + } { + t.Run(name, func(t *testing.T) { + if got := AppleGPUMemoryBudgetMB(tc.totalBytes); got != tc.wantMB { + t.Fatalf("budget = %d MB, want %d MB", got, tc.wantMB) + } + }) + } + + // Never hand the GPU the entire pool. + total := int64(17179869184) + if AppleGPUMemoryBudgetMB(total) >= total/1024/1024 { + t.Fatal("budget must be strictly less than total unified memory") + } +} + +func TestDetectGPUVendorOnDarwin(t *testing.T) { + t.Run("apple silicon", func(t *testing.T) { + runner := &appleRunner{displays: spDisplaysM4, memsize: memsize16GB} + if got := DetectGPUVendor(context.Background(), runner, "darwin"); got != VendorApple { + t.Fatalf("vendor = %q, want apple", got) + } + }) + + t.Run("intel mac with amd card is not apple", func(t *testing.T) { + runner := &appleRunner{displays: spDisplaysIntelAMD} + if got := DetectGPUVendor(context.Background(), runner, "darwin"); got != VendorUnknown { + t.Fatalf("vendor = %q, want unknown", got) + } + }) + + t.Run("system_profiler failure is unknown, not a crash", func(t *testing.T) { + runner := &appleRunner{displaysErr: errors.New("not found")} + if got := DetectGPUVendor(context.Background(), runner, "darwin"); got != VendorUnknown { + t.Fatalf("vendor = %q, want unknown", got) + } + }) + + t.Run("darwin never probes windows WMI", func(t *testing.T) { + runner := &appleRunner{displays: spDisplaysM4, memsize: memsize16GB} + DetectGPUVendor(context.Background(), runner, "darwin") + for _, call := range runner.calls { + if call == "powershell" { + t.Fatal("WMI was probed on darwin") + } + } + }) +} + +func TestDetectHostResourcesOnAppleSilicon(t *testing.T) { + for name, tc := range map[string]struct { + memsize string + wantVRAMMB int64 + wantName string + }{ + "16GB M4": {memsize16GB, 10649, "Apple M4"}, + "64GB M3 Max": {memsize64GB, 42598, "Apple M3 Max"}, + } { + t.Run(name, func(t *testing.T) { + displays := spDisplaysM4 + if tc.wantName == "Apple M3 Max" { + displays = spDisplaysM3Max + } + runner := &appleRunner{displays: displays, memsize: tc.memsize} + resources, err := DetectHostResources(context.Background(), runner, ".", "darwin") + if err != nil { + t.Fatalf("detect: %v", err) + } + if resources.GPUVendor != VendorApple { + t.Fatalf("vendor = %q", resources.GPUVendor) + } + if resources.GPUName != tc.wantName { + t.Fatalf("gpu name = %q, want %q", resources.GPUName, tc.wantName) + } + if resources.VRAMTotalMB != tc.wantVRAMMB { + t.Fatalf("gpu budget = %d MB, want %d MB", resources.VRAMTotalMB, tc.wantVRAMMB) + } + if !resources.VRAMKnown() { + t.Fatal("a measured unified-memory budget should count as known") + } + }) + } + + // Unified memory unreadable: vendor still apple, budget unknown, and the + // caller must not be handed a fabricated number. + runner := &appleRunner{displays: spDisplaysM4, memsizeErr: errors.New("boom")} + resources, err := DetectHostResources(context.Background(), runner, ".", "darwin") + if err != nil { + t.Fatalf("detect: %v", err) + } + if resources.GPUVendor != VendorApple { + t.Fatalf("vendor = %q, want apple", resources.GPUVendor) + } + if resources.VRAMKnown() { + t.Fatalf("budget = %d, want unknown when hw.memsize fails", resources.VRAMTotalMB) + } +} diff --git a/internal/node/hardware/doctor.go b/internal/node/hardware/doctor.go index ba82ee6..a68fc86 100644 --- a/internal/node/hardware/doctor.go +++ b/internal/node/hardware/doctor.go @@ -72,11 +72,18 @@ func checkPlatform() CheckResult { Message: "Windows x64 host platform detected.", } } + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" { + return CheckResult{ + Name: "platform", + Status: StatusPass, + Message: "Apple Silicon host platform detected; llama.cpp will use the Metal backend.", + } + } return CheckResult{ Name: "platform", Status: StatusUnsupported, Message: fmt.Sprintf("%s/%s is not a supported host platform for v0.1.", runtime.GOOS, runtime.GOARCH), - Remediation: "Run the host node on Windows 11 x64 with an NVIDIA or AMD GPU. macOS can still run local fake and CPU demo paths.", + Remediation: "Run the host node on Windows 11 x64 with an NVIDIA or AMD GPU, or on an Apple Silicon Mac.", } } @@ -88,6 +95,12 @@ func checkGPU(ctx context.Context, opts DoctorOptions) ([]GPU, string, []CheckRe if len(gpus) > 0 { return gpus, VendorNvidia, checks } + if runtime.GOOS == "darwin" { + if appleGPUs, appleChecks, ok := checkApple(ctx, opts); ok { + return appleGPUs, VendorApple, appleChecks + } + return gpus, VendorUnknown, checks + } if runtime.GOOS != "windows" { return gpus, VendorUnknown, checks } @@ -98,6 +111,72 @@ func checkGPU(ctx context.Context, opts DoctorOptions) ([]GPU, string, []CheckRe return amdGPUs, VendorAMD, amdChecks } +// checkApple reports the Apple Silicon GPU and its share of unified memory. +// There is no discrete VRAM here: the budget is a conservative fraction of the +// one shared pool, so the number is a working-set allowance rather than a +// hardware capacity. +func checkApple(ctx context.Context, opts DoctorOptions) ([]GPU, []CheckResult, bool) { + appleGPUs, err := DetectAppleGPUs(ctx, opts.Runner) + if err != nil { + return nil, nil, false + } + primary, ok := SelectAppleGPU(appleGPUs) + if !ok { + return nil, nil, false + } + name := strings.TrimSpace(firstNonEmptyString(primary.Model, primary.Name)) + + totalBytes, memErr := DetectAppleUnifiedMemoryBytes(ctx, opts.Runner) + budgetMB := VRAMUnknown + if memErr == nil { + budgetMB = AppleGPUMemoryBudgetMB(totalBytes) + } + + gpuCheck := CheckResult{ + Name: "gpu", + Status: StatusPass, + Message: fmt.Sprintf("%s detected; llama.cpp will use the Metal backend.", name), + Details: map[string]any{ + "vendor": VendorApple, + "backend": "metal", + "cores": strings.TrimSpace(primary.Cores), + }, + } + + if budgetMB <= 0 { + return []GPU{{Name: name, VRAMTotalMB: VRAMUnknown}}, []CheckResult{ + gpuCheck, + { + Name: "vram", + Status: StatusWarn, + Message: "Unified memory could not be read, so the GPU working-set budget is unknown.", + Remediation: "Confirm `sysctl -n hw.memsize` works on this host.", + Details: map[string]any{"vendor": VendorApple}, + }, + }, true + } + + vram := CheckResult{ + Name: "vram", + Status: StatusPass, + Message: fmt.Sprintf("Apple Silicon shares %0.1f GB unified memory; %0.1f GB (%d%%) is budgeted for the GPU.", + bytesToGiB(uint64(totalBytes)), float64(budgetMB)/1024, AppleGPUMemoryBudgetPercent), + Details: map[string]any{ + "vendor": VendorApple, + "unified_memory_gb": bytesToGiB(uint64(totalBytes)), + "gpu_budget_gb": float64(budgetMB) / 1024, + "budget_percent": AppleGPUMemoryBudgetPercent, + "minimum_gb": bytesToGiB(MinimumVRAMBytes), + }, + } + if uint64(budgetMB)*1024*1024 < MinimumVRAMBytes { + vram.Status = StatusFail + vram.Remediation = "Use an Apple Silicon Mac with more unified memory; the GPU budget is below the 8 GB floor." + } + + return []GPU{{Name: name, VRAMTotalMB: budgetMB}}, []CheckResult{gpuCheck, vram}, true +} + // checkAMD reports whether an AMD adapter was found, and the checks to use in // place of the NVIDIA failure pair when one was. func checkAMD(ctx context.Context, opts DoctorOptions) ([]GPU, []CheckResult, bool) { diff --git a/internal/node/hardware/resources.go b/internal/node/hardware/resources.go index e7afeb8..240a079 100644 --- a/internal/node/hardware/resources.go +++ b/internal/node/hardware/resources.go @@ -3,6 +3,7 @@ package hardware import ( "context" "fmt" + "strings" ) // HostResources is what the node knows about its own capacity, in megabytes. @@ -43,6 +44,18 @@ func DetectHostResources(ctx context.Context, runner CommandRunner, diskPath, go resources.GPUName = gpu.Name } } + } else if goos == "darwin" { + if gpus, err := DetectAppleGPUs(ctx, runner); err == nil { + if gpu, ok := SelectAppleGPU(gpus); ok { + resources.GPUVendor = VendorApple + resources.GPUName = strings.TrimSpace(firstNonEmptyString(gpu.Model, gpu.Name)) + // Unified memory: there is no discrete VRAM, so the GPU budget + // is a conservative share of the one shared pool. + if totalBytes, err := DetectAppleUnifiedMemoryBytes(ctx, runner); err == nil { + resources.VRAMTotalMB = AppleGPUMemoryBudgetMB(totalBytes) + } + } + } } else if goos == "windows" { if controllers, err := DetectWindowsVideoControllers(ctx, runner); err == nil { primary, vendor := SelectPrimaryController(controllers) @@ -68,3 +81,12 @@ func DetectHostResources(ctx context.Context, runner CommandRunner, diskPath, go return resources, nil } + +func firstNonEmptyString(values ...string) string { + for _, value := range values { + if strings.TrimSpace(value) != "" { + return value + } + } + return "" +} diff --git a/internal/node/hardware/vendor.go b/internal/node/hardware/vendor.go index 58c74ea..004e55d 100644 --- a/internal/node/hardware/vendor.go +++ b/internal/node/hardware/vendor.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "strconv" "strings" ) @@ -12,6 +13,7 @@ import ( const ( VendorNvidia = "nvidia" VendorAMD = "amd" + VendorApple = "apple" VendorUnknown = "unknown" ) @@ -162,6 +164,14 @@ func DetectGPUVendor(ctx context.Context, runner CommandRunner, goos string) str if gpus, err := DetectNvidiaGPUs(ctx, runner); err == nil && len(gpus) > 0 { return VendorNvidia } + if goos == "darwin" { + if gpus, err := DetectAppleGPUs(ctx, runner); err == nil { + if _, ok := SelectAppleGPU(gpus); ok { + return VendorApple + } + } + return VendorUnknown + } if goos != "windows" { return VendorUnknown } @@ -172,3 +182,107 @@ func DetectGPUVendor(ctx context.Context, runner CommandRunner, goos string) str _, vendor := SelectPrimaryController(controllers) return vendor } + +// Apple Silicon detection. macOS exposes the GPU through system_profiler and +// unified memory through sysctl; both run behind CommandRunner so this is +// testable on any OS. +var ( + AppleDisplaysArgs = []string{"SPDisplaysDataType", "-json"} + AppleMemsizeArgs = []string{"-n", "hw.memsize"} +) + +// AppleGPU is one entry from SPDisplaysDataType. +type AppleGPU struct { + Name string `json:"_name"` + Model string `json:"sppci_model"` + Cores string `json:"sppci_cores"` + Vendor string `json:"spdisplays_vendor"` + DeviceType string `json:"sppci_device_type"` +} + +type appleDisplaysReport struct { + Displays []AppleGPU `json:"SPDisplaysDataType"` +} + +// DetectAppleGPUs reads the macOS display adapter inventory. +func DetectAppleGPUs(ctx context.Context, runner CommandRunner) ([]AppleGPU, error) { + output, err := runner.Run(ctx, "system_profiler", AppleDisplaysArgs...) + if err != nil { + return nil, fmt.Errorf("query SPDisplaysDataType: %w", err) + } + return ParseAppleDisplaysJSON(string(output)) +} + +func ParseAppleDisplaysJSON(input string) ([]AppleGPU, error) { + trimmed := strings.TrimSpace(input) + if trimmed == "" { + return nil, fmt.Errorf("SPDisplaysDataType returned no output") + } + var report appleDisplaysReport + if err := json.Unmarshal([]byte(trimmed), &report); err != nil { + return nil, fmt.Errorf("parse SPDisplaysDataType: %w", err) + } + if len(report.Displays) == 0 { + return nil, fmt.Errorf("SPDisplaysDataType returned no GPUs") + } + return report.Displays, nil +} + +// IsAppleGPU reports whether an entry is an Apple Silicon integrated GPU. +func IsAppleGPU(gpu AppleGPU) bool { + haystack := strings.ToLower(gpu.Vendor + " " + gpu.Model + " " + gpu.Name) + return strings.Contains(haystack, "apple") +} + +// SelectAppleGPU returns the Apple Silicon GPU from the inventory, if present. +func SelectAppleGPU(gpus []AppleGPU) (AppleGPU, bool) { + for _, gpu := range gpus { + if IsAppleGPU(gpu) { + return gpu, true + } + } + return AppleGPU{}, false +} + +// DetectAppleUnifiedMemoryBytes reads total unified memory. +func DetectAppleUnifiedMemoryBytes(ctx context.Context, runner CommandRunner) (int64, error) { + output, err := runner.Run(ctx, "sysctl", AppleMemsizeArgs...) + if err != nil { + return 0, fmt.Errorf("read hw.memsize: %w", err) + } + return ParseMemsize(string(output)) +} + +func ParseMemsize(input string) (int64, error) { + trimmed := strings.TrimSpace(input) + if trimmed == "" { + return 0, fmt.Errorf("hw.memsize returned no output") + } + parsed, err := strconv.ParseInt(trimmed, 10, 64) + if err != nil { + return 0, fmt.Errorf("hw.memsize %q is not an integer", trimmed) + } + if parsed <= 0 { + return 0, fmt.Errorf("hw.memsize %d is not a plausible memory size", parsed) + } + return parsed, nil +} + +// AppleGPUMemoryBudgetPercent is the share of unified memory treated as usable +// GPU working set. Apple Silicon has no discrete VRAM: the GPU and the OS share +// one pool, and Metal's own recommended working set on a 16 GB machine is +// roughly 70 percent. We sit deliberately below that because a host Mac is +// somebody's daily-driver laptop, not a headless rig — auto-selection must not +// choose a model that makes the machine unusable while it serves. Hosts who +// want to push further can pin a larger model with --model. +const AppleGPUMemoryBudgetPercent = 65 + +// AppleGPUMemoryBudgetMB converts total unified memory into the working-set +// budget that gates model selection. +func AppleGPUMemoryBudgetMB(totalBytes int64) int64 { + if totalBytes <= 0 { + return VRAMUnknown + } + totalMB := totalBytes / 1024 / 1024 + return totalMB * AppleGPUMemoryBudgetPercent / 100 +} diff --git a/internal/node/models/select_test.go b/internal/node/models/select_test.go index cf2e541..f03e0a5 100644 --- a/internal/node/models/select_test.go +++ b/internal/node/models/select_test.go @@ -217,3 +217,33 @@ func TestResolveModelExplicitOverride(t *testing.T) { } }) } + +// Apple Silicon feeds a unified-memory budget in as VRAM, so the same tier +// logic applies. These are the two machines the operator actually has access to. +func TestSelectModelOnAppleSilicon(t *testing.T) { + manifests := catalogForTest(t) + + // 16 GB M4: 65% of unified memory is 10649 MB, which clears the 7B floor + // (8192) but not the 14B floor (11264). + m4 := HostCapacity{VRAMTotalMB: 10649, RAMTotalMB: 16384, DiskFreeMB: 200000} + selection, err := SelectModel(manifests, m4) + if err != nil { + t.Fatalf("select on a 16GB M4: %v", err) + } + if selection.ModelID != "qwen2.5-7b-instruct" { + t.Fatalf("16GB M4 selected %q, want the 7B (reason: %s)", selection.ModelID, selection.Reason) + } + if selection.VRAMAssumed { + t.Fatal("the unified-memory budget is measured, so nothing is assumed") + } + + // 64 GB Mac: 42598 MB budget clears the 32B floor. + big := HostCapacity{VRAMTotalMB: 42598, RAMTotalMB: 65536, DiskFreeMB: 500000} + selection, err = SelectModel(manifests, big) + if err != nil { + t.Fatalf("select on a 64GB Mac: %v", err) + } + if selection.ModelID != "qwen2.5-32b-instruct" { + t.Fatalf("64GB Mac selected %q, want the 32B", selection.ModelID) + } +} diff --git a/internal/node/runtime/artifact_select_test.go b/internal/node/runtime/artifact_select_test.go index 5fdc1c1..4fa5afe 100644 --- a/internal/node/runtime/artifact_select_test.go +++ b/internal/node/runtime/artifact_select_test.go @@ -144,3 +144,57 @@ func TestShippedManifestCarriesVendorArtifacts(t *testing.T) { t.Fatal("the bare windows/amd64 fallback and the cuda artifact must be the same build") } } + +// macOS ships exactly one llama.cpp build and Metal is native to it, so there +// is no darwin backend suffix and none must be invented. +func TestDarwinResolvesTheBarePlatformArtifact(t *testing.T) { + manifest := vendorManifest() + + if keys := ArtifactKeys("darwin/arm64", VendorApple); strings.Join(keys, ",") != "darwin/arm64" { + t.Fatalf("apple keys = %#v, want only the bare platform key", keys) + } + if backend := BackendForVendor(VendorApple); backend != "" { + t.Fatalf("apple backend = %q, want empty so no suffix is appended", backend) + } + + artifact, key, err := manifest.SelectArtifact("darwin/arm64", VendorApple) + if err != nil { + t.Fatalf("select: %v", err) + } + if key != "darwin/arm64" || artifact.URL != "mac" { + t.Fatalf("key = %q url = %q, want the bare darwin artifact", key, artifact.URL) + } + + // The AMD-era fallback must not misfire here: even if a darwin host somehow + // reported an AMD vendor, there is no darwin/arm64/vulkan artifact and it + // has to fall back to the one Mac build rather than failing. + artifact, key, err = manifest.SelectArtifact("darwin/arm64", VendorAMD) + if err != nil { + t.Fatalf("select with amd vendor on darwin: %v", err) + } + if key != "darwin/arm64" || artifact.URL != "mac" { + t.Fatalf("key = %q url = %q, want fallback to the bare darwin artifact", key, artifact.URL) + } +} + +// The shipped manifest must carry the Mac build, and must not have grown a +// bogus metal-suffixed key. +func TestShippedManifestDarwinArtifact(t *testing.T) { + manifest, err := LoadReleaseManifest("../../../models/catalog/llama-cpp-b10182.runtime.json") + if err != nil { + t.Fatalf("load shipped manifest: %v", err) + } + if _, ok := manifest.Artifacts["darwin/arm64"]; !ok { + t.Fatal("shipped manifest has no darwin/arm64 artifact") + } + if _, ok := manifest.Artifacts["darwin/arm64/metal"]; ok { + t.Fatal("shipped manifest invented a darwin/arm64/metal key; there is only one Mac build") + } + artifact, key, err := manifest.SelectArtifact("darwin/arm64", VendorApple) + if err != nil { + t.Fatalf("select from shipped manifest: %v", err) + } + if key != "darwin/arm64" || artifact.ExecutablePath == "" { + t.Fatalf("resolved %q with executable %q", key, artifact.ExecutablePath) + } +} diff --git a/internal/node/runtime/launcher.go b/internal/node/runtime/launcher.go index c1f73df..6cac354 100644 --- a/internal/node/runtime/launcher.go +++ b/internal/node/runtime/launcher.go @@ -14,6 +14,13 @@ import ( "time" ) +const ( + defaultHealthInterval = 10 * time.Second + defaultStartupTimeout = 5 * time.Minute + defaultShutdownTimeout = 5 * time.Second + defaultMaxLogBytes = int64(1 << 20) +) + type LaunchConfig struct { ExecutablePath string ModelPath string @@ -48,6 +55,8 @@ type RunningRuntime struct { stopOnce sync.Once waitMu sync.Mutex waitErr error + + healthClock healthClock } func StartLlamaServer(ctx context.Context, cfg LaunchConfig) (*RunningRuntime, error) { @@ -61,16 +70,16 @@ func StartLlamaServer(ctx context.Context, cfg LaunchConfig) (*RunningRuntime, e return nil, fmt.Errorf("runtime host %q is rejected; llama-server must bind to 127.0.0.1", cfg.Args.Host) } if cfg.HealthInterval == 0 { - cfg.HealthInterval = 10 * time.Second + cfg.HealthInterval = defaultHealthInterval } if cfg.StartupTimeout == 0 { - cfg.StartupTimeout = 30 * time.Second + cfg.StartupTimeout = defaultStartupTimeout } if cfg.ShutdownTimeout == 0 { - cfg.ShutdownTimeout = 5 * time.Second + cfg.ShutdownTimeout = defaultShutdownTimeout } if cfg.MaxLogBytes == 0 { - cfg.MaxLogBytes = 1 << 20 + cfg.MaxLogBytes = defaultMaxLogBytes } if cfg.WorkDir == "" { cfg.WorkDir = filepath.Dir(cfg.ExecutablePath) @@ -146,7 +155,7 @@ func (r *RunningRuntime) Stop(ctx context.Context, timeout time.Duration) error } if timeout == 0 { - timeout = 5 * time.Second + timeout = defaultShutdownTimeout } _ = r.cmd.Process.Signal(os.Interrupt) waitCh := make(chan error, 1) @@ -175,25 +184,64 @@ func (r *RunningRuntime) Stop(ctx context.Context, timeout time.Duration) error } func (r *RunningRuntime) waitForHealth(ctx context.Context, timeout, interval time.Duration) error { - deadline, cancel := context.WithTimeout(ctx, timeout) + clock := r.healthClock + if clock == nil { + clock = realHealthClock{} + } + deadline, cancel := clock.WithTimeout(ctx, timeout) defer cancel() - ticker := time.NewTicker(interval) + ticker := clock.NewTicker(interval) defer ticker.Stop() + go func() { + select { + case <-r.done: + cancel() + case <-deadline.Done(): + } + }() + for { + select { + case <-r.done: + return r.processExitedBeforeHealthErr() + case <-deadline.Done(): + return r.healthDeadlineErr(deadline) + default: + } if ok := r.checkHealth(deadline); ok { return nil } select { case <-r.done: - return fmt.Errorf("llama-server exited before health check passed: %w", r.getWaitErr()) + return r.processExitedBeforeHealthErr() case <-deadline.Done(): - return fmt.Errorf("llama-server health check timed out: %w", deadline.Err()) - case <-ticker.C: + return r.healthDeadlineErr(deadline) + case <-ticker.C(): } } } +func (r *RunningRuntime) processExitedBeforeHealthErr() error { + if err := r.getWaitErr(); err != nil { + return fmt.Errorf("llama-server process exited before health check passed: %w", err) + } + return fmt.Errorf("llama-server process exited before health check passed") +} + +func (r *RunningRuntime) healthDeadlineErr(deadline context.Context) error { + select { + case <-r.done: + return r.processExitedBeforeHealthErr() + default: + } + err := deadline.Err() + if err == nil { + err = context.DeadlineExceeded + } + return fmt.Errorf("llama-server health check timed out: %w", err) +} + func (r *RunningRuntime) setWaitErr(err error) { r.waitMu.Lock() defer r.waitMu.Unlock() @@ -219,6 +267,34 @@ func (r *RunningRuntime) checkHealth(ctx context.Context) bool { return resp.StatusCode >= 200 && resp.StatusCode < 300 } +type healthClock interface { + WithTimeout(context.Context, time.Duration) (context.Context, context.CancelFunc) + NewTicker(time.Duration) healthTicker +} + +type healthTicker interface { + C() <-chan time.Time + Stop() +} + +type realHealthClock struct{} + +func (realHealthClock) WithTimeout(ctx context.Context, timeout time.Duration) (context.Context, context.CancelFunc) { + return context.WithTimeout(ctx, timeout) +} + +func (realHealthClock) NewTicker(interval time.Duration) healthTicker { + return realHealthTicker{Ticker: time.NewTicker(interval)} +} + +type realHealthTicker struct { + *time.Ticker +} + +func (t realHealthTicker) C() <-chan time.Time { + return t.Ticker.C +} + func buildLlamaArgs(cfg LaunchConfig, port int) []string { args := []string{ "--host", cfg.Args.Host, diff --git a/internal/node/runtime/launcher_test.go b/internal/node/runtime/launcher_test.go index 5efb7e9..531ec82 100644 --- a/internal/node/runtime/launcher_test.go +++ b/internal/node/runtime/launcher_test.go @@ -2,9 +2,13 @@ package runtime import ( "context" + "fmt" + "net/http" + "net/http/httptest" "os" "path/filepath" "strings" + "sync" "testing" "time" @@ -15,6 +19,11 @@ func TestMain(m *testing.M) { if os.Getenv("THIRDSHIFT_FAKE_LLAMA_SERVER") == "1" { os.Exit(fakellama.Main(os.Args[1:])) } + if os.Getenv("THIRDSHIFT_FAKE_LLAMA_CRASH_STARTUP") == "1" { + time.Sleep(50 * time.Millisecond) + fmt.Fprintln(os.Stderr, "fake llama-server process crashed during startup") + os.Exit(42) + } os.Exit(m.Run()) } @@ -90,3 +99,179 @@ func TestLauncherHealthShutdownAndLogRedaction(t *testing.T) { t.Fatalf("runtime log does not contain redaction marker: %s", string(logData)) } } + +func TestLauncherDefaultStartupTimeoutAllowsHealthAfterOldDefault(t *testing.T) { + const oldStartupTimeout = 30 * time.Second + if defaultStartupTimeout != 5*time.Minute { + t.Fatalf("default startup timeout = %v, want 5m", defaultStartupTimeout) + } + + clock := &manualHealthClock{} + healthCalls := make(chan time.Duration, 16) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + elapsed := clock.Elapsed() + healthCalls <- elapsed + if elapsed <= oldStartupTimeout { + http.Error(w, "model loading", http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + running := &RunningRuntime{ + BaseURL: server.URL, + done: make(chan struct{}), + healthClock: clock, + } + errCh := make(chan error, 1) + go func() { + errCh <- running.waitForHealth(context.Background(), defaultStartupTimeout, defaultHealthInterval) + }() + + if elapsed := waitForHealthCall(t, healthCalls); elapsed != 0 { + t.Fatalf("first health check at %v, want 0", elapsed) + } + for _, want := range []time.Duration{10 * time.Second, 20 * time.Second, oldStartupTimeout} { + clock.Advance(defaultHealthInterval) + if elapsed := waitForHealthCall(t, healthCalls); elapsed != want { + t.Fatalf("health check at %v, want %v", elapsed, want) + } + } + select { + case err := <-errCh: + t.Fatalf("launcher returned before health passed old 30s default: %v", err) + default: + } + + clock.Advance(defaultHealthInterval) + elapsed := waitForHealthCall(t, healthCalls) + if elapsed <= oldStartupTimeout { + t.Fatalf("successful health check happened at %v, want after %v", elapsed, oldStartupTimeout) + } + if err := waitForHealthResult(t, errCh); err != nil { + t.Fatalf("wait for health: %v", err) + } +} + +func TestLauncherFailsFastWhenProcessExitsDuringStartup(t *testing.T) { + dir := t.TempDir() + modelPath := filepath.Join(dir, "model.gguf") + if err := os.WriteFile(modelPath, []byte("fake model"), 0o644); err != nil { + t.Fatalf("write model: %v", err) + } + + started := time.Now() + _, err := StartLlamaServer(context.Background(), LaunchConfig{ + ExecutablePath: os.Args[0], + ModelPath: modelPath, + WorkDir: dir, + LogDir: dir, + HealthInterval: 20 * time.Millisecond, + ShutdownTimeout: time.Second, + MaxLogBytes: 1024, + Env: []string{"THIRDSHIFT_FAKE_LLAMA_CRASH_STARTUP=1"}, + Args: ApprovedArgs{ + Host: "127.0.0.1", + }, + }) + elapsed := time.Since(started) + if err == nil { + t.Fatal("start fake crashed runtime succeeded, want error") + } + if elapsed > 2*time.Second { + t.Fatalf("crashed process failure took %v, want under 2s", elapsed) + } + errText := err.Error() + if !strings.Contains(errText, "process") || !strings.Contains(errText, "exit status") { + t.Fatalf("error = %q, want process exit error", errText) + } + if strings.Contains(errText, "timed out") || strings.Contains(errText, "deadline") { + t.Fatalf("error = %q, want process exit instead of deadline", errText) + } +} + +func waitForHealthCall(t *testing.T, calls <-chan time.Duration) time.Duration { + t.Helper() + select { + case elapsed := <-calls: + return elapsed + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for health call") + return 0 + } +} + +func waitForHealthResult(t *testing.T, errCh <-chan error) error { + t.Helper() + select { + case err := <-errCh: + return err + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for health result") + return nil + } +} + +type manualHealthClock struct { + mu sync.Mutex + elapsed time.Duration + timeout time.Duration + cancel context.CancelFunc + ticker *manualHealthTicker +} + +func (c *manualHealthClock) WithTimeout(ctx context.Context, timeout time.Duration) (context.Context, context.CancelFunc) { + deadline, cancel := context.WithCancel(ctx) + c.mu.Lock() + c.timeout = timeout + c.cancel = cancel + c.mu.Unlock() + return deadline, cancel +} + +func (c *manualHealthClock) NewTicker(time.Duration) healthTicker { + ticker := &manualHealthTicker{ch: make(chan time.Time, 64)} + c.mu.Lock() + c.ticker = ticker + c.mu.Unlock() + return ticker +} + +func (c *manualHealthClock) Advance(delta time.Duration) { + c.mu.Lock() + c.elapsed += delta + elapsed := c.elapsed + timeout := c.timeout + cancel := c.cancel + ticker := c.ticker + c.mu.Unlock() + + if timeout > 0 && elapsed >= timeout && cancel != nil { + cancel() + return + } + if ticker != nil { + ticker.Tick(elapsed) + } +} + +func (c *manualHealthClock) Elapsed() time.Duration { + c.mu.Lock() + defer c.mu.Unlock() + return c.elapsed +} + +type manualHealthTicker struct { + ch chan time.Time +} + +func (t *manualHealthTicker) C() <-chan time.Time { + return t.ch +} + +func (t *manualHealthTicker) Stop() {} + +func (t *manualHealthTicker) Tick(elapsed time.Duration) { + t.ch <- time.Unix(0, int64(elapsed)) +} diff --git a/internal/node/runtime/manifest.go b/internal/node/runtime/manifest.go index 60de92c..fa9f822 100644 --- a/internal/node/runtime/manifest.go +++ b/internal/node/runtime/manifest.go @@ -117,6 +117,7 @@ func DecodePublicKey(encoded string) (ed25519.PublicKey, error) { const ( VendorNvidia = "nvidia" VendorAMD = "amd" + VendorApple = "apple" VendorUnknown = "unknown" ) @@ -139,7 +140,13 @@ func BackendForVendor(vendor string) string { switch strings.ToLower(strings.TrimSpace(vendor)) { case VendorNvidia: return BackendCUDA + case VendorApple: + // macOS ships one llama.cpp build with Metal compiled in, so the bare + // platform key is the Metal build and there is no suffix to select. + return "" default: + // Unknown and AMD both take Vulkan, which runs on every Windows GPU we + // might not have identified. return BackendVulkan } } diff --git a/packages/protocol/schemas/node.heartbeat.schema.json b/packages/protocol/schemas/node.heartbeat.schema.json index 9934648..45c9ae7 100644 --- a/packages/protocol/schemas/node.heartbeat.schema.json +++ b/packages/protocol/schemas/node.heartbeat.schema.json @@ -92,6 +92,7 @@ "enum": [ "nvidia", "amd", + "apple", "unknown" ] },