Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 39 additions & 4 deletions cmd/spinloop/fleet_dashboard_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ func (f *fakeDashNode) Metrics(ctx context.Context) (metrics.Stats, error) {
s.UptimeSeconds = 60
s.CPU = &metrics.CpuStat{Utilization: 42}
s.GPUs = []metrics.GpuStat{{Index: 0, Name: "H100", Utilization: 10, MemoryUsed: 1, MemoryTotal: 10}}
s.Tokens = &metrics.TokenStats{Running: 1, PromptTokens: 100, GenerationTokens: 50, Requests: 3}
s.Tokens = &metrics.TokenStats{Running: 1, PromptTokens: 100, GenerationTokens: 50, Requests: ptrInt(3)}
}
return s, nil
}
Expand Down Expand Up @@ -349,7 +349,7 @@ func TestDashTileRunningByteStable(t *testing.T) {
CPU: &metrics.CpuStat{Utilization: 42},
Memory: &metrics.MemoryStat{Total: 1000, Used: 300},
GPUs: []metrics.GpuStat{{Index: 0, Name: "H100", Utilization: 61, MemoryUsed: 80, MemoryTotal: 160}},
Tokens: &metrics.TokenStats{Running: 2, PromptTokens: 4096, GenerationTokens: 1024, Requests: 17},
Tokens: &metrics.TokenStats{Running: 2, PromptTokens: 4096, GenerationTokens: 1024, Requests: ptrInt(17)},
},
}
want := dashTileExpected([]string{
Expand All @@ -371,6 +371,41 @@ func TestDashTileRunningByteStable(t *testing.T) {
}
}

// A llamacpp node's statistics carry no request figure, and the tile draws no
// line for it — the block keeps its height, and the space the line would have
// taken stays blank.
func TestDashTileRunningNoRequestCount(t *testing.T) {
lipgloss.SetColorProfile(termenv.Ascii)
r := fleet.NodeResult{
Name: "up", Outcome: fleet.OutcomeOK,
Metrics: metrics.Stats{
State: "running", Runner: "llamacpp", ModelID: "org/qwen:q4",
UptimeSeconds: 7200, LastActiveAt: "2026-08-21T10:00:00Z", IdleSeconds: 12,
CPU: &metrics.CpuStat{Utilization: 42},
Memory: &metrics.MemoryStat{Total: 1000, Used: 300},
GPUs: []metrics.GpuStat{{Index: 0, Name: "H100", Utilization: 61, MemoryUsed: 80, MemoryTotal: 160}},
Tokens: &metrics.TokenStats{Running: 2, PromptTokens: 4096, GenerationTokens: 1024},
},
}
want := dashTileExpected([]string{
dashExpectedHeader("up running (up 2h 0m 0s)", dashHealthy),
"llamacpp org/qwen:q4",
" active 12s ago",
dashBar("CPU", 42),
dashBar("RAM", 30),
dashBar("GPU util", 61),
dashBar("GPU mem", 50),
"",
" running: 2",
" prompt tokens: 4096",
" generation tokens: 1024",
"",
})
if got := dashTestTile("up", r, false, dashAction{}); got != want {
t.Errorf("tile mismatch:\ngot:\n%q\nwant:\n%q", got, want)
}
}

func TestDashTileOutcomeAndEmpty(t *testing.T) {
lipgloss.SetColorProfile(termenv.Ascii)
dead := fleet.NodeResult{
Expand Down Expand Up @@ -896,7 +931,7 @@ func TestDashTileTruncatesTallContent(t *testing.T) {
CPU: &metrics.CpuStat{Utilization: 42},
Memory: &metrics.MemoryStat{Total: 1000, Used: 300},
GPUs: gpus,
Tokens: &metrics.TokenStats{Running: 1, PromptTokens: 100, GenerationTokens: 50, Requests: 3},
Tokens: &metrics.TokenStats{Running: 1, PromptTokens: 100, GenerationTokens: 50, Requests: ptrInt(3)},
},
}
lines := strings.Split(dashTestTile("many", r, false, dashAction{}), "\n")
Expand Down Expand Up @@ -3370,7 +3405,7 @@ func dashHistoryNode() fleet.NodeResult {
CPU: &metrics.CpuStat{Utilization: 42},
Memory: &metrics.MemoryStat{Total: 1000, Used: 300},
GPUs: []metrics.GpuStat{{Index: 0, Name: "H100", Utilization: 61, MemoryUsed: 80, MemoryTotal: 160}},
Tokens: &metrics.TokenStats{Running: 2, PromptTokens: 4096, GenerationTokens: 1024, Requests: 17},
Tokens: &metrics.TokenStats{Running: 2, PromptTokens: 4096, GenerationTokens: 1024, Requests: ptrInt(17)},
History: []metrics.HistorySample{
{Time: 1786276800, CPU: ptrPct(10), Mem: ptrPct(20), GPUs: []metrics.HistoryGPU{{Index: 0, Util: 50, Mem: ptrPct(40)}}},
{Time: 1786276815, CPU: ptrPct(20), Mem: ptrPct(30), GPUs: []metrics.HistoryGPU{{Index: 0, Util: 61, Mem: ptrPct(50)}}},
Expand Down
9 changes: 7 additions & 2 deletions cmd/spinloop/metrics_render.go
Original file line number Diff line number Diff line change
Expand Up @@ -428,7 +428,10 @@ func renderStatGauges(w io.Writer, cpu *metrics.CpuStat, mem *metrics.MemoryStat
}

// renderTokenLines draws the engine's token and request counters, the block
// both formats share.
// both formats share. Each line is drawn only for a figure the statistics
// carry: the request count is absent, not zero, for an engine family whose
// metrics expose no cumulative request counter, so the line is that
// figure's to omit, not the renderer's to keep drawing.
func renderTokenLines(w io.Writer, tokens *metrics.TokenStats) {
if tokens == nil {
return
Expand All @@ -437,7 +440,9 @@ func renderTokenLines(w io.Writer, tokens *metrics.TokenStats) {
fmt.Fprintf(w, " running: %d\n", tokens.Running)
fmt.Fprintf(w, " prompt tokens: %d\n", tokens.PromptTokens)
fmt.Fprintf(w, " generation tokens: %d\n", tokens.GenerationTokens)
fmt.Fprintf(w, " requests: %d\n", tokens.Requests)
if tokens.Requests != nil {
fmt.Fprintf(w, " requests: %d\n", *tokens.Requests)
}
}

// renderGPUTable draws the per-GPU lines of the table format, plus the
Expand Down
25 changes: 24 additions & 1 deletion cmd/spinloop/metrics_render_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ import (

func ptrPct(v float64) *float64 { return &v }

func ptrInt(v int) *int { return &v }

func TestBarGlyph(t *testing.T) {
cases := []struct {
pct float64
Expand Down Expand Up @@ -485,7 +487,7 @@ func TestFormatMetricsBarRunning(t *testing.T) {
CPU: &metrics.CpuStat{Utilization: 62},
Memory: &metrics.MemoryStat{Total: 1000, Used: 300},
GPUs: []metrics.GpuStat{{Index: 0, Name: "H100", Utilization: 61, MemoryUsed: 80, MemoryTotal: 160}},
Tokens: &remote.TokenStats{Running: 2, PromptTokens: 4096, GenerationTokens: 1024, Requests: 17},
Tokens: &remote.TokenStats{Running: 2, PromptTokens: 4096, GenerationTokens: 1024, Requests: ptrInt(17)},
History: []metrics.HistorySample{
{Time: 1, CPU: ptrPct(10), Mem: ptrPct(20), GPUs: []metrics.HistoryGPU{{Index: 0, Util: 50, Mem: ptrPct(50)}}},
{Time: 2, CPU: ptrPct(20), Mem: ptrPct(30), GPUs: []metrics.HistoryGPU{{Index: 0, Util: 61, Mem: ptrPct(50)}}},
Expand Down Expand Up @@ -516,6 +518,27 @@ func TestFormatMetricsBarRunning(t *testing.T) {
}
}

// An engine family whose metrics expose no cumulative request counter yields
// statistics without the figure, and the token block draws no line for it.
func TestFormatMetricsBarNoRequestCount(t *testing.T) {
resp := &remote.StatsResponse{
Environment: "prod", State: "running", InstanceType: "g5.xlarge",
ModelID: "org/qwen:q4", Version: "0.4.3",
Tokens: &remote.TokenStats{Running: 2, PromptTokens: 4096, GenerationTokens: 1024},
}
var b bytes.Buffer
if err := renderFleetMetrics(&b, nodeResultsFor(resp), "bar"); err != nil {
t.Fatal(err)
}
got := b.String()
if !strings.Contains(got, " running: 2\n") {
t.Errorf("running line missing: %q", got)
}
if strings.Contains(got, " requests:") {
t.Errorf("requests line drawn for an engine that exposes no request count: %q", got)
}
}

func TestFormatMetricsJSONCarriesHistory(t *testing.T) {
resp := &remote.StatsResponse{
Environment: "prod", State: "running",
Expand Down
32 changes: 31 additions & 1 deletion cmd/spinloop/serve_view_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -433,7 +433,7 @@ func TestServeViewFrame(t *testing.T) {
CPU: &metrics.CpuStat{Utilization: 42},
Memory: &metrics.MemoryStat{Total: 1000, Used: 500},
History: []metrics.HistorySample{{Time: 1, CPU: f64ptr(10), Mem: f64ptr(40)}},
Tokens: &metrics.TokenStats{PromptTokens: 10, GenerationTokens: 5, Requests: 2},
Tokens: &metrics.TokenStats{PromptTokens: 10, GenerationTokens: 5, Requests: ptrInt(2)},
}
m := newTestServeView(
func() (metrics.Stats, error) { return stats, nil },
Expand Down Expand Up @@ -482,6 +482,36 @@ func TestServeViewFrame(t *testing.T) {
}
}

// Statistics without a request figure draw no requests line: the view shows
// the figures the engine exposes and nothing it does not.
func TestServeViewNoRequestCount(t *testing.T) {
fixDashNow(t, time.Date(2026, 9, 6, 12, 0, 0, 0, time.UTC))
stats := metrics.Stats{
State: "running",
UptimeSeconds: 90,
Runner: "llama.cpp",
ModelID: "org/model",
CPU: &metrics.CpuStat{Utilization: 42},
Tokens: &metrics.TokenStats{PromptTokens: 10, GenerationTokens: 5},
}
m := newTestServeView(
func() (metrics.Stats, error) { return stats, nil },
func(offset int64, limit int) (daemon.LogsResponse, error) {
return daemon.LogsResponse{Content: "alpha\n", NextOffset: 6}, nil
},
)
m.Update(m.startMetricsRead()())
m.Update(m.startLogPoll()())

v := m.View()
if strings.Contains(v, "requests:") {
t.Errorf("a requests line drawn for statistics without the figure:\n%s", v)
}
if !strings.Contains(v, "prompt tokens:") {
t.Errorf("the prompt tokens line is missing:\n%s", v)
}
}

// The same frame, the follow paused: the title bar says so.
func TestServeViewFramePaused(t *testing.T) {
fixDashNow(t, time.Date(2026, 9, 6, 12, 0, 0, 0, time.UTC))
Expand Down
6 changes: 6 additions & 0 deletions docs/openapi.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -481,6 +481,12 @@ components:
type: integer
requests:
type: integer
description: |
The engine's cumulative request counter, present only where the
engine exposes one — vLLM's request_success_total. Absent for an
engine family whose metrics carry no cumulative request count
(llama.cpp's, today): a missing figure is not a zero, it is a
counter the engine never produced.

GpuStat:
type: object
Expand Down
4 changes: 3 additions & 1 deletion examples/fleet-docker/engine/engine-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,9 @@ resources:
llamacpp:n_decode_total 900
llamacpp:requests_processing 2
llamacpp:requests_deferred 1
llamacpp:request_success_total 17
# No cumulative request counter: a real llama.cpp server serves none,
# and spinloop reports no requests figure for an engine that exposes
# none.

# An OpenAI-compatible route, so the node looks like a real endpoint if you
# curl it while exploring.
Expand Down
5 changes: 3 additions & 2 deletions examples/gateway-docker/engine/engine-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,9 @@ resources:
llamacpp:requests_processing 2
# HELP llamacpp:requests_deferred Number of deferred requests.
llamacpp:requests_deferred 1
# HELP llamacpp:request_success_total Number of successful requests.
llamacpp:request_success_total 17
# No cumulative request counter: a real llama.cpp server serves none,
# and spinloop reports no requests figure for an engine that exposes
# none.

- path: /v1/models
method: GET
Expand Down
42 changes: 42 additions & 0 deletions internal/daemon/activity_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -446,6 +446,48 @@ while true; do sleep 0.05; done`)
}
}

// TestMetricsOmitsRequestCountWhereUnexposed covers the request figure's
// absence through the daemon's own path: the sampler scrapes an engine whose
// metrics expose no cumulative request counter, and the metrics reply carries
// the token stats without the requests field — a missing figure, not a zero
// the engine never produced.
func TestMetricsOmitsRequestCountWhereUnexposed(t *testing.T) {
engineMetrics := &fakeEngine{counter: 100}
engine := httptest.NewServer(engineMetrics)
defer engine.Close()

d := testDaemon(t, `trap 'exit 0' TERM
while true; do sleep 0.05; done`)
d.SetScrape(metrics.ScrapeTarget{BaseURL: engine.URL, Engine: "llamacpp"})
if err := d.Push(inference.DeployConfig{Runner: "llamacpp", ModelID: "m"}); err != nil {
t.Fatal(err)
}
if err := d.StartEngine(); err != nil {
t.Fatal(err)
}
waitForState(t, d.Sup, StateRunning)
defer d.Sup.Stop()
d.sampleOnce(context.Background())

stats := d.Metrics(context.Background())
if stats.Tokens == nil {
t.Fatal("metrics carried no token stats")
}
if stats.Tokens.Requests != nil {
t.Errorf("requests = %d, want absent: the engine exposes no cumulative request counter", *stats.Tokens.Requests)
}
body, err := json.Marshal(stats)
if err != nil {
t.Fatal(err)
}
if bytes.Contains(body, []byte(`"requests"`)) {
t.Errorf("an absent request count still serialised: %s", body)
}
if !bytes.Contains(body, []byte(`"promptTokens"`)) {
t.Errorf("the token stats are missing from the reply: %s", body)
}
}

// TestMetricsReportsActivity covers what /v1/metrics now says about activity:
// the same answer /v1/status gives, from the same record, including after the
// engine has stopped and when there is nothing to report at all.
Expand Down
7 changes: 4 additions & 3 deletions internal/fleet/remote_node_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,8 @@ func TestStatusFromRemote(t *testing.T) {
}

func TestStatsFromRemote(t *testing.T) {
tokens := &metrics.TokenStats{Running: 2, PromptTokens: 5, GenerationTokens: 7, Requests: 3}
requests := 3
tokens := &metrics.TokenStats{Running: 2, PromptTokens: 5, GenerationTokens: 7, Requests: &requests}
cpuPct := 30.0
history := []metrics.HistorySample{{Time: 1, CPU: &cpuPct, GPUs: []metrics.HistoryGPU{{Index: 0, Util: 61, Mem: &cpuPct}}}}
got := statsFromRemote(remote.StatsResponse{
Expand All @@ -139,7 +140,7 @@ func TestStatsFromRemote(t *testing.T) {
if got.State != "running" || got.Runner != "llamacpp" || got.ModelID != "org/m" || got.UptimeSeconds != 10 {
t.Errorf("statsFromRemote = %+v", got)
}
if got.Tokens == nil || got.Tokens.Running != 2 || got.Tokens.Requests != 3 {
if got.Tokens == nil || got.Tokens.Running != 2 || got.Tokens.Requests == nil || *got.Tokens.Requests != 3 {
t.Errorf("token stats not carried over: %+v", got.Tokens)
}
if got.IdleSeconds != 5 || got.LastActiveAt == "" {
Expand Down Expand Up @@ -521,7 +522,7 @@ func TestRemoteNodeStartStopMetricsOverTheControlPlane(t *testing.T) {
if err != nil || stats.State != "running" || stats.ModelID != "org/m" {
t.Errorf("Metrics = %+v, %v", stats, err)
}
if stats.Tokens == nil || stats.Tokens.Running != 1 || stats.Tokens.Requests != 2 {
if stats.Tokens == nil || stats.Tokens.Running != 1 || stats.Tokens.Requests == nil || *stats.Tokens.Requests != 2 {
t.Errorf("metrics tokens not mapped: %+v", stats.Tokens)
}
stopped, err := node.Stop(ctx)
Expand Down
9 changes: 8 additions & 1 deletion internal/metrics/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,14 @@ type TokenStats struct {
Counter int `json:"counter"`
PromptTokens int `json:"promptTokens"`
GenerationTokens int `json:"generationTokens"`
Requests int `json:"requests"`
// Requests is the engine's cumulative request counter, present only
// where the engine family's metrics expose one — vLLM's
// request_success_total, today. A family whose metrics carry no
// cumulative request count (llama.cpp's, today) leaves it nil, so a
// missing figure and a genuine zero stay distinguishable: a zero is a
// counter the engine served, a missing one is a figure no engine
// produced.
Requests *int `json:"requests,omitempty"`
}

// GpuStat holds per-GPU metrics from nvidia-smi.
Expand Down
Loading
Loading