diff --git a/app/renderer/lib/types.ts b/app/renderer/lib/types.ts index ed8b3800..7571446e 100644 --- a/app/renderer/lib/types.ts +++ b/app/renderer/lib/types.ts @@ -172,6 +172,15 @@ export type MenubarPayload = { savingsUSD: number savingsBaselineModel: string calls: number + // Per-model token counts (src/menubar-json.ts buildTopModels): billable + // output, cache read = reused input, cache write separate. Optional: + // older CLIs omit them, and a row whose contributing legacy data lacked + // counts omits them even on a new CLI. Absent means unknown — render a + // dash, never zero, and never substitute a period-wide figure. + inputTokens?: number + outputTokens?: number + cacheReadTokens?: number + cacheWriteTokens?: number }> unpricedModels?: Array<{ model: string; calls: number; tokens: number }> localModelSavings: LocalModelSavings diff --git a/app/renderer/sections/Models.test.tsx b/app/renderer/sections/Models.test.tsx index f3625cfa..a7508250 100644 --- a/app/renderer/sections/Models.test.tsx +++ b/app/renderer/sections/Models.test.tsx @@ -223,14 +223,21 @@ describe('Models', () => { expect(screen.queryByText('add alias ›')).not.toBeInTheDocument() }) - it('renders unpriced proxy rows as dim with alias affordance and dashes', async () => { + it('renders unpriced proxy rows as dim with alias affordance, keeping observed tokens visible', async () => { getModels.mockResolvedValue([rows[3]]) render() expect(await screen.findByText('my-proxy-model')).toHaveClass('dim') expect(screen.getByText('add alias ›')).toHaveClass('alias') - expect(screen.getAllByText('—')).toHaveLength(5) + // Tokens are observed usage, not a pricing artifact: they render even + // though the model has no pricing entry. Cache read shows its known zero. + expect(screen.getByText('4.8M')).toBeInTheDocument() + expect(screen.getByText('400K')).toBeInTheDocument() + expect(screen.getByText('0')).toBeInTheDocument() + expect(screen.getByText('4.8M')).not.toHaveClass('dim') + // Only cost and saved collapse to dashes. + expect(screen.getAllByText('—')).toHaveLength(2) expect(screen.queryByText('$0.00')).not.toBeInTheDocument() }) diff --git a/app/renderer/sections/Models.tsx b/app/renderer/sections/Models.tsx index baab9e52..38009950 100644 --- a/app/renderer/sections/Models.tsx +++ b/app/renderer/sections/Models.tsx @@ -271,7 +271,10 @@ function ModelsByTaskTable({ rows, onAddAlias }: { rows: ModelReportRow[]; onAdd function ModelTableRow({ row, onAddAlias }: { row: ModelReportRow; onAddAlias: () => void }) { const unpriced = row.costUSD === 0 && row.savingsUSD === 0 const cellClass = unpriced ? 'dim' : undefined - const tokenValue = (value: number) => (unpriced ? '—' : formatCompact(value)) + // Token columns are observed usage, not a pricing artifact: a model with no + // pricing entry still burned real input/output/cache-read tokens, so they + // render regardless. Only cost/saved collapse to dashes behind the alias + // affordance — there is no attributed cost to show for them. const dotStyle = { display: 'inline-block', background: seriesColorForModel(row.modelDisplayName || row.model), @@ -292,9 +295,9 @@ function ModelTableRow({ row, onAddAlias }: { row: ModelReportRow; onAddAlias: ( {row.providerDisplayName} {fmtInt(row.calls)} - {tokenValue(row.inputTokens)} - {tokenValue(row.outputTokens)} - {tokenValue(row.cacheReadTokens)} + {formatCompact(row.inputTokens)} + {formatCompact(row.outputTokens)} + {formatCompact(row.cacheReadTokens)} {unpriced ? '—' : formatUsd(row.costUSD)} 0 ? 'pos' : undefined}>{unpriced ? '—' : formatUsd(row.savingsUSD)} @@ -336,15 +339,15 @@ function ModelGroupRow({ rows, onAddAlias }: { rows: ModelReportRow[]; onAddAlia function ModelTaskRow({ row }: { row: ModelReportRow }) { const unpriced = row.costUSD === 0 && row.savingsUSD === 0 const cellClass = unpriced ? 'dim' : undefined - const tokenValue = (value: number) => (unpriced ? '—' : formatCompact(value)) return ( {row.category ?? 'general'} {fmtInt(row.calls)} - {tokenValue(row.inputTokens)} - {tokenValue(row.outputTokens)} - {tokenValue(row.cacheReadTokens)} + {/* Observed usage renders even for unpriced models — see ModelTableRow. */} + {formatCompact(row.inputTokens)} + {formatCompact(row.outputTokens)} + {formatCompact(row.cacheReadTokens)} {unpriced ? '—' : formatUsd(row.costUSD)} 0 ? 'pos' : undefined}>{unpriced ? '—' : formatUsd(row.savingsUSD)} diff --git a/app/renderer/sections/Overview.test.tsx b/app/renderer/sections/Overview.test.tsx index ddf9146c..0adcce5b 100644 --- a/app/renderer/sections/Overview.test.tsx +++ b/app/renderer/sections/Overview.test.tsx @@ -624,8 +624,55 @@ describe('Overview', () => { expect(rows[1]).toHaveTextContent('$120.00') expect(rows[1]).toHaveTextContent('240') expect(rows[2]).toHaveTextContent('claude-opus-4') - // current.topModels carries no per-model tokens → both token cells show a dash. - expect(within(rows[1] as HTMLElement).getAllByText('—')).toHaveLength(2) + // This legacy-shaped payload carries no per-model counts → all three token + // cells (input, output, cache read) show a dash. + expect(within(rows[1] as HTMLElement).getAllByText('—')).toHaveLength(3) + }) + + it('prefers current.topModels for the models table when the payload carries per-model counts', async () => { + const now = new Date() + const payload = makePayload(now) + // New-CLI payload: per-model counts ride on current.topModels, including + // cache read. history.daily still carries different (per-day, truncated) + // aggregates that the table must NOT fall back to. + payload.current.topModels = [ + { name: 'claude-opus-4', cost: 200, savingsUSD: 0, savingsBaselineModel: '', calls: 100, inputTokens: 1_200_000, outputTokens: 340_000, cacheReadTokens: 56_000_000, cacheWriteTokens: 7_000 }, + { name: 'claude-haiku-4', cost: 4, savingsUSD: 0, savingsBaselineModel: '', calls: 12, inputTokens: 0, outputTokens: 0, cacheReadTokens: 900, cacheWriteTokens: 0 }, + ] + + render() + + const modelsTable = await screen.findByRole('table', { name: 'Models this period' }) + expect(within(modelsTable).getByRole('columnheader', { name: 'Cache read' })).toBeInTheDocument() + const rows = within(modelsTable).getAllByRole('row') + // Counts come from current.topModels (1.2M in), not the daily aggregation (40M in). + expect(rows[1]).toHaveTextContent('claude-opus-4') + expect(rows[1]).toHaveTextContent('1.2M') + expect(rows[1]).toHaveTextContent('340K') + expect(rows[1]).toHaveTextContent('56M') + expect(within(modelsTable).queryByText('40M')).not.toBeInTheDocument() + // Known zeros stay zeros: haiku's fresh input/output render as 0, its cache + // read as the real 900. + expect(within(rows[2] as HTMLElement).getAllByText('0')).toHaveLength(2) + expect(within(rows[2] as HTMLElement).getByText('900')).toBeInTheDocument() + }) + + it('falls back to aggregating history.daily when the payload predates per-model counts', async () => { + const now = new Date() + const payload = makePayload(now) + // Legacy all-provider payload: current.topModels has no counts, history.daily + // does (input/output only — the CLI never emitted per-model cache read there). + + render() + + const modelsTable = await screen.findByRole('table', { name: 'Models this period' }) + const rows = within(modelsTable).getAllByRole('row') + // Input/output still come from the daily aggregation (30 days × 40M/2M) ... + expect(rows[1]).toHaveTextContent('claude-opus-4') + expect(rows[1]).toHaveTextContent('1.2B') + expect(rows[1]).toHaveTextContent('60M') + // ... and the absent per-model cache read shows as a dash, not zero. + expect(within(rows[1] as HTMLElement).getAllByText('—')).toHaveLength(1) }) it('suppresses the week-over-week signal and MTD card for a custom range', async () => { diff --git a/app/renderer/sections/Overview.tsx b/app/renderer/sections/Overview.tsx index f2ac8753..cb32838f 100644 --- a/app/renderer/sections/Overview.tsx +++ b/app/renderer/sections/Overview.tsx @@ -464,16 +464,25 @@ type AggregatedModel = { name: string cost: number calls: number - // Absent in provider-filtered mode: `current.topModels` carries no per-model - // token counts, so the table shows "—" rather than a misleading zero. + // Absent when the payload carries no count for the row (an older CLI, or a + // row whose contributing legacy data lacked counts): the table shows "—" + // rather than a misleading zero. inputTokens?: number outputTokens?: number + cacheReadTokens?: number } /** Provider-filtered source: `current.topModels` is already period/range/provider-scoped by the CLI. */ function topModelsToAggregated(models: MenubarPayload['current']['topModels']): AggregatedModel[] { return models - .map(model => ({ name: model.name, cost: model.cost, calls: model.calls })) + .map(model => ({ + name: model.name, + cost: model.cost, + calls: model.calls, + ...(model.inputTokens === undefined ? {} : { inputTokens: model.inputTokens }), + ...(model.outputTokens === undefined ? {} : { outputTokens: model.outputTokens }), + ...(model.cacheReadTokens === undefined ? {} : { cacheReadTokens: model.cacheReadTokens }), + })) .sort((a, b) => b.cost - a.cost) } @@ -509,6 +518,8 @@ function ModelsTable({ models }: { models: AggregatedModel[] }) { Model Input tok Output tok + {/* Reused input tokens: prompts the provider served from cache. */} + Cache read Cost Calls @@ -519,6 +530,7 @@ function ModelsTable({ models }: { models: AggregatedModel[] }) { {model.name} {model.inputTokens === undefined ? '—' : formatCompact(model.inputTokens)} {model.outputTokens === undefined ? '—' : formatCompact(model.outputTokens)} + {model.cacheReadTokens === undefined ? '—' : formatCompact(model.cacheReadTokens)} {formatUsd(model.cost)} {model.calls.toLocaleString('en-US')} @@ -780,9 +792,16 @@ export function OverviewContent({ periodDaily[0] && periodDaily[0].date < defaultChartStart ? periodDaily[0].date : defaultChartStart, localDateKey(now), ) - // Provider-filtered history.daily has empty topModels, so source the models - // table from current.topModels (already period/range/provider-scoped) instead. - const models = provider !== 'all' + // Models this period come from `current.topModels` — period/range/provider- + // scoped by the CLI, and (on CLIs that emit per-model counts) carrying input/ + // output/cache-read counts for every model in the period, including days + // whose per-day top-5 history list no longer names them. history.daily is + // the fallback for payloads from older CLIs: its rows know input/output but + // not cache read, so the cache column shows "—" there. + const topModelsCarryCounts = data.current.topModels.some(model => + model.inputTokens !== undefined || model.outputTokens !== undefined, + ) + const models = provider !== 'all' || topModelsCarryCounts ? topModelsToAggregated(data.current.topModels) : aggregateModels(rangeActive ? sliceDailyToRange(data.history.daily, range.from, range.to) : periodDaily) const recent14 = data.history.daily.slice(-14) diff --git a/mac/Sources/CodeBurnMenubar/Data/MenubarPayload.swift b/mac/Sources/CodeBurnMenubar/Data/MenubarPayload.swift index 4165ddc7..939ca77c 100644 --- a/mac/Sources/CodeBurnMenubar/Data/MenubarPayload.swift +++ b/mac/Sources/CodeBurnMenubar/Data/MenubarPayload.swift @@ -488,6 +488,41 @@ struct ModelEntry: Codable, Sendable { let savingsUSD: Double let savingsBaselineModel: String let calls: Int + /// Per-model token counts: input, output, cache read (reused input), and + /// cache write, kept separate so the two cache flavors are never summed. + /// Nil on every CLI up to the token-breakdown release and on any row whose + /// contributing legacy data lacked counts: absent means "unknown", which + /// renders as a dash — never as zero, and never as a period-wide figure. + let inputTokens: Int? + let outputTokens: Int? + let cacheReadTokens: Int? + let cacheWriteTokens: Int? + + /// Whether any per-model count arrived. A row with none (legacy payload) + /// renders without the secondary token line rather than as a run of dashes. + var hasTokenCounts: Bool { + inputTokens != nil || outputTokens != nil || cacheReadTokens != nil + } + + init(name: String, + cost: Double, + savingsUSD: Double, + savingsBaselineModel: String, + calls: Int, + inputTokens: Int? = nil, + outputTokens: Int? = nil, + cacheReadTokens: Int? = nil, + cacheWriteTokens: Int? = nil) { + self.name = name + self.cost = cost + self.savingsUSD = savingsUSD + self.savingsBaselineModel = savingsBaselineModel + self.calls = calls + self.inputTokens = inputTokens + self.outputTokens = outputTokens + self.cacheReadTokens = cacheReadTokens + self.cacheWriteTokens = cacheWriteTokens + } init(from decoder: Decoder) throws { let c = try decoder.container(keyedBy: CodingKeys.self) @@ -496,10 +531,15 @@ struct ModelEntry: Codable, Sendable { savingsUSD = try c.decodeIfPresent(Double.self, forKey: .savingsUSD) ?? 0 savingsBaselineModel = try c.decodeIfPresent(String.self, forKey: .savingsBaselineModel) ?? "" calls = try c.decode(Int.self, forKey: .calls) + inputTokens = try c.decodeIfPresent(Int.self, forKey: .inputTokens) + outputTokens = try c.decodeIfPresent(Int.self, forKey: .outputTokens) + cacheReadTokens = try c.decodeIfPresent(Int.self, forKey: .cacheReadTokens) + cacheWriteTokens = try c.decodeIfPresent(Int.self, forKey: .cacheWriteTokens) } private enum CodingKeys: String, CodingKey { case name, cost, savingsUSD, savingsBaselineModel, calls + case inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens } } diff --git a/mac/Sources/CodeBurnMenubar/Views/ModelsSection.swift b/mac/Sources/CodeBurnMenubar/Views/ModelsSection.swift index 4cc90d4b..2f3b8995 100644 --- a/mac/Sources/CodeBurnMenubar/Views/ModelsSection.swift +++ b/mac/Sources/CodeBurnMenubar/Views/ModelsSection.swift @@ -41,45 +41,107 @@ struct ModelsSection: View { } } +/// Compact token count for the narrow popover: `1.2K` / `3.4M`, plain digits +/// below a thousand. The exact values ride along in the row's accessibility +/// label, so compact rounding never hides the real number. +private func compactTokenCount(_ n: Int) -> String { + if n >= 1_000_000 { + return String(format: "%.1fM", Double(n) / 1_000_000) + } else if n >= 1_000 { + return String(format: "%.1fK", Double(n) / 1_000) + } + return "\(n)" +} + private struct ModelRow: View { let model: ModelEntry let maxCost: Double let showSavings: Bool var body: some View { - HStack(spacing: 8) { - // Bar tracks actual cost; for local models the cost is $0 and the - // bar will be empty. Saved counterfactual (if any) renders as - // green text in the saved column, never summed into the bar. - FixedBar(fraction: model.cost / maxCost) - .frame(width: 56, height: 6) - - Text(model.name) - .font(.system(size: 12.5, weight: .medium)) - .frame(maxWidth: .infinity, alignment: .leading) - - Text(model.cost.asCompactCurrency()) - .font(.codeMono(size: 12, weight: .medium)) - .tracking(-0.2) - .frame(minWidth: 54, alignment: .trailing) - - if showSavings { - Text(model.savingsUSD > 0 ? model.savingsUSD.asCompactCurrency() : "—") - .font(.codeMono(size: 12)) + VStack(alignment: .leading, spacing: 2) { + HStack(spacing: 8) { + // Bar tracks actual cost; for local models the cost is $0 and the + // bar will be empty. Saved counterfactual (if any) renders as + // green text in the saved column, never summed into the bar. + FixedBar(fraction: model.cost / maxCost) + .frame(width: 56, height: 6) + + Text(model.name) + .font(.system(size: 12.5, weight: .medium)) + .lineLimit(1) + .frame(maxWidth: .infinity, alignment: .leading) + + Text(model.cost.asCompactCurrency()) + .font(.codeMono(size: 12, weight: .medium)) .tracking(-0.2) - .foregroundStyle(model.savingsUSD > 0 ? Color.green : Color.secondary) .frame(minWidth: 54, alignment: .trailing) + + if showSavings { + Text(model.savingsUSD > 0 ? model.savingsUSD.asCompactCurrency() : "—") + .font(.codeMono(size: 12)) + .tracking(-0.2) + .foregroundStyle(model.savingsUSD > 0 ? Color.green : Color.secondary) + .frame(minWidth: 54, alignment: .trailing) + } + + Text("\(model.calls)") + .font(.system(size: 11)) + .monospacedDigit() + .foregroundStyle(.secondary) + .frame(minWidth: 52, alignment: .trailing) } - Text("\(model.calls)") - .font(.system(size: 11)) - .monospacedDigit() - .foregroundStyle(.secondary) - .frame(minWidth: 52, alignment: .trailing) + // Token counts sit on their own secondary line under the model name: + // seven squashed columns cannot stay legible in the narrow popover, + // and the cost stays visually attached to its model either way. The + // line appears only when the payload carries counts for the row — an + // older CLI renders exactly as before. + if model.hasTokenCounts { + Text("\(count(model.inputTokens)) in · \(count(model.outputTokens)) out · \(count(model.cacheReadTokens)) cache read") + .font(.system(size: 10.5)) + .monospacedDigit() + .foregroundStyle(.secondary) + .lineLimit(1) + .truncationMode(.tail) + .padding(.leading, 64) + .accessibilityLabel(model.tokenAccessibilityText) + } } .padding(.horizontal, 2) .padding(.vertical, 1) } + + // Unknown (absent count) renders as a dash; a known zero renders as "0". + private func count(_ value: Int?) -> String { + value.map(compactTokenCount) ?? "—" + } +} + +extension ModelEntry { + /// Exact token counts for assistive tech, since the visible line rounds + /// compactly. Cache read is labelled as reused input, and cache write is + /// named separately so the two are never read as one bucket. Locale-pinned + /// comma grouping so the text is deterministic. + var tokenAccessibilityText: String { + func exact(_ value: Int) -> String { + var digits = String(value) + var grouped = "" + while digits.count > 3 { + let cut = digits.index(digits.endIndex, offsetBy: -3) + grouped = "," + digits[cut...] + grouped + digits = String(digits[.. 0 { parts.append("\(exact(cacheWriteTokens)) cache write") } + return parts.joined(separator: ", ") + } } private struct TokensLine: View { @@ -109,11 +171,6 @@ private struct TokensLine: View { } private func formatTokens(_ n: Int) -> String { - if n >= 1_000_000 { - return String(format: "%.1fM", Double(n) / 1_000_000) - } else if n >= 1_000 { - return String(format: "%.1fK", Double(n) / 1_000) - } - return "\(n)" + compactTokenCount(n) } } diff --git a/mac/Tests/CodeBurnMenubarTests/ModelEntryTokenCountsTests.swift b/mac/Tests/CodeBurnMenubarTests/ModelEntryTokenCountsTests.swift new file mode 100644 index 00000000..5236f96b --- /dev/null +++ b/mac/Tests/CodeBurnMenubarTests/ModelEntryTokenCountsTests.swift @@ -0,0 +1,138 @@ +import Foundation +import Testing +@testable import CodeBurnMenubar + +/// Per-model token counts on `current.topModels` rows: decode shape, the +/// unknown-vs-zero rule, and the row presentation contract (compact secondary +/// line, exact values in accessibility, dashes for unknown). +@Suite("ModelEntry token counts") +struct ModelEntryTokenCountsTests { + + private func payloadJSON(topModels: String) -> Data { + Data(""" + { + "generated": "2026-09-07T00:00:00Z", + "current": { + "label": "Today", + "cost": 3.25, + "calls": 12, + "sessions": 2, + "inputTokens": 1000, + "outputTokens": 500, + "cacheHitPercent": 40, + "topModels": \(topModels) + }, + "optimize": { "findingCount": 0, "savingsUSD": 0, "topFindings": [] }, + "history": { "daily": [] } + } + """.utf8) + } + + private func decode(_ topModels: String) throws -> MenubarPayload { + try JSONDecoder().decode(MenubarPayload.self, from: payloadJSON(topModels: topModels)) + } + + @Test("decodes per-model counts when the payload carries them") + func decodesCounts() throws { + let payload = try decode(""" + [ + { "name": "Sonnet 4.6", "cost": 2.5, "savingsUSD": 0, "savingsBaselineModel": "", "calls": 9, + "inputTokens": 152300, "outputTokens": 40200, "cacheReadTokens": 1180000, "cacheWriteTokens": 46000 } + ] + """) + let row = payload.current.topModels[0] + #expect(row.inputTokens == 152_300) + #expect(row.outputTokens == 40_200) + #expect(row.cacheReadTokens == 1_180_000) + #expect(row.cacheWriteTokens == 46_000) + #expect(row.hasTokenCounts) + } + + @Test("counts stay nil on legacy rows that predate the fields") + func legacyRowsStayNil() throws { + let payload = try decode(""" + [ + { "name": "Sonnet 4.6", "cost": 2.5, "savingsUSD": 0, "savingsBaselineModel": "", "calls": 9 } + ] + """) + let row = payload.current.topModels[0] + #expect(row.inputTokens == nil) + #expect(row.outputTokens == nil) + #expect(row.cacheReadTokens == nil) + #expect(row.cacheWriteTokens == nil) + #expect(!row.hasTokenCounts) + } + + @Test("a known zero decodes as zero, never as unknown") + func knownZeroStaysZero() throws { + let payload = try decode(""" + [ + { "name": "Haiku 4.5", "cost": 0, "savingsUSD": 0, "savingsBaselineModel": "", "calls": 3, + "inputTokens": 0, "outputTokens": 0, "cacheReadTokens": 900000, "cacheWriteTokens": 0 } + ] + """) + let row = payload.current.topModels[0] + #expect(row.inputTokens == 0) + #expect(row.outputTokens == 0) + #expect(row.cacheReadTokens == 900_000) + #expect(row.cacheWriteTokens == 0) + #expect(row.hasTokenCounts) + } + + @Test("partially present counts keep the missing ones nil") + func partialCountsStayNil() throws { + let payload = try decode(""" + [ + { "name": "Sonnet 4.6", "cost": 1, "savingsUSD": 0, "savingsBaselineModel": "", "calls": 1, + "inputTokens": 100 } + ] + """) + let row = payload.current.topModels[0] + #expect(row.inputTokens == 100) + #expect(row.outputTokens == nil) + #expect(row.cacheReadTokens == nil) + #expect(row.hasTokenCounts) + } + + @Test("secondary line: known zeros render as 0, unknown as a dash") + func secondaryLineRenderings() throws { + let zero = ModelEntry(name: "zero", cost: 0, savingsUSD: 0, savingsBaselineModel: "", calls: 1, + inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0) + #expect(zero.hasTokenCounts) + let unknown = ModelEntry(name: "unknown", cost: 1, savingsUSD: 0, savingsBaselineModel: "", calls: 1, + outputTokens: 12) + #expect(unknown.hasTokenCounts) + // hasTokenCounts drives whether the secondary line renders at all. + #expect(!ModelEntry(name: "legacy", cost: 1, savingsUSD: 0, savingsBaselineModel: "", calls: 1).hasTokenCounts) + } + + @Test("accessibility text: exact counts, cache read labelled reused input, cache write separate") + func accessibilityTextKeepsCacheKindsDistinct() throws { + let payload = try decode(""" + [ + { "name": "Sonnet 4.6", "cost": 2.5, "savingsUSD": 0, "savingsBaselineModel": "", "calls": 9, + "inputTokens": 152300, "outputTokens": 40200, "cacheReadTokens": 1180000, "cacheWriteTokens": 46000 } + ] + """) + let label = payload.current.topModels[0].tokenAccessibilityText + #expect(label.contains("152,300 input")) + #expect(label.contains("40,200 output")) + #expect(label.contains("1,180,000 cache read (reused input)")) + #expect(label.contains("46,000 cache write")) + // The two cache flavors must never merge into one bucket. + #expect(!label.contains("cache 1,226,000")) + } + + @Test("accessibility text omits zero cache write instead of pairing it with cache read") + func accessibilityTextOmitsZeroCacheWrite() throws { + let payload = try decode(""" + [ + { "name": "Haiku 4.5", "cost": 0, "savingsUSD": 0, "savingsBaselineModel": "", "calls": 3, + "inputTokens": 0, "outputTokens": 0, "cacheReadTokens": 900000, "cacheWriteTokens": 0 } + ] + """) + let label = payload.current.topModels[0].tokenAccessibilityText + #expect(label.contains("900,000 cache read (reused input)")) + #expect(!label.contains("cache write")) + } +} diff --git a/mac/Tests/CodeBurnMenubarTests/ModelsSectionLayoutProofTests.swift b/mac/Tests/CodeBurnMenubarTests/ModelsSectionLayoutProofTests.swift new file mode 100644 index 00000000..1d068707 --- /dev/null +++ b/mac/Tests/CodeBurnMenubarTests/ModelsSectionLayoutProofTests.swift @@ -0,0 +1,211 @@ +import AppKit +import SwiftUI +import Testing +@testable import CodeBurnMenubar + +/// Native layout proof for the Models section's per-model token line, rendered +/// through NSHostingView in an offscreen window at the popover's REAL 360pt +/// width using the actual popover root (`MenuBarContent`), so every horizontal +/// padding that affects a row is present. No status item is created, no +/// popover is shown, nothing is ordered onto the screen, and no CLI is +/// invoked: payloads are injected through the AppStore testing hooks and +/// refreshes are suppressed. +/// +/// When `CODEBURN_LAYOUT_PROOF_DIR` is set, each variant is written there as a +/// 2x PNG (the review evidence artifacts). The suite always renders through a +/// real AppKit layout pass and asserts the image comes out; PNG writing is a +/// best-effort side effect. +/// This is fixture / native-view evidence — NOT installed-app validation. +@Suite("Models section layout proof") +@MainActor +struct ModelsSectionLayoutProofTests { + + // MARK: - Fixtures + + /// Cost-descending model rows the way `buildTopModels` emits them: a long + /// display name with savings, a ≥1B cache-read count, a known-zero row, + /// and a legacy row that predates the counts (secondary line hidden). + private static func savingsPresentPayload() -> MenubarPayload { + payload(topModels: [ + ModelEntry(name: "Gemini 3.7 Flash Thinking (Preview Channel)", cost: 84.7, savingsUSD: 12.4, savingsBaselineModel: "", calls: 3311, + inputTokens: 152_300_456, outputTokens: 40_234_112, cacheReadTokens: 1_180_456_789, cacheWriteTokens: 46_112_003), + ModelEntry(name: "gpt-6-astra", cost: 51.2, savingsUSD: 0, savingsBaselineModel: "", calls: 8455, + inputTokens: 33_624_660, outputTokens: 5_018_920, cacheReadTokens: 12_345_678_901, cacheWriteTokens: 0), + ModelEntry(name: "Llama Local", cost: 0, savingsUSD: 9.9, savingsBaselineModel: "", calls: 82, + inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0), + ModelEntry(name: "Legacy Snapshot Model", cost: 9.99, savingsUSD: 0, savingsBaselineModel: "", calls: 4), + ]) + } + + /// No savings anywhere in the period, so the Saved column is absent and + /// the token line gets the extra room — plus a $0-cost row whose observed + /// tokens must still render. + private static func savingsAbsentPayload() -> MenubarPayload { + payload(topModels: [ + ModelEntry(name: "Claude Opus 4.8", cost: 331.2, savingsUSD: 0, savingsBaselineModel: "", calls: 4812, + inputTokens: 152_600_000, outputTokens: 9_640_000, cacheReadTokens: 119_400_000, cacheWriteTokens: 16_000_000), + ModelEntry(name: "my-proxy-model", cost: 0, savingsUSD: 0, savingsBaselineModel: "", calls: 176, + inputTokens: 4_800_000, outputTokens: 400_000, cacheReadTokens: 0, cacheWriteTokens: 0), + ]) + } + + private static func payload(topModels: [ModelEntry]) -> MenubarPayload { + MenubarPayload( + generated: "2026-09-07T00:00:00Z", + current: CurrentBlock( + label: "Today", + cost: 155.89, + calls: 11852, + sessions: 14, + oneShotRate: 0.74, + inputTokens: 186_000_000, + outputTokens: 45_000_000, + cacheHitPercent: 63.4, + codexCredits: 0, + topActivities: [], + topModels: topModels, + localModelSavings: LocalModelSavings(totalUSD: 9.9, calls: 82, byModel: [], byProvider: []), + providers: [:], + topProjects: [], + modelEfficiency: [], + topSessions: [], + retryTax: RetryTax(totalUSD: 0, retries: 0, editTurns: 0, byModel: []), + routingWaste: RoutingWaste(totalSavingsUSD: 0, baselineModel: "", baselineCostPerEdit: 0, byModel: []), + tools: [], + skills: [], + subagents: [], + mcpServers: [] + ), + optimize: OptimizeBlock(findingCount: 0, savingsUSD: 0, topFindings: []), + history: HistoryBlock(daily: []), + combined: nil + ) + } + + // MARK: - Harness + + private func makeStore(payload: MenubarPayload) -> AppStore { + let store = AppStore() + store.setCacheDateToTodayForTesting() + store.suppressRefreshesForTesting() + store.menuPopoverVisible = true + // Pin the whole selection: saved menubar defaults must not decide what + // a fixture render shows. + store.selectedScope = .local + store.selectedPeriod = .today + store.selectedProvider = .all + store.selectedDays = [] + store.selectedClaudeConfigSourceId = nil + store.setCachedPayloadForTesting(payload, period: .today, provider: .all, fetchedAt: Date()) + // The popover renders its cold-cache overlay unless the store actually + // serves the fixture from `payload`. + if store.payload.current.cost != payload.current.cost { + Issue.record("store failed to serve the fixture payload for the current key") + } + return store + } + + /// The exact view the popover installs (CodeBurnApp.makePopoverContent): + /// the real root, the real width, the real environments. The popover + /// surface renders dark, so the scheme is pinned or every `.primary` text + /// renders black-on-black. + private func popoverContent(store: AppStore) -> some View { + MenuBarContent() + .environment(store) + .environment(UpdateChecker()) + .environment(\.colorScheme, .dark) + .frame(width: 360) + } + + /// Render through a REAL NSHostingView inside a borderless window that is + /// never ordered on screen, forcing a genuine AppKit layout pass (ImageRenderer + /// alone does not run the scroll-content layout this popover root needs). + /// Height comes from the hosting view's own fittingSize — the same signal + /// the popover's `.preferredContentSize` sizing uses — so the full Models + /// section is captured at the real 360pt width without inventing a canvas. + @discardableResult + private func render(name: String, store: AppStore, proofDir: String?) throws -> NSSize { + let hosting = NSHostingView(rootView: popoverContent(store: store)) + hosting.frame = NSRect(x: 0, y: 0, width: 360, height: 1) + hosting.layoutSubtreeIfNeeded() + let fitted = hosting.fittingSize + #expect(fitted.width == 360) + let size = NSSize(width: 360, height: max(fitted.height, 660)) // floor: popoverHeight + + let window = NSWindow( + contentRect: NSRect(origin: .zero, size: size), + styleMask: [.borderless], + backing: .buffered, + defer: false, + ) + hosting.frame = NSRect(origin: .zero, size: size) + window.contentView = hosting + // Offscreen by construction: never ordered front, never visible. + window.orderOut(nil) + hosting.layoutSubtreeIfNeeded() + + guard let bitmap = hosting.bitmapImageRepForCachingDisplay(in: NSRect(origin: .zero, size: size)) else { + Issue.record("bitmapImageRepForCachingDisplay failed for \(name)") + return .zero + } + hosting.cacheDisplay(in: NSRect(origin: .zero, size: size), to: bitmap) + + if let proofDir { + let data = try #require(bitmap.representation(using: .png, properties: [:])) + let url = URL(fileURLWithPath: proofDir).appendingPathComponent("menubar-\(name).png") + try data.write(to: url) + } + return size + } + + // MARK: - Tests + + @Test("token line renders at the real 360pt popover width with savings present and absent") + func rendersAtPopoverWidth() throws { + let proofDir = ProcessInfo.processInfo.environment["CODEBURN_LAYOUT_PROOF_DIR"] + + // Full popover at its REAL 360×660 (context: the Models section sits + // below the fold, exactly as in the popover — the first row and the + // column header row are what fit). + let store = makeStore(payload: Self.savingsPresentPayload()) + #expect(store.hasCachedData) + let withSavings = try render(name: "savings-present", store: store, proofDir: proofDir) + #expect(withSavings.width == 360) + + let withoutSavings = try render(name: "savings-absent", store: makeStore(payload: Self.savingsAbsentPayload()), proofDir: proofDir) + #expect(withoutSavings.width == 360) + + // Section-only renders at the same real width: the section carries its + // own row padding and no extra horizontal wrapper in the popover, so + // this is the exact row layout context, uncut. (The section has no + // ScrollView, so ImageRenderer runs its layout faithfully.) + if let proofDir { + for (name, payload) in [("section-savings-present", Self.savingsPresentPayload()), + ("section-savings-absent", Self.savingsAbsentPayload())] { + let sectionStore = makeStore(payload: payload) + let renderer = ImageRenderer(content: ModelsSection() + .environment(sectionStore) + .environment(\.colorScheme, .dark) + .frame(width: 360)) + renderer.scale = 2 + if let cgImage = renderer.cgImage { + let rep = NSBitmapImageRep(cgImage: cgImage) + if let data = rep.representation(using: .png, properties: [:]) { + try data.write(to: URL(fileURLWithPath: proofDir).appendingPathComponent("menubar-\(name).png")) + } + } + } + } + } + + @Test("exact token counts ride in the row accessibility text at every count magnitude") + func accessibilityCarriesExactCounts() throws { + let store = makeStore(payload: Self.savingsPresentPayload()) + let rows = store.payload.current.topModels + #expect(rows[0].tokenAccessibilityText.contains("152,300,456 input")) + #expect(rows[0].tokenAccessibilityText.contains("1,180,456,789 cache read (reused input)")) + #expect(rows[1].tokenAccessibilityText.contains("12,345,678,901 cache read (reused input)")) + // The legacy row has no counts at all, so no accessibility line either. + #expect(rows[3].tokenAccessibilityText.isEmpty) + } +} diff --git a/src/day-aggregator.ts b/src/day-aggregator.ts index 101d6097..f9ffca99 100644 --- a/src/day-aggregator.ts +++ b/src/day-aggregator.ts @@ -258,7 +258,20 @@ export function buildPeriodDataFromDays(days: DailyEntry[], label: string): Peri let cost = 0, savingsUSD = 0, calls = 0, sessions = 0 let inputTokens = 0, outputTokens = 0, cacheReadTokens = 0, cacheWriteTokens = 0 const catTotals: Record = {} - const modelTotals: Record = {} + // Per-model token counts, normalized the same way the day entries were + // written (output already billable — day-aggregator folds reasoning in per + // call). Merge keys stay the raw ids here; the payload resolves display + // names later (buildTopModels), so both aggregation paths land in the same + // rows as cost. + const modelTotals: Record = {} for (const d of days) { cost += d.cost @@ -271,10 +284,17 @@ export function buildPeriodDataFromDays(days: DailyEntry[], label: string): Peri cacheWriteTokens += d.cacheWriteTokens for (const [name, m] of Object.entries(d.models)) { - const acc = modelTotals[name] ?? { calls: 0, cost: 0, savingsUSD: 0 } + const acc = modelTotals[name] ?? { + calls: 0, cost: 0, savingsUSD: 0, + inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0, + } acc.calls += m.calls acc.cost += m.cost acc.savingsUSD += (m.savingsUSD ?? 0) + acc.inputTokens += m.inputTokens + acc.outputTokens += m.outputTokens + acc.cacheReadTokens += m.cacheReadTokens + acc.cacheWriteTokens += m.cacheWriteTokens modelTotals[name] = acc } for (const [cat, c] of Object.entries(d.categories)) { @@ -303,6 +323,9 @@ export function buildPeriodDataFromDays(days: DailyEntry[], label: string): Peri .map(([cat, d]) => ({ name: CATEGORY_LABELS[cat as TaskCategory] ?? cat, ...d })), models: Object.entries(modelTotals) .sort(([, a], [, b]) => b.cost - a.cost) - .map(([name, d]) => ({ name, ...d })), + .map(([name, d]) => ({ + name, + ...d, + })), } } diff --git a/src/main.ts b/src/main.ts index 6c6a11f3..e7fdb169 100644 --- a/src/main.ts +++ b/src/main.ts @@ -58,7 +58,13 @@ const { version } = require('../package.json') // v5: providerDetails carries per-provider tokens and sessions, which a v4 // record predates — the dock glance would read a provider as having no token // breakdown purely because the snapshot was written before this build. -const STATUS_SNAPSHOT_RENDER_VERSION = 5 +// v6: current.topModels rows carry per-model input/output/cache-read/write +// counts, which a v5 record predates — the Models sections would show no +// per-model token breakdown purely because the snapshot was written before +// this build. A v5 record is treated as a miss (one real recompute per +// query), then the fresh record is served; daily/session caches are separate +// version domains and are not touched. +const STATUS_SNAPSHOT_RENDER_VERSION = 6 const STATUS_SNAPSHOT_SEMANTIC_KEY = `${version}:render-${STATUS_SNAPSHOT_RENDER_VERSION}:daily-${DAILY_CACHE_VERSION}` import { loadCurrency, getCurrency, isValidCurrencyCode } from './currency.js' import { CodexThroughputReader, newestCodexSession, renderCodexThroughput } from './codex-throughput.js' diff --git a/src/menubar-json.ts b/src/menubar-json.ts index 273e9d9a..2c30e3c2 100644 --- a/src/menubar-json.ts +++ b/src/menubar-json.ts @@ -24,7 +24,27 @@ export type PeriodData = { /// non-menubar PeriodData producers don't have to compute it. codexCredits?: number categories: Array<{ name: string; cost: number; savingsUSD: number; turns: number; editTurns: number; oneShotTurns: number }> - models: Array<{ name: string; cost: number; savingsUSD: number; calls: number; estimatedCostUSD?: number }> + models: Array<{ + name: string + cost: number + savingsUSD: number + calls: number + estimatedCostUSD?: number + /// Per-model token counts for the period, normalized exactly like the + /// headline totals: billable output (reasoning tokens are added only + /// where the provider reports them separately from output — where output + /// already includes them they are never added twice), `cacheReadTokens` + /// = reused input, `cacheWriteTokens` kept separate so the two are never + /// summed. The attributed cost already includes cache pricing; the counts + /// never restate or rescale it. Optional so PeriodData producers + /// predating the field keep compiling; a consumer must render absent + /// counts as unknown — never as zero, and never substitute the + /// period-wide totals. + inputTokens?: number + outputTokens?: number + cacheReadTokens?: number + cacheWriteTokens?: number + }> /// Models with usage in the period whose pricing lookup fails against the /// current tables (#638): their calls contribute $0 to `cost`. Optional so /// PeriodData producers that predate the field keep compiling. @@ -267,6 +287,15 @@ export type MenubarPayload = { /// Estimated portion of this model's `cost`; > 0 marks the row as priced /// from estimated tokens. Optional for payload back-compat. estimatedCostUSD?: number + /// Per-model token counts, same normalization as `PeriodData.models`: + /// billable output, cache read = reused input, cache write separate. + /// Add-only and optional — omitted when the period carries no count for + /// the row (an older producer, or any contributing legacy row without + /// counts), so a consumer must render absence as unknown, never as zero. + inputTokens?: number + outputTokens?: number + cacheReadTokens?: number + cacheWriteTokens?: number }> /// See PeriodData.unpricedModels: usage priced at $0 for lack of pricing /// data. Empty when every model in the period resolved a price. Optional @@ -453,25 +482,64 @@ function buildTopActivities(categories: PeriodData['categories']): MenubarPayloa })) } +/// Per-model token counts merged alongside cost. A `undefined` accumulator is +/// "unknown", not zero: a legacy row that predates the counts must not turn the +/// merged row into a plausible-looking 0, so one unknown contributor marks the +/// merged count unknown and the field is omitted from the payload. +const MODEL_COUNT_KEYS = ['inputTokens', 'outputTokens', 'cacheReadTokens', 'cacheWriteTokens'] as const +type ModelCountKey = (typeof MODEL_COUNT_KEYS)[number] + +function mergeCount(target: { counts: Partial>; unknown: Set }, key: ModelCountKey, value: number | undefined): void { + // Once a contributor without this count has been seen, the merged count is + // unknown for good — later contributors must not resurrect a partial sum. + if (value === undefined) { + target.unknown.add(key) + delete target.counts[key] + return + } + if (target.unknown.has(key)) return + target.counts[key] = (target.counts[key] ?? 0) + value +} + function buildTopModels(models: PeriodData['models']): MenubarPayload['current']['topModels'] { // Day entries key models by the raw provider id (day-aggregator), so resolve // display names here — the menubar shows "Kimi K3" rather than "k3". Ids that - // collapse to one display name (e.g. k3 and kimi-k3) merge into a single row. - const merged = new Map() + // collapse to one display name (e.g. k3 and kimi-k3) merge into a single row, + // and their token counts merge under the same grouping as cost. + const merged = new Map> + unknown: Set + }>() for (const m of models) { if (m.name === SYNTHETIC_MODEL_NAME) continue const name = getShortModelName(m.name) - const acc = merged.get(name) ?? { cost: 0, calls: 0, savingsUSD: 0, estimatedCostUSD: 0 } + const acc = merged.get(name) ?? { cost: 0, calls: 0, savingsUSD: 0, estimatedCostUSD: 0, counts: {}, unknown: new Set() } acc.cost += m.cost acc.calls += m.calls acc.savingsUSD += m.savingsUSD ?? 0 acc.estimatedCostUSD += m.estimatedCostUSD ?? 0 + for (const key of MODEL_COUNT_KEYS) mergeCount(acc, key, m[key]) merged.set(name, acc) } return [...merged.entries()] .sort(([, a], [, b]) => b.cost - a.cost) .slice(0, TOP_MODELS_LIMIT) - .map(([name, d]) => ({ name, cost: d.cost, calls: d.calls, savingsUSD: d.savingsUSD, savingsBaselineModel: '', estimatedCostUSD: d.estimatedCostUSD })) + .map(([name, d]) => ({ + name, + cost: d.cost, + calls: d.calls, + savingsUSD: d.savingsUSD, + savingsBaselineModel: '', + estimatedCostUSD: d.estimatedCostUSD, + ...(d.counts.inputTokens === undefined ? {} : { inputTokens: d.counts.inputTokens }), + ...(d.counts.outputTokens === undefined ? {} : { outputTokens: d.counts.outputTokens }), + ...(d.counts.cacheReadTokens === undefined ? {} : { cacheReadTokens: d.counts.cacheReadTokens }), + ...(d.counts.cacheWriteTokens === undefined ? {} : { cacheWriteTokens: d.counts.cacheWriteTokens }), + })) } function buildOptimize(optimize: OptimizeResult | null): MenubarPayload['optimize'] { diff --git a/src/usage-aggregator.ts b/src/usage-aggregator.ts index ff273f23..7b06ffa4 100644 --- a/src/usage-aggregator.ts +++ b/src/usage-aggregator.ts @@ -3,7 +3,7 @@ import { CATEGORY_LABELS, type ProjectSummary, type TaskCategory, type DateRange import { isBehavioralCall } from './behavioral-weight.js' import { type PeriodData, type ProviderCost, type BreakdownArrays, type MenubarPayload, type ClaudeConfigSelector, type HydrationState, buildMenubarPayload } from './menubar-json.js' import { parseAllSessions, filterProjectsByName, filterProjectsByDays, filterProjectsByClaudeConfigSource, filterProjectsByDateRange, isSessionHydrationComplete, sessionHydrationSnapshot } from './parser.js' -import { findUnpricedModels, getFlatRateModelsConfigHash, getLocalModelSavingsConfigHash, getPriceOverridesConfigHash, getShortModelName, isExpectedFreeModel } from './models.js' +import { findUnpricedModels, getFlatRateModelsConfigHash, getLocalModelSavingsConfigHash, getPriceOverridesConfigHash, getShortModelName, isExpectedFreeModel, billableOutputTokens } from './models.js' import { getAllProviders, safeDiscoverSessions } from './providers/index.js' import { loadPlugins, pluginPayloadSections } from './plugins/loader.js' import { collectLiveSessions } from './live-sessions.js' @@ -16,7 +16,7 @@ import { aggregateModelTaskTurns, sessionDurationMinutes } from './telemetry-sna import { scanUserCorrections, medianTimeToFirstEditMs, aggregateFileChurn, computePricingCoverage } from './workflow-insights.js' import { buildPrAttribution, aggregateByBranch } from './sessions-report.js' import { scanAndDetect } from './optimize.js' -import { callBillableOutputTokens, sessionBillableOutputTokens } from './session-output.js' +import { callBillableOutputTokens, sessionBillableOutputTokens, sessionModelBillableOutputTokens, inferSessionProvider } from './session-output.js' import { getDaysInRange, ensureCacheHydrated, loadDailyCache, emptyCache, mergeDayEntries, BACKFILL_DAYS, toDateString, type DailyCache, type DailyEntry, type ProjectDayStats, type ProviderDaySlice } from './daily-cache.js' import { buildGranularHistory } from './granular-history.js' @@ -63,7 +63,17 @@ export function providerSliceHasUsage(slice: ProviderDaySlice): boolean { export function buildPeriodData(label: string, projects: ProjectSummary[]): PeriodData { const sessions = projects.flatMap(p => p.sessions) const catTotals: Record = {} - const modelTotals: Record = {} + const modelTotals: Record = {} let inputTokens = 0, outputTokens = 0, cacheReadTokens = 0, cacheWriteTokens = 0 for (const sess of sessions) { @@ -71,6 +81,13 @@ export function buildPeriodData(label: string, projects: ProjectSummary[]): Peri outputTokens += sessionBillableOutputTokens(sess) cacheReadTokens += sess.totalCacheReadTokens cacheWriteTokens += sess.totalCacheWriteTokens + // Per-model output uses the same billable-output rule as the headline: + // reasoning tokens are added only where the provider reports them + // separately from output (never twice where output already includes + // them, #1075). modelBreakdown's raw token counters cannot be summed + // for display without it. A bucket no surviving call maps to falls + // back to its own counters under the session's provider. + const sessionModelOut = sessionModelBillableOutputTokens(sess) for (const [cat, d] of Object.entries(sess.categoryBreakdown)) { if (!catTotals[cat]) catTotals[cat] = { turns: 0, cost: 0, savingsUSD: 0, editTurns: 0, oneShotTurns: 0 } catTotals[cat].turns += d.turns @@ -80,12 +97,17 @@ export function buildPeriodData(label: string, projects: ProjectSummary[]): Peri catTotals[cat].oneShotTurns += d.oneShotTurns } for (const [model, d] of Object.entries(sess.modelBreakdown)) { - if (!modelTotals[model]) modelTotals[model] = { calls: 0, cost: 0, savingsUSD: 0, estimatedCostUSD: 0, tokens: 0 } - modelTotals[model].calls += d.calls - modelTotals[model].cost += d.costUSD - modelTotals[model].savingsUSD += d.savingsUSD - modelTotals[model].estimatedCostUSD += d.estimatedCostUSD ?? 0 - modelTotals[model].tokens += d.tokens.inputTokens + d.tokens.outputTokens + d.tokens.cacheReadInputTokens + d.tokens.cacheCreationInputTokens + if (!modelTotals[model]) modelTotals[model] = { calls: 0, cost: 0, savingsUSD: 0, estimatedCostUSD: 0, tokens: 0, inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 } + const acc = modelTotals[model] + acc.calls += d.calls + acc.cost += d.costUSD + acc.savingsUSD += d.savingsUSD + acc.estimatedCostUSD += d.estimatedCostUSD ?? 0 + acc.tokens += d.tokens.inputTokens + d.tokens.outputTokens + d.tokens.cacheReadInputTokens + d.tokens.cacheCreationInputTokens + acc.inputTokens += d.tokens.inputTokens + acc.outputTokens += sessionModelOut[model] ?? billableOutputTokens(inferSessionProvider(sess), d.tokens.outputTokens, d.tokens.reasoningTokens) + acc.cacheReadTokens += d.tokens.cacheReadInputTokens + acc.cacheWriteTokens += d.tokens.cacheCreationInputTokens } } @@ -109,7 +131,17 @@ export function buildPeriodData(label: string, projects: ProjectSummary[]): Peri .map(([cat, d]) => ({ name: CATEGORY_LABELS[cat as TaskCategory] ?? cat, ...d })), models: Object.entries(modelTotals) .sort(([, a], [, b]) => b.cost - a.cost) - .map(([name, d]) => ({ name, calls: d.calls, cost: d.cost, savingsUSD: d.savingsUSD, estimatedCostUSD: d.estimatedCostUSD })), + .map(([name, d]) => ({ + name, + calls: d.calls, + cost: d.cost, + savingsUSD: d.savingsUSD, + estimatedCostUSD: d.estimatedCostUSD, + inputTokens: d.inputTokens, + outputTokens: d.outputTokens, + cacheReadTokens: d.cacheReadTokens, + cacheWriteTokens: d.cacheWriteTokens, + })), unpricedModels, workflow: { corrections: corrections.corrections, diff --git a/tests/cli-status-menubar.test.ts b/tests/cli-status-menubar.test.ts index f3bfa903..e5d83710 100644 --- a/tests/cli-status-menubar.test.ts +++ b/tests/cli-status-menubar.test.ts @@ -888,6 +888,75 @@ describe('codeburn status --format menubar-json', () => { } }) + it('recomputes over a pre-change snapshot so topModels carry token counts, then reuses the fresh record without re-invalidating', async () => { + // The per-model token counts changed the payload's rendering semantics + // without changing the envelope. A snapshot written by the pre-change + // binary carries the same corpus fingerprint and query key, so a record + // that predates the fields must be rejected by the semantic key (one real + // recompute), after which the fresh record is served as-is — the third + // identical call must neither rebuild nor rewrite the snapshot. + const home = await mkdtemp(join(tmpdir(), 'codeburn-menubar-token-render-')) + + try { + const projectDir = join(home, '.claude', 'projects', 'myapp') + await mkdir(projectDir, { recursive: true }) + const now = new Date() + const todayUtcMidnight = Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate()) + const base = new Date(Math.max(todayUtcMidnight, now.getTime() - 2 * 3600_000)) + const ts = (offset: number) => new Date(base.getTime() + offset).toISOString().replace(/\.\d+Z$/, 'Z') + await writeFile( + join(projectDir, 'session.jsonl'), + [userLine('s1', ts(0)), assistantLine('s1', ts(60_000), 'msg-1')].join('\n'), + ) + + const args = ['status', '--format', 'menubar-json', '--period', 'today', '--provider', 'all', '--no-optimize'] + const first = runCli(args, home) + expect(first.status, `stderr: ${first.stderr}`).toBe(0) + const seeded = JSON.parse(first.stdout) as { current: { topModels: Array> } } + expect(seeded.current.topModels[0]?.inputTokens).toBe(500) + + const snapshotFiles = findSnapshotFiles(join(home, '.cache', 'codeburn')) + expect(snapshotFiles).toHaveLength(1) + const record = JSON.parse(await readFile(snapshotFiles[0]!, 'utf-8')) as { + semanticKey: string + payload: { current: { topModels: unknown[] } } + } + // Rewind the record to the pre-change contract: previous render version, + // topModels rows without any token counts. + record.semanticKey = record.semanticKey.replace(/:render-\d+:/, ':render-5:') + record.payload.current.topModels = [ + { name: 'Legacy Snapshot Model', cost: 9.99, calls: 4, savingsUSD: 0, savingsBaselineModel: '' }, + ] + await writeFile(snapshotFiles[0]!, JSON.stringify(record)) + + const second = runCli(args, home) + expect(second.status, `stderr: ${second.stderr}`).toBe(0) + const payload = JSON.parse(second.stdout) as { + current: { topModels: Array<{ name: string; inputTokens?: number; outputTokens?: number; cacheReadTokens?: number; cacheWriteTokens?: number }> } + } + expect(payload.current.topModels.map(model => model.name)).not.toContain('Legacy Snapshot Model') + expect(payload.current.topModels[0]).toMatchObject({ + inputTokens: 500, + outputTokens: 50, + // The helper's usage carries no cache traffic: a KNOWN zero, which the + // pre-change record could not have expressed at all. + cacheReadTokens: 0, + cacheWriteTokens: 0, + }) + + const freshRecord = await readFile(snapshotFiles[0]!, 'utf-8') + const third = runCli(args, home) + expect(third.status, `stderr: ${third.stderr}`).toBe(0) + expect(JSON.parse(third.stdout)).toEqual(payload) + // No repeated invalidation: a warm record is served, not rebuilt or + // rewritten (loadStatusSnapshot only persists settle-window bookkeeping + // on a corpus mismatch, and saveStatusSnapshot only runs on a miss). + expect(await readFile(snapshotFiles[0]!, 'utf-8')).toBe(freshRecord) + } finally { + await rm(home, { recursive: true, force: true }) + } + }) + it('reprices from an updated live LiteLLM cache instead of serving a stale snapshot', async () => { const home = await mkdtemp(join(tmpdir(), 'codeburn-menubar-pricing-gen-')) diff --git a/tests/menubar-json.test.ts b/tests/menubar-json.test.ts index fe96a440..ed3cd66e 100644 --- a/tests/menubar-json.test.ts +++ b/tests/menubar-json.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest' import { buildMenubarPayload, type CombinedUsage, type LocalModelSavings, type PeriodData, type ProviderCost } from '../src/menubar-json.js' +import { getShortModelName } from '../src/models.js' import type { OptimizeResult } from '../src/optimize.js' function emptyPeriod(label: string): PeriodData { @@ -184,6 +185,88 @@ describe('buildMenubarPayload', () => { expect(payload.current.topModels.find(m => m.name === 'k3-agent')).toBeUndefined() }) + it('merges per-model token counts under the same display-name grouping as cost', () => { + const period: PeriodData = { + label: 'Today', + cost: 0, calls: 0, sessions: 0, + inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0, + categories: [], + models: [ + { name: 'k3', cost: 2.5, calls: 78, inputTokens: 1000, outputTokens: 200, cacheReadTokens: 3000, cacheWriteTokens: 400 }, + { name: 'kimi-k3', cost: 0.5, calls: 2, inputTokens: 10, outputTokens: 20, cacheReadTokens: 30, cacheWriteTokens: 40 }, + ], + } + const payload = buildMenubarPayload(period, [], null) + const kimiK3 = payload.current.topModels.find(m => m.name === 'Kimi K3')! + expect(kimiK3.inputTokens).toBe(1010) + expect(kimiK3.outputTokens).toBe(220) + expect(kimiK3.cacheReadTokens).toBe(3030) + expect(kimiK3.cacheWriteTokens).toBe(440) + }) + + it('keeps known-zero per-model counts as zeros instead of dashes or drops', () => { + const period: PeriodData = { + label: 'Today', + cost: 0, calls: 0, sessions: 0, + inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0, + categories: [], + models: [ + { name: 'k3', cost: 0, calls: 3, inputTokens: 0, outputTokens: 0, cacheReadTokens: 500, cacheWriteTokens: 0 }, + ], + } + const payload = buildMenubarPayload(period, [], null) + const row = payload.current.topModels[0]! + expect(row.inputTokens).toBe(0) + expect(row.outputTokens).toBe(0) + expect(row.cacheReadTokens).toBe(500) + expect(row.cacheWriteTokens).toBe(0) + }) + + it('omits per-model counts for a row any legacy contributor without counts folded into', () => { + // A period assembled from older rows that never carried counts must not + // grow a plausible-looking partial sum: unknown stays absent. + const period: PeriodData = { + label: 'Today', + cost: 0, calls: 0, sessions: 0, + inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0, + categories: [], + models: [ + { name: 'k3', cost: 2.5, calls: 78, inputTokens: 1000, outputTokens: 200, cacheReadTokens: 3000, cacheWriteTokens: 400 }, + { name: 'kimi-k3', cost: 0.5, calls: 2 }, + { name: 'kimi-for-coding', cost: 0.06, calls: 13, inputTokens: 5, outputTokens: 6, cacheReadTokens: 7, cacheWriteTokens: 8 }, + ], + } + const payload = buildMenubarPayload(period, [], null) + const merged = payload.current.topModels.find(m => m.name === 'Kimi K3')! + expect(merged.inputTokens).toBeUndefined() + expect(merged.outputTokens).toBeUndefined() + expect(merged.cacheReadTokens).toBeUndefined() + expect(merged.cacheWriteTokens).toBeUndefined() + // A row whose every contributor carried counts keeps them. + const intact = payload.current.topModels.find(m => m.name === getShortModelName('kimi-for-coding'))! + expect(intact.inputTokens).toBe(5) + expect(intact.outputTokens).toBe(6) + expect(intact.cacheReadTokens).toBe(7) + expect(intact.cacheWriteTokens).toBe(8) + }) + + it('keeps merged counts unknown regardless of the order contributors arrive in', () => { + const period: PeriodData = { + label: 'Today', + cost: 0, calls: 0, sessions: 0, + inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0, + categories: [], + models: [ + { name: 'kimi-k3', cost: 0.5, calls: 2 }, + { name: 'k3', cost: 2.5, calls: 78, inputTokens: 1000, outputTokens: 200, cacheReadTokens: 3000, cacheWriteTokens: 400 }, + ], + } + const payload = buildMenubarPayload(period, [], null) + const merged = payload.current.topModels.find(m => m.name === 'Kimi K3')! + expect(merged.inputTokens).toBeUndefined() + expect(merged.cacheWriteTokens).toBeUndefined() + }) + it('caps topActivities at 20 so all task categories can surface', () => { const period: PeriodData = { label: 'Today', diff --git a/tests/menubar-model-tokens.test.ts b/tests/menubar-model-tokens.test.ts new file mode 100644 index 00000000..1cbbf1d0 --- /dev/null +++ b/tests/menubar-model-tokens.test.ts @@ -0,0 +1,233 @@ +import { mkdir, mkdtemp, rm, writeFile } from 'fs/promises' +import { tmpdir } from 'os' +import { join } from 'path' +import { beforeAll, afterEach, beforeEach, describe, expect, it } from 'vitest' + +import { getShortModelName, loadPricing, setModelAliases } from '../src/models.js' +import { buildMenubarPayloadForRange } from '../src/usage-aggregator.js' +import { clearSessionCache } from '../src/parser.js' +import type { DateRange } from '../src/types.js' + +// Per-model token counts through the menubar payload, exercised against a real +// parsed fixture (not arithmetic helpers): two priced models with unequal +// input/output/cache-read/cache-write mixes, a cache-read-only model, an +// unpriced model, the durable-day path after the sources expire, and a +// provider-scoped build. + +const FIXTURE_DAY = Date.UTC(2026, 3, 16) +const RANGE: DateRange = { + start: new Date(FIXTURE_DAY - 24 * 60 * 60 * 1000), + end: new Date(FIXTURE_DAY + 24 * 60 * 60 * 1000), +} +const PERIOD = { range: RANGE, label: 'Fixture window' } + +const SONNET = 'claude-3-7-sonnet-20250219' +const HAIKU = 'claude-3-haiku-20240307' +const OPUS = 'claude-3-opus-20240229' +const UNPRICED = 'totally-unknown-model-xyz' + +let base: string +let cacheDir: string +const tmpDirs: string[] = [] + +beforeAll(async () => { + await loadPricing() +}) + +beforeEach(() => { + // Runs AFTER the global env-isolation beforeEach, so these win for the test body. + setModelAliases({}) +}) + +afterEach(async () => { + clearSessionCache() + while (tmpDirs.length > 0) { + const d = tmpDirs.pop() + if (d) await rm(d, { recursive: true, force: true }) + } +}) + +function claudeLine(id: string, model: string, ts: string, usage: { + input: number + output: number + cacheW: number + cacheR: number +}): string { + return JSON.stringify({ + type: 'assistant', + timestamp: ts, + sessionId: `s-${id}`, + message: { + type: 'message', role: 'assistant', model, id, + content: [], + usage: { + input_tokens: usage.input, + output_tokens: usage.output, + cache_creation_input_tokens: usage.cacheW, + cache_read_input_tokens: usage.cacheR, + }, + }, + }) +} + +/** Four sessions, one model each, with deliberately unequal token mixes. */ +async function seedFixture(): Promise { + base = await mkdtemp(join(tmpdir(), 'codeburn-model-tokens-src-')) + cacheDir = await mkdtemp(join(tmpdir(), 'codeburn-model-tokens-cache-')) + tmpDirs.push(base, cacheDir) + + const projectDir = join(base, 'projects', 'p') + await mkdir(projectDir, { recursive: true }) + const t = (h: number): string => new Date(FIXTURE_DAY + h * 60 * 60 * 1000).toISOString() + const sessions: Array<{ id: string; model: string; usage: { input: number; output: number; cacheW: number; cacheR: number } }> = [ + // Two assistant turns → the counts must sum across calls of one session. + { id: 'sonnet', model: SONNET, usage: { input: 100_000, output: 20_000, cacheW: 30_000, cacheR: 400_000 } }, + { id: 'sonnet-2', model: SONNET, usage: { input: 100_000, output: 20_000, cacheW: 30_000, cacheR: 400_000 } }, + { id: 'haiku', model: HAIKU, usage: { input: 50_000, output: 10_000, cacheW: 5_000, cacheR: 100_000 } }, + // Cache-read-only: zero fresh input/output, all reused input. + { id: 'opus', model: OPUS, usage: { input: 0, output: 0, cacheW: 0, cacheR: 900_000 } }, + // Unpriced: tokens observed, pricing lookup fails → $0 attributed cost. + { id: 'unknown', model: UNPRICED, usage: { input: 7_000, output: 2_000, cacheW: 0, cacheR: 0 } }, + ] + for (const s of sessions) { + await writeFile( + join(projectDir, `${s.id}.jsonl`), + claudeLine(`msg-${s.id}`, s.model, t(1), s.usage) + '\n', + 'utf-8', + ) + } + + process.env['CLAUDE_CONFIG_DIR'] = base + process.env['CODEBURN_CACHE_DIR'] = cacheDir +} + +function rowFor(payload: { current: { topModels: Array<{ name: string }> } }, model: string): { + name: string + cost: number + calls: number + inputTokens?: number + outputTokens?: number + cacheReadTokens?: number + cacheWriteTokens?: number +} { + const row = payload.current.topModels.find(m => m.name === getShortModelName(model)) + expect(row, `topModels row for ${model}`).toBeDefined() + return row! +} + +describe('per-model token counts in the menubar payload', () => { + it('carries unequal per-model counts through the fresh parse, reconciling with the headline totals', async () => { + await seedFixture() + + clearSessionCache() + const payload = await buildMenubarPayloadForRange(PERIOD, { provider: 'all', optimize: false, timeline: false }) + + const sonnet = rowFor(payload, SONNET) + expect(sonnet.calls).toBe(2) + expect(sonnet.inputTokens).toBe(200_000) + expect(sonnet.outputTokens).toBe(40_000) + expect(sonnet.cacheReadTokens).toBe(800_000) + expect(sonnet.cacheWriteTokens).toBe(60_000) + + const haiku = rowFor(payload, HAIKU) + expect(haiku.inputTokens).toBe(50_000) + expect(haiku.outputTokens).toBe(10_000) + expect(haiku.cacheReadTokens).toBe(100_000) + expect(haiku.cacheWriteTokens).toBe(5_000) + + // Cache-only model: a known zero in every non-cache column, real reused + // input in the cache column — never folded into input, never dropped. + const opus = rowFor(payload, OPUS) + expect(opus.inputTokens).toBe(0) + expect(opus.outputTokens).toBe(0) + expect(opus.cacheReadTokens).toBe(900_000) + expect(opus.cacheWriteTokens).toBe(0) + + // Unpriced model: counts are observed usage and must survive even though + // its attributed cost is $0. + const unpriced = rowFor(payload, UNPRICED) + expect(unpriced.cost).toBe(0) + expect(unpriced.inputTokens).toBe(7_000) + expect(unpriced.outputTokens).toBe(2_000) + + // Per-model rows reconcile with the period headline on a single-provider + // fixture (claude folds reasoning into output, so billable == raw here). + const models = payload.current.topModels + expect(models.reduce((s, m) => s + (m.inputTokens ?? 0), 0)).toBe(payload.current.inputTokens) + expect(models.reduce((s, m) => s + (m.outputTokens ?? 0), 0)).toBe(payload.current.outputTokens) + expect(models.reduce((s, m) => s + (m.cacheReadTokens ?? 0), 0)).toBe(payload.current.cacheReadTokens) + expect(models.reduce((s, m) => s + (m.cacheWriteTokens ?? 0), 0)).toBe(payload.current.cacheWriteTokens) + }) + + it('carries the same counts through the durable-day path after the session files are gone', async () => { + await seedFixture() + + // Warm the daily cache, then expire the sources: the headline and the + // per-model counts must both survive off the sealed day entries. + clearSessionCache() + await buildMenubarPayloadForRange(PERIOD, { provider: 'all', optimize: false, timeline: false }) + await rm(base, { recursive: true, force: true }) + + clearSessionCache() + const payload = await buildMenubarPayloadForRange(PERIOD, { provider: 'all', optimize: false, timeline: false }) + + const sonnet = rowFor(payload, SONNET) + expect(sonnet.calls).toBe(2) + expect(sonnet.inputTokens).toBe(200_000) + expect(sonnet.outputTokens).toBe(40_000) + expect(sonnet.cacheReadTokens).toBe(800_000) + expect(sonnet.cacheWriteTokens).toBe(60_000) + + const opus = rowFor(payload, OPUS) + expect(opus.cacheReadTokens).toBe(900_000) + expect(opus.inputTokens).toBe(0) + }) + + it('emits the same counts on the provider-scoped build', async () => { + await seedFixture() + + clearSessionCache() + const payload = await buildMenubarPayloadForRange(PERIOD, { provider: 'claude', optimize: false, timeline: false }) + + const sonnet = rowFor(payload, SONNET) + expect(sonnet.inputTokens).toBe(200_000) + expect(sonnet.cacheReadTokens).toBe(800_000) + const haiku = rowFor(payload, HAIKU) + expect(haiku.outputTokens).toBe(10_000) + expect(haiku.cacheWriteTokens).toBe(5_000) + }) + + it('returns no models for a range the fixture day is outside of', async () => { + await seedFixture() + + clearSessionCache() + const before = { + range: { + start: new Date(FIXTURE_DAY - 96 * 60 * 60 * 1000), + end: new Date(FIXTURE_DAY - 72 * 60 * 60 * 1000), + }, + label: 'Before fixture', + } + const payload = await buildMenubarPayloadForRange(before, { provider: 'all', optimize: false, timeline: false }) + expect(payload.current.topModels).toEqual([]) + }) + + it('merges aliased raw ids into one row whose counts sum like the cost does', async () => { + await seedFixture() + // Route haiku through sonnet: pricing, display name and now token counts + // must all land in the sonnet row. + setModelAliases({ [HAIKU]: SONNET }) + + clearSessionCache() + const payload = await buildMenubarPayloadForRange(PERIOD, { provider: 'all', optimize: false, timeline: false }) + + const sonnet = rowFor(payload, SONNET) + expect(sonnet.calls).toBe(3) + expect(sonnet.inputTokens).toBe(250_000) + expect(sonnet.outputTokens).toBe(50_000) + expect(sonnet.cacheReadTokens).toBe(900_000) + expect(sonnet.cacheWriteTokens).toBe(65_000) + // Four fixture models, one folded away by the alias. + expect(payload.current.topModels).toHaveLength(3) + }) +})