fix(daily-cache): stop a frozen under-read from outranking the sources on disk

src/daily-cache.ts

@@ -1034,6 +1034,15 @@ function isPartialSurvival(date: string, baseline: ProviderDaySlice, fresh: Prov
   return date < settleCutoff && fresh.calls < baseline.calls
 }
 
+/// The read path's rule: keep whichever derivation explains MORE calls, on
+/// every date, with ties going to the baseline. Unlike the write path this is
+/// not deciding what to freeze, so it has no reason to prefer a thinner slice
+/// on a recent day — but it is also not the place a re-pricing lands, so an
+/// equal-call slice leaves the durable value alone.
+function baselineExplainsMore(baseline: ProviderDaySlice, fresh: ProviderDaySlice): boolean {
+  return fresh.calls <= baseline.calls
+}
+
 /// Index `freshUnderOldTz` (the same parse re-aggregated under the cache's OLD
 /// tzKey) by date then provider, so the merge can subtract exactly what the
 /// fresh parse still explains under the old bucketing.
@@ -1088,7 +1097,14 @@ export function mergeDayEntries(
   /// baseline wins on settled days (`isPartialSurvival`). The adoption union
   /// leaves it off - both sides are cache generations there and the newer
   /// schema deliberately wins per (date, provider).
-  guardPartialSurvival = false,
+  ///
+  /// `'prefer-richer'` is the read-path union's rule (`unionDaysForPeriod`):
+  /// on EVERY date, keep the slice explaining more calls, ties to the baseline.
+  /// That path is choosing what to REPORT, not what to freeze, so a thinner
+  /// derivation is never the better answer there — while the write path still
+  /// lets a recent day shrink, on the grounds that a still-settling day is
+  /// defined by its fresh parse.
+  guardPartialSurvival: boolean | 'prefer-richer' = false,
   /// Providers whose baseline slices were recorded under an accounting where a
   /// call meant something else, so a shrink is not evidence of source loss for
   /// this one re-derivation (see PENDING_REDERIVE_PROVIDERS). Only consulted
@@ -1141,7 +1157,10 @@ export function mergeDayEntries(
         }
       }
       if (existingSlice && hasSliceData(existingSlice) && !residual) {
-        if (!guardPartialSurvival || pendingRederive?.has(provider) || !isPartialSurvival(day.date, slice, existingSlice, settleCutoff)) continue
+        const keepBaseline = guardPartialSurvival === 'prefer-richer'
+          ? baselineExplainsMore(slice, existingSlice)
+          : isPartialSurvival(day.date, slice, existingSlice, settleCutoff)
+        if (!guardPartialSurvival || pendingRederive?.has(provider) || !keepBaseline) continue
         // The baseline holds more evidence than the sources can still produce:
         // swap the fresh slice back out for it (inverse of addSliceIntoDay, so
         // the day's totals and nested maps stay reconciled with its slices).

src/usage-aggregator.ts

@@ -15,7 +15,7 @@ import { scanUserCorrections, medianTimeToFirstEditMs, aggregateFileChurn, compu
 import { buildPrAttribution, aggregateByBranch } from './sessions-report.js'
 import { scanAndDetect } from './optimize.js'
 import { callBillableOutputTokens, sessionBillableOutputTokens } from './session-output.js'
-import { getDaysInRange, ensureCacheHydrated, emptyCache, BACKFILL_DAYS, toDateString, type DailyCache, type DailyEntry, type ProjectDayStats, type ProviderDaySlice } from './daily-cache.js'
+import { getDaysInRange, ensureCacheHydrated, emptyCache, mergeDayEntries, BACKFILL_DAYS, toDateString, type DailyCache, type DailyEntry, type ProjectDayStats, type ProviderDaySlice } from './daily-cache.js'
 import { buildGranularHistory } from './granular-history.js'
 
 // Row caps for the by-PR / by-branch payload aggregations, ranked by cost.
@@ -393,6 +393,10 @@ function unionDaysForPeriod(
   periodInfo: PeriodInfo,
   daysSelection: Set<string> | null,
   sliceHistorical?: (day: DailyEntry) => DailyEntry,
+  /// Historical days from the parse this period already ran. They are evidence
+  /// about the same dates the cache is answering for, so where one explains
+  /// more of a day than the other, that one is used (#1217).
+  liveHistoricalDays: DailyEntry[] = [],
 ): DailyEntry[] {
   const now = new Date()
   const yesterdayStr = toDateString(new Date(now.getFullYear(), now.getMonth(), now.getDate() - 1))
@@ -406,8 +410,24 @@ function unionDaysForPeriod(
   // never reaches the slicer (which tallies what it could not attribute).
   const selectedCacheDays = daysSelection ? cacheDays.filter(d => daysSelection.has(d.date)) : cacheDays
   const historicalDays = sliceHistorical ? selectedCacheDays.map(d => sliceHistorical(d)) : selectedCacheDays
+  // A cached day is derived once and then frozen behind the watermark, so a
+  // derivation that missed sources stays the answer forever — the Overview
+  // headline reading below the live panels beneath it (#1217). This run already
+  // parsed these dates. Reconcile the two per (date, provider), keeping
+  // whichever explains MORE calls: a cached day whose transcripts have expired
+  // still wins (nothing live can outbid it), and an under-read cached row stops
+  // suppressing evidence that is sitting on disk. Only dates the cache already
+  // holds are reconciled — filling absent dates is a separate decision each
+  // caller makes for itself.
+  const cachedDates = new Set(historicalDays.map(d => d.date))
+  const liveForCachedDates = liveHistoricalDays.filter(d =>
+    cachedDates.has(d.date) && (!daysSelection || daysSelection.has(d.date)),
+  )
+  const reconciledDays = liveForCachedDates.length > 0
+    ? mergeDayEntries(liveForCachedDates, historicalDays, true, undefined, 'prefer-richer')
+    : historicalDays
   const todayInRange = todayAllDays.filter(d => d.date >= rangeStartStr && d.date <= rangeEndStr)
-  const unfiltered = [...historicalDays, ...todayInRange].sort((a, b) => a.date.localeCompare(b.date))
+  const unfiltered = [...reconciledDays, ...todayInRange].sort((a, b) => a.date.localeCompare(b.date))
   return daysSelection ? unfiltered.filter(d => daysSelection.has(d.date)) : unfiltered
 }
 
@@ -450,7 +470,14 @@ export function buildDurableOverviewFromNormalizedIndex(
   const historicalSlice = hasProjectFilter
     ? (day: DailyEntry): DailyEntry => sliceDayToProject(day, include, exclude)
     : undefined
-  const cachedAllDays = unionDaysForPeriod(cache, todayDays, periodInfo, null, historicalSlice)
+  const cachedAllDays = unionDaysForPeriod(
+    cache,
+    todayDays,
+    periodInfo,
+    null,
+    historicalSlice,
+    normalizedDays.filter(day => day.date !== todayStr),
+  )
   const cachedDates = new Set(cache.days.map(day => day.date))
   const rangeStartStr = toDateString(periodInfo.range.start)
   const rangeEndStr = toDateString(periodInfo.range.end)
@@ -619,7 +646,11 @@ export async function buildDurablePeriod(periodInfo: PeriodInfo, opts: Aggregate
       }
     : undefined
 
-  const allDays = unionDaysForPeriod(cache, todayAllDays, periodInfo, daysSelection?.days ?? null, sliceHistorical)
+  // The period parse above already read these dates; today is excluded because
+  // the union takes it from `todayAllDays`, which re-anchors a turn straddling
+  // midnight (see the todayAllDays note above) and must stay the today source.
+  const liveHistoricalDays = aggregateProjectsIntoDays(liveProjects).filter(d => d.date < todayStr)
+  const allDays = unionDaysForPeriod(cache, todayAllDays, periodInfo, daysSelection?.days ?? null, sliceHistorical, liveHistoricalDays)
   const days = pf === 'all' ? allDays : allDays.map(d => sliceDayToProvider(d, pf))
   const data = buildPeriodDataFromDays(days, periodInfo.label)
 

tests/durable-underread-reconcile.test.ts

@@ -0,0 +1,242 @@
+import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'
+import { mkdir, rm, writeFile } from 'fs/promises'
+import { existsSync } from 'fs'
+import { tmpdir } from 'os'
+import { join } from 'path'
+
+import {
+  DAILY_CACHE_VERSION,
+  currentTzKey,
+  mergeDayEntries,
+  type DailyCache,
+  type DailyEntry,
+  type ProviderDaySlice,
+} from '../src/daily-cache.js'
+import { loadPricing } from '../src/models.js'
+import { buildDurablePeriod, buildPeriodData, getDailyCacheConfigHash } from '../src/usage-aggregator.js'
+import { parseAllSessions, filterProjectsByName, clearSessionCache } from '../src/parser.js'
+import type { DateRange } from '../src/types.js'
+
+// #1217. A day is derived into the durable cache once and then frozen behind
+// the watermark, while `isPartialSurvival` lets a fresh derivation SHRINK a day
+// inside the settle window. One parse that missed sources therefore became
+// permanent: the Overview headline sat 27% below the Daily Activity rows
+// printed underneath it, on days whose transcripts were intact the whole time.
+// The headline must never report less than the parse it is standing on.
+
+const ROOT = join(tmpdir(), `codeburn-underread-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`)
+const ENV_KEYS = ['HOME', 'CODEBURN_CACHE_DIR', 'CLAUDE_CONFIG_DIR', 'CLAUDE_CONFIG_DIRS', 'CODEX_HOME', 'USERPROFILE', 'KIMI_CODE_HOME', 'CODEBURN_DESKTOP_SESSIONS_DIR'] as const
+let savedEnv: Record<string, string | undefined>
+
+// The codex provider captures its home at import time, so redirect it before
+// module evaluation — otherwise a real ~/.codex leaks into these counts.
+vi.hoisted(() => {
+  process.env['CODEX_HOME'] = `${process.env['TMPDIR'] || '/tmp'}/codeburn-underread-codex-${process.pid}-${Date.now()}`
+})
+
+function daysAgoStr(n: number): string {
+  const d = new Date(Date.now() - n * 24 * 60 * 60 * 1000)
+  return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')}`
+}
+
+function slice(cost: number, calls: number, extra: Partial<ProviderDaySlice> = {}): ProviderDaySlice {
+  return { cost, calls, savingsUSD: 0, ...extra }
+}
+
+function dayEntry(date: string, providers: Record<string, ProviderDaySlice>, extra: Partial<DailyEntry> = {}): DailyEntry {
+  return {
+    date,
+    cost: Object.values(providers).reduce((s, p) => s + p.cost, 0),
+    savingsUSD: 0,
+    calls: Object.values(providers).reduce((s, p) => s + p.calls, 0),
+    sessions: 1,
+    inputTokens: 0,
+    outputTokens: 0,
+    cacheReadTokens: 0,
+    cacheWriteTokens: 0,
+    editTurns: 0,
+    oneShotTurns: 0,
+    models: {},
+    categories: {},
+    providers,
+    ...extra,
+  }
+}
+
+/// Four priced assistant turns on a past date — the day the cache under-read.
+async function seedHistoricalSession(date: string): Promise<void> {
+  const projectDir = join(ROOT, 'home', '.claude', 'projects', 'p')
+  await mkdir(projectDir, { recursive: true })
+  const at = (hour: number): string => new Date(`${date}T0${hour}:30:00`).toISOString()
+  const line = (id: string, hour: number): string => JSON.stringify({
+    type: 'assistant',
+    timestamp: at(hour),
+    sessionId: 's-past',
+    message: {
+      type: 'message', role: 'assistant', model: 'claude-3-5-sonnet-20241022', id,
+      content: [],
+      usage: { input_tokens: 90000, output_tokens: 12000, cache_creation_input_tokens: 0, cache_read_input_tokens: 300000 },
+    },
+  })
+  await writeFile(
+    join(projectDir, 's-past.jsonl'),
+    [line('m1', 1), line('m2', 2), line('m3', 3), line('m4', 4)].join('\n') + '\n',
+    'utf-8',
+  )
+}
+
+async function seedCache(days: DailyEntry[]): Promise<void> {
+  const cache: DailyCache = {
+    version: DAILY_CACHE_VERSION,
+    savingsConfigHash: getDailyCacheConfigHash(),
+    tzKey: currentTzKey(),
+    lastComputedDate: daysAgoStr(1),
+    days,
+    complete: true,
+    watermarkTrusted: true,
+  }
+  await writeFile(join(ROOT, 'cache', `daily-cache.v${DAILY_CACHE_VERSION}.json`), JSON.stringify(cache), 'utf-8')
+}
+
+async function liveOnly(range: DateRange): Promise<{ cost: number; calls: number }> {
+  clearSessionCache()
+  const projects = filterProjectsByName(await parseAllSessions(range, 'all'), [], [])
+  const data = buildPeriodData('live', projects)
+  return { cost: data.cost, calls: data.calls }
+}
+
+function weekRange(): DateRange {
+  const now = new Date()
+  return {
+    start: new Date(now.getFullYear(), now.getMonth(), now.getDate() - 7),
+    end: new Date(now.getFullYear(), now.getMonth(), now.getDate(), 23, 59, 59, 999),
+  }
+}
+
+beforeAll(async () => {
+  await loadPricing()
+})
+
+beforeEach(async () => {
+  savedEnv = Object.fromEntries(ENV_KEYS.map(k => [k, process.env[k]]))
+  await mkdir(join(ROOT, 'home', '.claude'), { recursive: true })
+  await mkdir(join(ROOT, 'cache'), { recursive: true })
+  await mkdir(join(ROOT, 'no-desktop-sessions'), { recursive: true })
+  await mkdir(join(ROOT, 'no-kimi-home'), { recursive: true })
+  process.env['HOME'] = join(ROOT, 'home')
+  process.env['USERPROFILE'] = join(ROOT, 'home')
+  process.env['CODEBURN_CACHE_DIR'] = join(ROOT, 'cache')
+  process.env['CLAUDE_CONFIG_DIR'] = join(ROOT, 'home', '.claude')
+  delete process.env['CLAUDE_CONFIG_DIRS']
+  process.env['KIMI_CODE_HOME'] = join(ROOT, 'no-kimi-home')
+  process.env['CODEBURN_DESKTOP_SESSIONS_DIR'] = join(ROOT, 'no-desktop-sessions')
+  clearSessionCache()
+})
+
+afterEach(async () => {
+  clearSessionCache()
+  for (const k of ENV_KEYS) {
+    if (savedEnv[k] === undefined) delete process.env[k]
+    else process.env[k] = savedEnv[k]
+  }
+  if (existsSync(ROOT)) await rm(ROOT, { recursive: true, force: true })
+})
+
+describe('a frozen under-read no longer suppresses the sources on disk', () => {
+  const past = daysAgoStr(3)
+
+  it('reports the live day when the cached row explains less of it', async () => {
+    await seedHistoricalSession(past)
+    const live = await liveOnly(weekRange())
+    expect(live.calls).toBe(4) // the fixture, proving the parse sees the whole day
+
+    // What the bad derivation froze: one of the day's four calls.
+    await seedCache([dayEntry(past, { claude: slice(live.cost / 4, 1) })])
+
+    clearSessionCache()
+    const durable = await buildDurablePeriod({ range: weekRange(), label: 'p' })
+
+    expect(durable.data.calls).toBe(live.calls)
+    expect(durable.data.cost).toBeCloseTo(live.cost, 6)
+  })
+
+  it('drops the "preserved from expired logs" mark from a day the sources fully explain', async () => {
+    await seedHistoricalSession(past)
+    const live = await liveOnly(weekRange())
+    await seedCache([dayEntry(past, { claude: slice(live.cost / 4, 1) }, { carried: true })])
+
+    clearSessionCache()
+    const durable = await buildDurablePeriod({ range: weekRange(), label: 'p' })
+
+    // The footnote claimed this day's money was preserved from expired logs
+    // while its transcript sat on disk the whole time.
+    expect(durable.carriedCostUSD).toBe(0)
+  })
+
+  it('still carries a day whose sources really are gone', async () => {
+    await seedHistoricalSession(past)
+    const live = await liveOnly(weekRange())
+    const expired = daysAgoStr(5)
+    await seedCache([
+      dayEntry(past, { claude: slice(live.cost / 4, 1) }),
+      dayEntry(expired, { claude: slice(100, 40) }, { carried: true }),
+    ])
+
+    clearSessionCache()
+    const durable = await buildDurablePeriod({ range: weekRange(), label: 'p' })
+
+    // Nothing live can outbid a day with no surviving source, so the carried
+    // day is untouched — and the healed day is added on top of it.
+    expect(durable.data.calls).toBe(live.calls + 40)
+    expect(durable.data.cost).toBeCloseTo(live.cost + 100, 6)
+    expect(durable.carriedCostUSD).toBe(100)
+  })
+
+  it('keeps a richer cached slice when the live parse only partly explains the day', async () => {
+    await seedHistoricalSession(past)
+    const live = await liveOnly(weekRange())
+    // The cache holds MORE than the sources can still produce: a genuine
+    // partial expiry, which must not be dragged down to the live figure.
+    await seedCache([dayEntry(past, { claude: slice(live.cost * 3, 40) })])
+
+    clearSessionCache()
+    const durable = await buildDurablePeriod({ range: weekRange(), label: 'p' })
+
+    expect(durable.data.calls).toBe(40)
+    expect(durable.data.cost).toBeCloseTo(live.cost * 3, 6)
+  })
+})
+
+describe("mergeDayEntries 'prefer-richer' guard mode", () => {
+  const recent = daysAgoStr(2)
+
+  it('keeps the richer slice on a recent day, where the write path takes the fresh one', () => {
+    const fresh = [dayEntry(recent, { claude: slice(20, 4) })]
+    const cached = [dayEntry(recent, { claude: slice(90, 40) })]
+
+    // Write path: a still-settling day is defined by its fresh derivation.
+    const written = mergeDayEntries(fresh, cached, true, undefined, true)[0]!
+    expect(written.providers['claude']).toMatchObject({ cost: 20, calls: 4 })
+
+    // Read path: reporting a thinner derivation is never the better answer.
+    const read = mergeDayEntries(fresh, cached, true, undefined, 'prefer-richer')[0]!
+    expect(read.providers['claude']).toMatchObject({ cost: 90, calls: 40 })
+    expect(read.cost).toBe(90)
+    expect(read.calls).toBe(40)
+  })
+
+  it('leaves the durable value alone on equal calls — re-pricing lands in the write path', () => {
+    // The same Grok re-pricing the write path applies (a cheaper cached slice,
+    // same call count) must NOT be re-applied while reporting: the read union
+    // corrects under-reads only, and the cached row stays authoritative for
+    // everything else. `dashboard-period-truth` pins the same expectation.
+    const fresh = [dayEntry(recent, { grok: slice(11.89, 21) })]
+    const cached = [dayEntry(recent, { grok: slice(3.29, 21) })]
+    const read = mergeDayEntries(fresh, cached, true, undefined, 'prefer-richer')[0]!
+    expect(read.providers['grok']).toMatchObject({ cost: 3.29, calls: 21 })
+
+    // The write path is where the new price does land.
+    const written = mergeDayEntries(fresh, cached, true, undefined, true)[0]!
+    expect(written.providers['grok']).toMatchObject({ cost: 11.89, calls: 21 })
+  })
+})