From d64b29f64dd3b32b6abdbc7a002d6715387c6f3c Mon Sep 17 00:00:00 2001 From: iamtoruk Date: Fri, 21 Aug 2026 13:17:33 -0700 Subject: [PATCH 1/2] feat(cli): rehome the cline-cli provider into the workspace layout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream `main` ships a provider this branch has no counterpart for: the Cline CLI (npm `cline`, 3.x), whose sessions live in a layout unrelated to the VS Code extension tree `cline.ts` reads. Files added on one side only produce no merge conflict, so a `main` merge would happily create `src/providers/ cline-cli.ts` — a path npm workspaces does not build and no registry imports. Green build, and a whole provider quietly does not exist. Ported from #940 (@ozymandiashh), which carried it faithfully: identical dedup keys, all 34 upstream test cases, and the clean phase-8 split — discovery and file I/O host-side, pure record decode in @codeburn/core, registered next to its neighbours and deliberately separate from the shared vscode-cline tier. Two changes on top of that port: - `observations.ts` follows the post-#1074 conventions: the model is routed through `normalizeModelIdentifier` at the observation boundary like every other provider, and measured cost is carried the way the sibling decoders carry it rather than through a cast. - the estimated-cost path reports zero web-search requests. Upstream prices that path with a hardcoded 0; letting the decoded `fetch_web_content` count reach the pricing pass would bill $0.01 per fetch on top of tokens, a billing change nobody asked for. A metered call keeps the real count — its dollar figure comes from the CLI, so nothing prices off it. DAILY_CACHE_VERSION takes 27 (MIN_SUPPORTED 27): every historical Cline CLI session contributes usage no older rollup ever contained, and usage-aggregator serves every day before today from that cache for ten years, so without the bump an upgrading user would keep cline-cli-less history forever while today's numbers silently included it. --- CHANGELOG.md | 1 + README.md | 1 + docs/providers/README.md | 1 + docs/providers/cline-cli.md | 58 ++ docs/providers/cline.md | 2 + packages/cli/src/daily-cache.ts | 13 +- packages/cli/src/parser.ts | 2 +- packages/cli/src/providers/cline-cli.ts | 177 ++++++ packages/cli/src/providers/index.ts | 3 +- packages/cli/src/session-cache.ts | 5 + packages/cli/tests/daily-cache.test.ts | 56 ++ .../tests/provider-env-declarations.test.ts | 1 + packages/cli/tests/provider-registry.test.ts | 2 +- .../cli/tests/provider-turn-grouping.test.ts | 51 ++ .../cli/tests/providers/cline-cli.test.ts | 543 ++++++++++++++++++ packages/cli/tests/setup/env-isolation.ts | 3 + packages/core/package.json | 4 + .../core/src/providers/cline-cli/decode.ts | 355 ++++++++++++ .../core/src/providers/cline-cli/index.ts | 30 + .../src/providers/cline-cli/observations.ts | 98 ++++ .../core/src/providers/cline-cli/types.ts | 83 +++ packages/core/tests/architecture-gate.test.ts | 2 + packages/core/tests/content-smuggling.test.ts | 80 +++ .../tests/providers/cline-cli-decode.test.ts | 182 ++++++ packages/core/tsup.config.ts | 1 + 25 files changed, 1749 insertions(+), 5 deletions(-) create mode 100644 docs/providers/cline-cli.md create mode 100644 packages/cli/src/providers/cline-cli.ts create mode 100644 packages/cli/tests/providers/cline-cli.test.ts create mode 100644 packages/core/src/providers/cline-cli/decode.ts create mode 100644 packages/core/src/providers/cline-cli/index.ts create mode 100644 packages/core/src/providers/cline-cli/observations.ts create mode 100644 packages/core/src/providers/cline-cli/types.ts create mode 100644 packages/core/tests/providers/cline-cli-decode.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 31aa12b63..178c8ebbb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Added (CLI) - `codeburn sync push --attribution` (opt-in): sends git attribution spans — the session→commit correlation from `codeburn yield` (`codeburn.session.attribution` and `codeburn.commit` span types with normalized repo remote, commit SHAs, merged/reverted state, and PR links). Nothing new is sent without the flag; local-only repos and Windows filesystem paths are never emitted as repo identities, and sessions whose project path no longer resolves never inherit the push-time working directory's repo. See docs/sync/README.md "Git attribution". +- **Cline CLI provider** — the standalone Cline command-line agent (npm `cline`, 3.x), separate from the VS Code extension's [Cline](docs/providers/cline.md) provider. Reads session metadata and rolled-up usage from `~/.cline/data/sessions//.json`, and per-message `metrics` (input, output, cache read/write, cost) from the co-located `.messages.json`; cost is read per message rather than estimated. (#940, thanks @ozymandiashh) ### Changed (@codeburn/core — breaking; version bump deferred to the next release, which must take at least a minor under 0.x) - **The `model` field is bounded at the observation boundary.** `model` (and the optional `pricingModel`) on a `CallObservation` must now match the identifier charset `[A-Za-z0-9._:/@-]`, max 128 chars, in both the zod schema and the published `schemas/observation-0.2.0.json`. Every provider's `toObservations` normalizes through `normalizeModelIdentifier`, so a provider display name — Antigravity's `"Gemini 3.5 Flash (High)"`, Warp's and Devin's display strings — surfaces as `unknown` in an emitted observation rather than rejecting the whole envelope. **CLI output is unaffected:** normalization happens only at the observation boundary, so terminal, dashboard, menubar, and desktop numbers and model labels are byte-identical to before. The bound is a tightening of `observation-0.2.0` in place; an ARCHIVED pre-hardening 0.2.0 envelope whose `model` held a display name now fails validation against that same version string and must be re-normalized before re-validating. diff --git a/README.md b/README.md index f6d04b6bd..fb96671a9 100644 --- a/README.md +++ b/README.md @@ -665,6 +665,7 @@ These are starting points, not verdicts. A 60% cache hit on a single experimenta | **Pi / OMP** | `~/.pi/agent/sessions//*.jsonl` (Pi), `~/.omp/agent/sessions//*.jsonl` (OMP) | Each assistant message carries usage (input, output, cacheRead, cacheWrite) plus inline `toolCall` blocks. Tool names normalize to the standard set (`bash` → `Bash`, `dispatch_agent` → `Agent`); bash commands come from `toolCall.arguments.command`. | | **Codebuff** (formerly Manicode) | `~/.config/manicode/projects//chats//chat-messages.json` (honors `CODEBUFF_DATA_DIR`; walks `manicode-dev` / `manicode-staging`) | Bills in credits, so each completed assistant message is costed at the public rate of $0.01/credit via `msg.credits`. When an upstream provider's stashed RunState records token-level usage (`message.metadata.runState.sessionState.mainAgentState.messageHistory[*].providerOptions`), the real tokens and LiteLLM cost take precedence. Native tool names (`read_files`, `str_replace`, `run_terminal_command`, `spawn_agents`) normalize to `Read`, `Edit`, `Bash`, `Agent`. | | **Cline / Roo Code / KiloCode** | VS Code `globalStorage`: Cline at `saoudrizwan.claude-dev` and `~/.cline/data`; Roo Code and KiloCode across VS Code, VS Code Insiders, and VSCodium | Cline-family agents. CodeBurn reads `ui_messages.json` from each task directory, extracting token counts from `type: "say"` entries with `say: "api_req_started"`. | +| **Cline CLI** | `~/.cline/data/sessions//` (honors `CLINE_SESSION_DATA_DIR`, `CLINE_DATA_DIR`, `CLINE_DIR`) | The Cline command-line agent, whose layout is unrelated to the VS Code extension's. Reads `.json` for session metadata and the rolled-up `usage`, and `.messages.json` for the per-message `metrics` block (input, output, cacheRead, cacheWrite, cost) that becomes one call each. | | **CodeWhale** | `~/.codewhale/sessions/*.json` plus unmigrated legacy `~/.deepseek/sessions/*.json`; `$CODEWHALE_HOME/sessions` is an exact override | Emits one cumulative record per saved session. CodeWhale exposes only `total_tokens`, so CodeBurn preserves that aggregate in the input column rather than inventing an input/output split. Cost is the exact stored parent-session plus subagent USD total; model pricing is used only when the cost snapshot is absent. Tool blocks, shell commands, skills, and subagent types are retained. | | **IBM Bob** | `User/globalStorage/ibm.bob-code/tasks//` (GA `IBM Bob` and preview `Bob-IDE` app folders) | Reads `ui_messages.json` for API request token/cost records and `api_conversation_history.json` for the selected model. | | **Kimi Code CLI** | `$KIMI_SHARE_DIR/sessions///` or `~/.kimi/sessions///` | Reads `wire.jsonl` `StatusUpdate.token_usage` records, mapping `input_other`, `input_cache_read`, `input_cache_creation`, and `output` into the standard token columns; includes subagents under each session's `subagents/` folder. | diff --git a/docs/providers/README.md b/docs/providers/README.md index f4d2aa4ce..3aae178a4 100644 --- a/docs/providers/README.md +++ b/docs/providers/README.md @@ -12,6 +12,7 @@ For the architectural picture, see `../architecture.md`. |---|---|---|---| | [Claude](claude.md) | JSONL (no parser) | `src/providers/claude.ts` | none (covered indirectly) | | [Cline](cline.md) | JSON | `src/providers/cline.ts` | `tests/providers/cline.test.ts` | +| [Cline CLI](cline-cli.md) | JSON | `src/providers/cline-cli.ts` | `tests/providers/cline-cli.test.ts` | | [CodeWhale](codewhale.md) | JSON | `src/providers/codewhale.ts` | `tests/providers/codewhale.test.ts` | | [Codex](codex.md) | JSONL | `src/providers/codex.ts` | `tests/providers/codex.test.ts` | | [Copilot](copilot.md) | JSONL + SQLite (OTel) + Nitrite .db (JetBrains) | `src/providers/copilot.ts` | `tests/providers/copilot.test.ts` | diff --git a/docs/providers/cline-cli.md b/docs/providers/cline-cli.md new file mode 100644 index 000000000..a6daac224 --- /dev/null +++ b/docs/providers/cline-cli.md @@ -0,0 +1,58 @@ +# Cline CLI + +The Cline command-line agent (npm `cline`, 3.x). Separate from the [Cline](cline.md) provider, which reads the VS Code extension's task tree. + +- **Source:** `src/providers/cline-cli.ts` +- **Loading:** eager (`src/providers/index.ts:3`) +- **Test:** `tests/providers/cline-cli.test.ts` + +## Where it reads from + +One root, resolved exactly as the CLI resolves it — each level independently overridable: + +| Level | Env var | Default | +|---|---|---| +| sessions | `CLINE_SESSION_DATA_DIR` | `/sessions` | +| data | `CLINE_DATA_DIR` | `/data` | +| root | `CLINE_DIR` | `~/.cline` | + +A directory is a session only when it contains `/.json`. `probeRoots()` reports the resolved sessions dir, so `codeburn doctor` distinguishes "CLI not installed" from "override pointing somewhere else". + +## Storage format + +``` +sessions// + .json metadata + rolled-up usage + .messages.json per-message metrics +``` + +`.json` carries `session_id`, `provider`, `model`, `cwd`, `workspace_root`, `started_at` / `ended_at`, `messages_path`, and a `metadata.usage` rollup (`inputTokens`, `outputTokens`, `cacheReadTokens`, `cacheWriteTokens`, `totalCost`). + +`.messages.json` holds `{ version, updated_at, agent, sessionId, messages[], system_prompt }`. Assistant messages carry Anthropic-style content blocks (`thinking` / `text` / `tool_use`) plus: + +```jsonc +"modelInfo": { "id": "z-ai/glm-5.2", "provider": "cline-pass" }, +"metrics": { "inputTokens": 6937, "outputTokens": 213, + "cacheReadTokens": 0, "cacheWriteTokens": 0, "cost": 0.002108502 } +``` + +One `metrics` block becomes one parsed call. Dedup key: `cline-cli::`. + +## Caching + +None at the provider level; the metadata file is the cached source path and the normal parser/cache layers apply. + +## Quirks + +- **`provider` in the session file is the upstream LLM route** (e.g. `cline-pass`), not the tool. The codeburn provider name is always `cline-cli`. +- **Model strings are not normalized by the CLI.** The same model appears as `z-ai/glm-5.2`, `cline-pass/glm-5.2`, and `GLM-5.2` across sessions, so pricing lookups may need a `model-alias`. +- **Cost is reported per message**, so `costIsEstimated` is false on the normal path; it falls back to `calculateCost` only when a message omits `cost`. +- **Rollup fallback.** A session whose messages carry no metrics (interrupted, or an older layout) emits a single call from `metadata.usage`. This reads `usage`, deliberately *not* `aggregateUsage` / `aggregatedAgentsCost`, which fold in spawned subagents that are themselves separate session directories and would double count. +- **`messages_path` is absolute** and goes stale when a session directory is copied between machines, so the co-located `.messages.json` is preferred and `messages_path` is only the fallback. +- **Tool names differ from the extension's.** `run_commands`, `read_files`, `search_codebase`, `editor`, `apply_patch`, `fetch_web_content`, `skills`, `spawn_agent`, and the `team_*` family. `run_commands` carries a JSON-encoded array of command lines in a single string field. + +## When fixing a bug here + +1. Reproduce with a minimal session directory: `.json` plus `.messages.json`. +2. Run `tests/providers/cline-cli.test.ts`. +3. This provider shares no code with `vscode-cline-parser.ts` — changes here cannot affect Cline, Roo Code, KiloCode, or IBM Bob. diff --git a/docs/providers/cline.md b/docs/providers/cline.md index 65f27eae4..594467f7b 100644 --- a/docs/providers/cline.md +++ b/docs/providers/cline.md @@ -2,6 +2,8 @@ Cline VS Code extension and Cline home-data task storage. +Sessions from the Cline **command-line** agent use an unrelated layout and are handled by [Cline CLI](cline-cli.md); this provider does not see them. + - **Source:** `src/providers/cline.ts` - **Loading:** eager (`src/providers/index.ts:2`) - **Test:** `tests/providers/cline.test.ts` diff --git a/packages/cli/src/daily-cache.ts b/packages/cli/src/daily-cache.ts index c8a6037dd..0461589ee 100644 --- a/packages/cli/src/daily-cache.ts +++ b/packages/cli/src/daily-cache.ts @@ -5,6 +5,15 @@ import { homedir } from 'os' import { join } from 'path' import type { DateRange, ProjectSummary } from './types.js' +// Bumped to 27: the Cline CLI (npm `cline`, 3.x) is a NEW provider, so every +// historical session under ~/.cline/data/sessions contributes usage that no +// older rollup ever contained. Those files were never scanned before they were +// parsed, so nothing downstream can notice on its own: `usage-aggregator` +// serves every day before today from this cache and retention is ten years, so +// an upgrading user with a warm complete cache would keep cline-cli-less +// history forever while freshly reparsed sessions disagreed with it. Raising +// MIN_SUPPORTED_VERSION forces the one-time re-derivation. +// // Bumped to 26: kiro chat-file input tokens are now estimated from every // human turn's full text instead of the last 500 chars (#909), so a v25 // rollup finalized by the pre-fix binary carries kiro costs off by up to @@ -100,8 +109,8 @@ import type { DateRange, ProjectSummary } from './types.js' // that older binaries skipped. v8 added local-model savings to the daily // rollup; the `savingsConfigHash` field is invalidated separately when the // user changes their `localModelSavings` mapping. -export const DAILY_CACHE_VERSION = 26 -const MIN_SUPPORTED_VERSION = 26 +export const DAILY_CACHE_VERSION = 27 +const MIN_SUPPORTED_VERSION = 27 // Version-suffixed so different binaries each own a distinct file and never // clobber an incompatible schema. Bumping the version mints a fresh filename; // adoptOlderDailyCaches then unions days out of every previous file (including diff --git a/packages/cli/src/parser.ts b/packages/cli/src/parser.ts index 8928af657..e0cb64263 100644 --- a/packages/cli/src/parser.ts +++ b/packages/cli/src/parser.ts @@ -1029,7 +1029,7 @@ function providerCallToCachedCall(call: ParsedProviderCall): CachedCall { webSearchRequests: call.webSearchRequests, cacheCreationOneHourTokens: 0, }, - costUSD: (call.provider === 'mistral-vibe' || call.provider === 'antigravity' || call.provider === 'devin' || call.provider === 'vercel-gateway' || call.provider === 'hermes' || call.provider === 'kiro' || call.provider === 'codewhale' || call.provider === 'quickdesk') ? call.costUSD : undefined, + costUSD: (call.provider === 'mistral-vibe' || call.provider === 'antigravity' || call.provider === 'devin' || call.provider === 'vercel-gateway' || call.provider === 'hermes' || call.provider === 'kiro' || call.provider === 'codewhale' || call.provider === 'quickdesk' || call.provider === 'cline-cli') ? call.costUSD : undefined, isEstimated: call.costIsEstimated || undefined, speed: call.speed, timestamp: call.timestamp, diff --git a/packages/cli/src/providers/cline-cli.ts b/packages/cli/src/providers/cline-cli.ts new file mode 100644 index 000000000..aa09bf228 --- /dev/null +++ b/packages/cli/src/providers/cline-cli.ts @@ -0,0 +1,177 @@ +import { readdir } from 'fs/promises' +import { homedir } from 'os' +import { basename, join } from 'path' + +import { decodeClineCli, clineCliToolNameMap } from '@codeburn/core/providers/cline-cli' +import type { ClineCliDecodedCall, ClineCliSessionRecords } from '@codeburn/core/providers/cline-cli' + +import { extractBashCommands } from '../bash-utils.js' +import { readSessionFile } from '../fs-utils.js' +import { getShortModelName } from '../models.js' +import { createBridgedProvider } from './bridge.js' +import type { Provider, ProbeRoot, SessionSource, ParsedProviderCall } from './types.js' + +const PROVIDER_NAME = 'cline-cli' +const DISPLAY_NAME = 'Cline CLI' + +// Mirrors the CLI's own resolution chain, each level individually overridable: +// sessions := CLINE_SESSION_DATA_DIR ?? /sessions +// data := CLINE_DATA_DIR ?? /data +// root := CLINE_DIR ?? ~/.cline +function clineRootDir(): string { + return process.env['CLINE_DIR']?.trim() || join(homedir(), '.cline') +} + +function clineDataDir(): string { + return process.env['CLINE_DATA_DIR']?.trim() || join(clineRootDir(), 'data') +} + +export function getClineCliSessionsDir(): string { + return process.env['CLINE_SESSION_DATA_DIR']?.trim() || join(clineDataDir(), 'sessions') +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value) +} + +function nonEmptyString(value: unknown): string | undefined { + return typeof value === 'string' && value.trim() ? value.trim() : undefined +} + +function projectName(workspace: string | undefined): string { + if (!workspace) return DISPLAY_NAME + const parts = workspace.replace(/[\\/]+$/, '').split(/[\\/]/).filter(Boolean) + return parts.at(-1) ?? DISPLAY_NAME +} + +async function readJson(path: string): Promise { + const raw = await readSessionFile(path) + if (raw === null) return null + try { + return JSON.parse(raw) as unknown + } catch { + return null + } +} + +// Map one rich, cost-free decoder call into the host's ParsedProviderCall. Cost +// re-enters here: a CLI-reported meter figure (present only when actually +// metered, a metered $0 included) is carried as `costBasis: 'measured'`; a +// missing/negative cost falls back to `costBasis: 'estimated'` so the parser.ts +// pricing pass fills `costUSD` from the token buckets — byte-identical to the +// pre-migration in-decoder `calculateCost` (Phase 0, Pattern B). Bash base-name +// extraction (and its `strip-ansi` dependency) stays CLI-side: the core decoder +// carries the raw command strings; the host reduces them to base names here. +function toProviderCall(rich: ClineCliDecodedCall): ParsedProviderCall { + const measured = rich.reportedCost !== undefined + return { + provider: 'cline-cli', + model: rich.model, + inputTokens: rich.inputTokens, + outputTokens: rich.outputTokens, + cacheCreationInputTokens: rich.cacheCreationInputTokens, + cacheReadInputTokens: rich.cacheReadInputTokens, + cachedInputTokens: rich.cachedInputTokens, + reasoningTokens: rich.reasoningTokens, + // Upstream prices the estimated path with a hardcoded 0 web-search requests, + // so the decoded count must not reach the pricing pass (which bills + // $0.01 each). A metered call carries the CLI's own dollar figure, so its + // real count rides along without touching cost. + webSearchRequests: measured ? rich.webSearchRequests : 0, + ...(measured + ? { costUSD: rich.reportedCost, costBasis: 'measured' as const } + : { costBasis: 'estimated' as const }), + costIsEstimated: !measured, + tools: rich.tools, + // Same flat list the pre-migration decode produced (no Set): per-command + // counts keep matching upstream behavior. + bashCommands: rich.rawBashCommands.flatMap(c => extractBashCommands(c)), + skills: rich.skills.length > 0 ? rich.skills : undefined, + subagentTypes: rich.subagentTypes.length > 0 ? rich.subagentTypes : undefined, + timestamp: rich.timestamp, + speed: rich.speed, + deduplicationKey: rich.deduplicationKey, + turnId: rich.turnId, + toolSequence: rich.toolSequence, + userMessage: rich.userMessage, + sessionId: rich.sessionId, + project: rich.project, + ...(rich.projectPath ? { projectPath: rich.projectPath } : {}), + ...(rich.workingDirectory ? { workingDirectory: rich.workingDirectory } : {}), + } +} + +export function createClineCliProvider(overrideDir?: string): Provider { + const sessionsDir = (): string => overrideDir ?? getClineCliSessionsDir() + + return createBridgedProvider({ + name: PROVIDER_NAME, + displayName: DISPLAY_NAME, + + modelDisplayName(model: string): string { + return getShortModelName(model) + }, + + toolDisplayName(rawTool: string): string { + return clineCliToolNameMap[rawTool] ?? rawTool + }, + + async probeRoots(): Promise { + return [{ path: sessionsDir(), label: 'Cline CLI sessions' }] + }, + + async discoverSessions(): Promise { + const dir = sessionsDir() + const entries = await readdir(dir, { withFileTypes: true }).catch(() => []) + const sources: SessionSource[] = [] + + for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) { + if (!entry.isDirectory()) continue + const sessionId = entry.name + const metaPath = join(dir, sessionId, `${sessionId}.json`) + const meta = await readJson(metaPath) + if (!isRecord(meta)) continue + + const workspace = nonEmptyString(meta['workspace_root']) ?? nonEmptyString(meta['cwd']) + sources.push({ + path: metaPath, + project: projectName(workspace), + provider: PROVIDER_NAME, + }) + } + + return sources + }, + + // I/O adapter: read + JSON-parse the session metadata file and its + // co-located messages file (falling back to the recorded absolute path, + // which is stale once a session directory is copied between machines), then + // hand the core decoder ONE composite { meta, messages } record. The + // decoder stays path-free: the session-id basename fallback and the + // discovered project label are injected here. + async readRecords(source: SessionSource): Promise { + const meta = await readJson(source.path) + if (!isRecord(meta)) return null + if (nonEmptyString(meta['session_id']) === undefined) { + meta['session_id'] = basename(source.path).replace(/\.json$/, '') + } + meta['project'] = source.project + + const sibling = join(source.path.replace(/\.json$/, '') + '.messages.json') + let doc = await readJson(sibling) + if (!isRecord(doc)) { + const recorded = nonEmptyString(meta['messages_path']) + if (recorded) doc = await readJson(recorded) + } + + const messages = isRecord(doc) && Array.isArray(doc['messages']) ? doc['messages'] : [] + const record: ClineCliSessionRecords = { meta, messages } + return [record] + }, + + decode: decodeClineCli, + toProviderCall, + }) +} + +export const clineCli = createClineCliProvider() diff --git a/packages/cli/src/providers/index.ts b/packages/cli/src/providers/index.ts index 07dc3903a..8c5380199 100644 --- a/packages/cli/src/providers/index.ts +++ b/packages/cli/src/providers/index.ts @@ -1,5 +1,6 @@ import { claude } from './claude.js' import { cline } from './cline.js' +import { clineCli } from './cline-cli.js' import { codewhale } from './codewhale.js' import { codebuff } from './codebuff.js' import { codex } from './codex.js' @@ -190,7 +191,7 @@ async function loadZed(): Promise { } } -const coreProviders: Provider[] = [claude, cline, codewhale, codebuff, codex, copilot, devin, droid, gemini, hermes, ibmBob, kiloCode, kiro, kimi, kimicode, lingtaiTui, mistralVibe, mux, openclaw, openDesign, pi, omp, qwen, quickdesk, rooCode, zerostack, grok] +const coreProviders: Provider[] = [claude, cline, clineCli, codewhale, codebuff, codex, copilot, devin, droid, gemini, hermes, ibmBob, kiloCode, kiro, kimi, kimicode, lingtaiTui, mistralVibe, mux, openclaw, openDesign, pi, omp, qwen, quickdesk, rooCode, zerostack, grok] // Lazily loaded providers, listed by name so --provider validation works even // when an optional module fails to load. Must stay in sync with getAllProviders. diff --git a/packages/cli/src/session-cache.ts b/packages/cli/src/session-cache.ts index afd9afb53..53f2e88ac 100644 --- a/packages/cli/src/session-cache.ts +++ b/packages/cli/src/session-cache.ts @@ -189,6 +189,7 @@ const TEMP_FILE_MAX_AGE_MS = 5 * 60 * 1000 // merge instead of drop. export const PROVIDER_ENV_VARS: Record = { claude: ['CLAUDE_CONFIG_DIRS', 'CLAUDE_CONFIG_DIR', 'CODEBURN_DESKTOP_SESSIONS_DIR', 'APPDATA', 'LOCALAPPDATA'], + 'cline-cli': ['CLINE_SESSION_DATA_DIR', 'CLINE_DATA_DIR', 'CLINE_DIR'], codebuff: ['CODEBUFF_DATA_DIR'], codewhale: ['CODEWHALE_HOME'], codex: ['CODEX_HOME'], @@ -277,6 +278,10 @@ export const PROVIDER_PARSE_VERSIONS: Record = { // the new optional fields. claude: 'advisor-usage-v1-skills-rich-capture-v1-cross-provider-pr-v1', cline: 'worktree-project-grouping-v1', + // reported-cost-v1: the CLI reports its own per-message cost, so entries + // cached before cline-cli joined the reported-cost allowlist in parser.ts + // hold costUSD: undefined and get re-priced from tokens on every read. + 'cline-cli': 'reported-cost-v1', codewhale: 'aggregate-session-v1-est-cost', // Bump when the Codex parser changes attribution so unchanged, already-cached // session files re-parse (session-cache.json serves them without invoking the diff --git a/packages/cli/tests/daily-cache.test.ts b/packages/cli/tests/daily-cache.test.ts index 345f866a2..c5a80f917 100644 --- a/packages/cli/tests/daily-cache.test.ts +++ b/packages/cli/tests/daily-cache.test.ts @@ -621,3 +621,59 @@ describe('ensureCacheHydrated: schema version invalidation (#873)', () => { expect(hydrated.days.find(d => d.date === '2026-06-11')?.cost).toBe(18.2) }) }) + +// The Cline CLI is a NEW provider: every historical session under +// ~/.cline/data/sessions contributes usage no v26 rollup ever contained. +// usage-aggregator serves every day before today from this cache and retention +// is ten years, so without the bump an upgrading user keeps cline-cli-less +// history forever while today's numbers silently include it. +describe('ensureCacheHydrated: schema version invalidation (cline-cli)', () => { + function clineCliDay(date: string, cost: number, calls: number): DailyEntry { + return { + ...emptyDay(date, cost, calls), + sessions: 1, + providers: { 'cline-cli': { cost, calls, savingsUSD: 0, sessions: 1, inputTokens: 500, outputTokens: 200 } }, + } + } + + it('re-derives a warm complete v26 cache instead of serving its cline-cli-less totals', async () => { + vi.useFakeTimers() + vi.setSystemTime(new Date('2026-06-12T12:00:00.000Z')) + + const { writeFile, mkdir } = await import('fs/promises') + await mkdir(TMP_CACHE_ROOT, { recursive: true }) + // A cache exactly as the pre-bump release left it: the schema current at + // the time (v26), finalized off a complete parse, watermark at yesterday, + // matching tz and savings hash. Nothing but the version bump can + // invalidate it: revert DAILY_CACHE_VERSION to 26 and this file becomes the + // active one again, served complete and frozen (parseCalls stays 0). + const v26 = { + version: 26, + savingsConfigHash: '', + tzKey: currentTzKey(), + lastComputedDate: '2026-06-11', + days: [emptyDay('2026-06-11', 4.55, 1)], + complete: true, + watermarkTrusted: true, + } + await writeFile(join(TMP_CACHE_ROOT, 'daily-cache.v26.json'), JSON.stringify(v26), 'utf-8') + + let parseCalls = 0 + const hydrated = await ensureCacheHydrated( + async () => { + parseCalls += 1 + return [] + }, + () => [clineCliDay('2026-06-11', 9.1, 3)], + ) + + expect(parseCalls).toBe(1) + const day = hydrated.days.find(d => d.date === '2026-06-11') + expect(day?.cost).toBe(9.1) + expect(day?.providers['cline-cli']?.cost).toBe(9.1) + expect(hydrated.version).toBe(DAILY_CACHE_VERSION) + expect(hydrated.complete).toBe(true) + // The v26 file is never rewritten or deleted — old binaries still own it. + expect(JSON.parse(await readFile(join(TMP_CACHE_ROOT, 'daily-cache.v26.json'), 'utf-8')).version).toBe(26) + }) +}) diff --git a/packages/cli/tests/provider-env-declarations.test.ts b/packages/cli/tests/provider-env-declarations.test.ts index 648b09556..6003ae150 100644 --- a/packages/cli/tests/provider-env-declarations.test.ts +++ b/packages/cli/tests/provider-env-declarations.test.ts @@ -28,6 +28,7 @@ import { getAllProviders } from '../src/providers/index.js' // the guard: add it, with the provider(s) the reads serve. const FILE_PROVIDERS: Record = { 'claude.ts': ['claude'], + 'cline-cli.ts': ['cline-cli'], 'codebuff.ts': ['codebuff'], 'codewhale.ts': ['codewhale'], 'codex.ts': ['codex'], diff --git a/packages/cli/tests/provider-registry.test.ts b/packages/cli/tests/provider-registry.test.ts index 515efa7ee..4ec70764d 100644 --- a/packages/cli/tests/provider-registry.test.ts +++ b/packages/cli/tests/provider-registry.test.ts @@ -14,7 +14,7 @@ function fakeProvider(name: string, discover: Provider['discoverSessions']): Pro describe('provider registry', () => { it('has core providers registered synchronously', () => { - expect(providers.map(p => p.name)).toEqual(['claude', 'cline', 'codewhale', 'codebuff', 'codex', 'copilot', 'devin', 'droid', 'gemini', 'hermes', 'ibm-bob', 'kilo-code', 'kiro', 'kimi', 'kimicode', 'lingtai-tui', 'mistral-vibe', 'mux', 'openclaw', 'open-design', 'pi', 'omp', 'qwen', 'quickdesk', 'roo-code', 'zerostack', 'grok']) + expect(providers.map(p => p.name)).toEqual(['claude', 'cline', 'cline-cli', 'codewhale', 'codebuff', 'codex', 'copilot', 'devin', 'droid', 'gemini', 'hermes', 'ibm-bob', 'kilo-code', 'kiro', 'kimi', 'kimicode', 'lingtai-tui', 'mistral-vibe', 'mux', 'openclaw', 'open-design', 'pi', 'omp', 'qwen', 'quickdesk', 'roo-code', 'zerostack', 'grok']) }) it('codebuff tool display names normalize codebuff-native names to canonical set', () => { diff --git a/packages/cli/tests/provider-turn-grouping.test.ts b/packages/cli/tests/provider-turn-grouping.test.ts index 0f950eb68..8aea1e626 100644 --- a/packages/cli/tests/provider-turn-grouping.test.ts +++ b/packages/cli/tests/provider-turn-grouping.test.ts @@ -319,4 +319,55 @@ describe('provider turn range filtering', () => { expect(session.totalInputTokens).toBe(180) expect(session.totalOutputTokens).toBe(50) }) + + it('preserves Cline CLI reported cost through cache conversion instead of re-pricing from tokens', async () => { + const sessionsDir = join(home, '.cline', 'data', 'sessions') + const sessionId = '1785701058566_vnwtz' + const dir = join(sessionsDir, sessionId) + await mkdir(dir, { recursive: true }) + process.env['CLINE_SESSION_DATA_DIR'] = sessionsDir + + // A large token count paired with a deliberately tiny reported cost: any + // token-based re-pricing would land orders of magnitude above $0.0123, + // so passing means the CLI's own per-message cost survived the round trip. + await writeFile(join(dir, `${sessionId}.json`), JSON.stringify({ + version: 1, + session_id: sessionId, + source: 'cli', + status: 'completed', + provider: 'cline-pass', + model: 'z-ai/glm-5.2', + cwd: '/Users/test/project-a', + workspace_root: '/Users/test/project-a', + started_at: '2026-05-16T10:00:00.000Z', + ended_at: '2026-05-16T10:01:00.000Z', + metadata: {}, + })) + await writeFile(join(dir, `${sessionId}.messages.json`), JSON.stringify({ + version: 1, + agent: 'lead', + sessionId, + messages: [ + { id: 'u1', role: 'user', content: [{ type: 'text', text: 'do the thing' }], ts: Date.parse('2026-05-16T10:00:00.000Z') }, + { + id: 'a1', + role: 'assistant', + content: [{ type: 'text', text: 'done' }], + ts: Date.parse('2026-05-16T10:00:30.000Z'), + modelInfo: { id: 'z-ai/glm-5.2', provider: 'cline-pass' }, + metrics: { inputTokens: 500000, outputTokens: 20000, cacheReadTokens: 100000, cacheWriteTokens: 0, cost: 0.0123 }, + }, + ], + })) + + try { + const parseAllSessions = await loadParser() + const projects = await parseAllSessions(dayRange(), 'cline-cli') + const session = projects[0]!.sessions[0]! + + expect(session.totalCostUSD).toBeCloseTo(0.0123, 8) + } finally { + delete process.env['CLINE_SESSION_DATA_DIR'] + } + }) }) diff --git a/packages/cli/tests/providers/cline-cli.test.ts b/packages/cli/tests/providers/cline-cli.test.ts new file mode 100644 index 000000000..1d197f85a --- /dev/null +++ b/packages/cli/tests/providers/cline-cli.test.ts @@ -0,0 +1,543 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { mkdtemp, mkdir, writeFile, rm } from 'fs/promises' +import { join } from 'path' +import { tmpdir } from 'os' + +import { clineCli, createClineCliProvider, getClineCliSessionsDir } from '../../src/providers/cline-cli.js' +import { priceProviderCall } from '../../src/pricing-pass.js' +import type { ParsedProviderCall } from '../../src/providers/types.js' + +let tmpDir: string + +type MessageSpec = { + role: 'user' | 'assistant' + text?: string + metrics?: Record + model?: string + ts?: number + toolUse?: { name: string; input: Record } +} + +async function writeSession(sessionsDir: string, sessionId: string, opts?: { + messages?: MessageSpec[] + usage?: Record + totalCost?: number + model?: string + workspaceRoot?: string + cwd?: string + startedAt?: string + endedAt?: string + messagesPath?: string + omitMeta?: boolean + omitMessagesFile?: boolean +}): Promise { + const dir = join(sessionsDir, sessionId) + await mkdir(dir, { recursive: true }) + const metaPath = join(dir, `${sessionId}.json`) + const messagesPath = join(dir, `${sessionId}.messages.json`) + + if (!opts?.omitMeta) { + const metadata: Record = {} + if (opts?.usage) metadata['usage'] = opts.usage + if (opts?.totalCost !== undefined) metadata['totalCost'] = opts.totalCost + + await writeFile(metaPath, JSON.stringify({ + version: 1, + session_id: sessionId, + source: 'cli', + status: 'completed', + provider: 'cline-pass', + model: opts?.model ?? 'z-ai/glm-5.2', + cwd: opts?.cwd ?? '/Users/dev/work/my-repo', + workspace_root: opts?.workspaceRoot ?? opts?.cwd ?? '/Users/dev/work/my-repo', + started_at: opts?.startedAt ?? '2026-08-02T20:04:18.628Z', + ended_at: opts?.endedAt ?? '2026-08-02T20:08:27.768Z', + metadata, + messages_path: opts?.messagesPath ?? messagesPath, + })) + } + + if (!opts?.omitMessagesFile) { + const messages = (opts?.messages ?? []).map((spec, index) => { + const content: unknown[] = [] + if (spec.text) content.push({ type: 'text', text: spec.text }) + if (spec.toolUse) content.push({ type: 'tool_use', id: `call_${index}`, ...spec.toolUse }) + + const message: Record = { + id: `msg_${index}`, + role: spec.role, + content, + ts: spec.ts ?? 1785701064304 + index * 1000, + } + if (spec.metrics) message['metrics'] = spec.metrics + if (spec.model) message['modelInfo'] = { id: spec.model, provider: 'cline-pass' } + return message + }) + + await writeFile(messagesPath, JSON.stringify({ + version: 1, updated_at: opts?.endedAt, agent: 'lead', sessionId, messages, system_prompt: 'sp', + })) + } + + return dir +} + +async function collect(sessionsDir: string): Promise { + const provider = createClineCliProvider(sessionsDir) + const sources = await provider.discoverSessions() + const seenKeys = new Set() + const calls: ParsedProviderCall[] = [] + for (const source of sources) { + for await (const call of provider.createSessionParser(source, seenKeys).parse()) calls.push(priceProviderCall(call)) + } + return calls +} + +beforeEach(async () => { + tmpDir = await mkdtemp(join(tmpdir(), 'cline-cli-test-')) +}) + +afterEach(async () => { + await rm(tmpDir, { recursive: true, force: true }) +}) + +describe('cline-cli provider - identity', () => { + it('registers under its own provider name', () => { + expect(clineCli.name).toBe('cline-cli') + expect(clineCli.displayName).toBe('Cline CLI') + }) + + it('maps CLI tool names onto codeburn canonical names', () => { + expect(clineCli.toolDisplayName('run_commands')).toBe('Bash') + expect(clineCli.toolDisplayName('read_files')).toBe('Read') + expect(clineCli.toolDisplayName('search_codebase')).toBe('Grep') + expect(clineCli.toolDisplayName('apply_patch')).toBe('Edit') + expect(clineCli.toolDisplayName('spawn_agent')).toBe('Agent') + // Unknown tools pass through rather than being dropped. + expect(clineCli.toolDisplayName('team_mission_log')).toBe('team_mission_log') + }) +}) + +describe('cline-cli provider - sessions dir resolution', () => { + beforeEach(() => { + delete process.env['CLINE_DIR'] + delete process.env['CLINE_DATA_DIR'] + delete process.env['CLINE_SESSION_DATA_DIR'] + }) + + it('defaults to ~/.cline/data/sessions', () => { + expect(getClineCliSessionsDir()).toBe(join(process.env['HOME'] ?? '', '.cline', 'data', 'sessions')) + }) + + it('honors CLINE_DIR', () => { + process.env['CLINE_DIR'] = '/custom/root' + expect(getClineCliSessionsDir()).toBe(join('/custom/root', 'data', 'sessions')) + }) + + it('honors CLINE_DATA_DIR over CLINE_DIR', () => { + process.env['CLINE_DIR'] = '/custom/root' + process.env['CLINE_DATA_DIR'] = '/custom/data' + expect(getClineCliSessionsDir()).toBe(join('/custom/data', 'sessions')) + }) + + it('honors CLINE_SESSION_DATA_DIR over everything else', () => { + process.env['CLINE_DIR'] = '/custom/root' + process.env['CLINE_DATA_DIR'] = '/custom/data' + process.env['CLINE_SESSION_DATA_DIR'] = '/custom/sessions' + expect(getClineCliSessionsDir()).toBe('/custom/sessions') + }) + + it('reports the resolved root for doctor', async () => { + process.env['CLINE_SESSION_DATA_DIR'] = '/custom/sessions' + expect(await clineCli.probeRoots?.()).toEqual([{ path: '/custom/sessions', label: 'Cline CLI sessions' }]) + }) +}) + +describe('cline-cli provider - discovery', () => { + it('discovers one source per session directory', async () => { + await writeSession(tmpDir, 'sess-a') + await writeSession(tmpDir, 'sess-b') + + const sources = await createClineCliProvider(tmpDir).discoverSessions() + + expect(sources).toHaveLength(2) + expect(sources.map(s => s.provider)).toEqual(['cline-cli', 'cline-cli']) + expect(sources[0]?.path).toBe(join(tmpDir, 'sess-a', 'sess-a.json')) + }) + + it('names the project from the workspace root', async () => { + await writeSession(tmpDir, 'sess-a', { workspaceRoot: '/Users/dev/work/awesome-repo' }) + + const [source] = await createClineCliProvider(tmpDir).discoverSessions() + + expect(source?.project).toBe('awesome-repo') + }) + + it('skips directories without a session metadata file', async () => { + await mkdir(join(tmpDir, 'not-a-session'), { recursive: true }) + await writeSession(tmpDir, 'sess-a') + + const sources = await createClineCliProvider(tmpDir).discoverSessions() + + expect(sources).toHaveLength(1) + }) + + it('skips a session whose metadata file is corrupt', async () => { + const dir = join(tmpDir, 'sess-bad') + await mkdir(dir, { recursive: true }) + await writeFile(join(dir, 'sess-bad.json'), '{ not json') + + expect(await createClineCliProvider(tmpDir).discoverSessions()).toHaveLength(0) + }) + + it('returns nothing when the sessions dir does not exist', async () => { + expect(await createClineCliProvider(join(tmpDir, 'missing')).discoverSessions()).toHaveLength(0) + }) +}) + +describe('cline-cli provider - parsing', () => { + it('emits one call per assistant message carrying metrics', async () => { + await writeSession(tmpDir, 'sess-a', { + messages: [ + { role: 'user', text: 'do the thing' }, + { role: 'assistant', text: 'ok', metrics: { inputTokens: 100, outputTokens: 10, cacheReadTokens: 5, cacheWriteTokens: 2, cost: 0.01 } }, + { role: 'user', text: '' }, + { role: 'assistant', text: 'done', metrics: { inputTokens: 200, outputTokens: 20, cacheReadTokens: 0, cacheWriteTokens: 0, cost: 0.02 } }, + ], + }) + + const calls = await collect(tmpDir) + + expect(calls).toHaveLength(2) + expect(calls.map(c => c.inputTokens)).toEqual([100, 200]) + expect(calls.map(c => c.outputTokens)).toEqual([10, 20]) + expect(calls[0]?.cacheReadInputTokens).toBe(5) + expect(calls[0]?.cacheCreationInputTokens).toBe(2) + expect(calls.map(c => c.costUSD)).toEqual([0.01, 0.02]) + expect(calls.every(c => c.costIsEstimated === false)).toBe(true) + expect(calls.every(c => c.provider === 'cline-cli')).toBe(true) + }) + + it('carries session identity, project and timestamps onto each call', async () => { + await writeSession(tmpDir, 'sess-a', { + workspaceRoot: '/Users/dev/work/awesome-repo', + cwd: '/Users/dev/work/awesome-repo/sub', + messages: [{ role: 'assistant', text: 'hi', ts: 1785701064304, metrics: { inputTokens: 1, outputTokens: 1, cost: 0.1 } }], + }) + + const [call] = await collect(tmpDir) + + expect(call?.sessionId).toBe('sess-a') + expect(call?.project).toBe('awesome-repo') + expect(call?.projectPath).toBe('/Users/dev/work/awesome-repo') + expect(call?.workingDirectory).toBe('/Users/dev/work/awesome-repo/sub') + expect(call?.timestamp).toBe(new Date(1785701064304).toISOString()) + }) + + it('prefers the per-message model over the session model', async () => { + await writeSession(tmpDir, 'sess-a', { + model: 'session-model', + messages: [ + { role: 'assistant', text: 'a', model: 'z-ai/glm-5.2', metrics: { inputTokens: 1, outputTokens: 1, cost: 0.1 } }, + { role: 'assistant', text: 'b', metrics: { inputTokens: 1, outputTokens: 1, cost: 0.1 } }, + ], + }) + + const calls = await collect(tmpDir) + + expect(calls.map(c => c.model)).toEqual(['z-ai/glm-5.2', 'session-model']) + }) + + it('extracts tools and bash commands from tool_use blocks', async () => { + await writeSession(tmpDir, 'sess-a', { + messages: [ + { + role: 'assistant', text: 'running', metrics: { inputTokens: 1, outputTokens: 1, cost: 0.1 }, + toolUse: { name: 'run_commands', input: { commands: JSON.stringify(['git status', 'ls -la']) } }, + }, + { + role: 'assistant', text: 'reading', metrics: { inputTokens: 1, outputTokens: 1, cost: 0.1 }, + toolUse: { name: 'read_files', input: { path: '/tmp/a.ts' } }, + }, + ], + }) + + const calls = await collect(tmpDir) + + expect(calls[0]?.tools).toEqual(['Bash']) + expect(calls[0]?.bashCommands).toContain('git') + expect(calls[0]?.bashCommands).toContain('ls') + expect(calls[1]?.tools).toEqual(['Read']) + expect(calls[1]?.toolSequence?.[0]?.[0]).toEqual({ tool: 'Read', file: '/tmp/a.ts' }) + }) + + it('treats a non-JSON commands string as a single command', async () => { + await writeSession(tmpDir, 'sess-a', { + messages: [{ + role: 'assistant', text: 'x', metrics: { inputTokens: 1, outputTokens: 1, cost: 0.1 }, + toolUse: { name: 'run_commands', input: { commands: 'git status' } }, + }], + }) + + const [call] = await collect(tmpDir) + + expect(call?.bashCommands).toContain('git') + }) + + it('uses the first user text as the session user message, skipping tool results', async () => { + await writeSession(tmpDir, 'sess-a', { + messages: [ + { role: 'user', text: 'the real prompt' }, + { role: 'assistant', text: 'ok', metrics: { inputTokens: 1, outputTokens: 1, cost: 0.1 } }, + ], + }) + + const [call] = await collect(tmpDir) + + expect(call?.userMessage).toBe('the real prompt') + }) + + it('deduplicates repeated parses via the shared seenKeys set', async () => { + await writeSession(tmpDir, 'sess-a', { + messages: [{ role: 'assistant', text: 'a', metrics: { inputTokens: 5, outputTokens: 1, cost: 0.1 } }], + }) + + const provider = createClineCliProvider(tmpDir) + const [source] = await provider.discoverSessions() + const seenKeys = new Set() + + const first: ParsedProviderCall[] = [] + for await (const call of provider.createSessionParser(source!, seenKeys).parse()) first.push(call) + const second: ParsedProviderCall[] = [] + for await (const call of provider.createSessionParser(source!, seenKeys).parse()) second.push(call) + + expect(first).toHaveLength(1) + expect(second).toHaveLength(0) + }) + + it('estimates cost when the message reports none', async () => { + await writeSession(tmpDir, 'sess-a', { + messages: [{ role: 'assistant', text: 'a', metrics: { inputTokens: 1000, outputTokens: 100 } }], + }) + + const [call] = await collect(tmpDir) + + expect(call?.costIsEstimated).toBe(true) + expect(call?.costUSD).toBeGreaterThan(0) + }) + + it('keeps a metered $0 cost reported instead of re-estimating it', async () => { + await writeSession(tmpDir, 'sess-a', { + messages: [{ role: 'assistant', text: 'a', metrics: { inputTokens: 1000, outputTokens: 100, cost: 0 } }], + }) + + const [call] = await collect(tmpDir) + + expect(call?.costUSD).toBe(0) + expect(call?.costIsEstimated).toBe(false) + }) + + it('treats a negative cost as absent rather than reporting a clamped $0', async () => { + await writeSession(tmpDir, 'sess-a', { + messages: [{ role: 'assistant', text: 'a', metrics: { inputTokens: 1000, outputTokens: 100, cost: -5 } }], + }) + + const [call] = await collect(tmpDir) + + expect(call?.costIsEstimated).toBe(true) + expect(call?.costUSD).toBeGreaterThan(0) + }) + + it('promotes a seconds-resolution timestamp instead of landing in 1970', async () => { + const seconds = Math.floor(Date.parse('2026-08-02T20:04:18.000Z') / 1000) + await writeSession(tmpDir, 'sess-a', { + messages: [{ role: 'assistant', text: 'a', ts: seconds, metrics: { inputTokens: 1, outputTokens: 1, cost: 0.1 } }], + }) + + const [call] = await collect(tmpDir) + + expect(call?.timestamp).toBe('2026-08-02T20:04:18.000Z') + }) + + it('falls back to the session start when a message carries no timestamp', async () => { + await writeSession(tmpDir, 'sess-a', { + startedAt: '2026-08-02T20:04:18.628Z', + messages: [{ role: 'assistant', text: 'a', ts: 0, metrics: { inputTokens: 1, outputTokens: 1, cost: 0.1 } }], + }) + + const [call] = await collect(tmpDir) + + expect(call?.timestamp).toBe('2026-08-02T20:04:18.628Z') + }) + + it('survives a messages file whose messages field is not an array', async () => { + const dir = join(tmpDir, 'sess-a') + await writeSession(tmpDir, 'sess-a', { messages: [] }) + await writeFile(join(dir, 'sess-a.messages.json'), JSON.stringify({ version: 1, messages: { nope: true } })) + + expect(await collect(tmpDir)).toHaveLength(0) + }) + + it('survives a corrupt messages file without dropping the session rollup', async () => { + const dir = join(tmpDir, 'sess-a') + await writeSession(tmpDir, 'sess-a', { + usage: { inputTokens: 100, outputTokens: 10, totalCost: 0.05 }, + messages: [], + }) + await writeFile(join(dir, 'sess-a.messages.json'), '{ not json') + + const calls = await collect(tmpDir) + + expect(calls).toHaveLength(1) + expect(calls[0]?.inputTokens).toBe(100) + }) + + it('ignores assistant messages with no usage at all', async () => { + await writeSession(tmpDir, 'sess-a', { + messages: [ + { role: 'assistant', text: 'no metrics here' }, + { role: 'assistant', text: 'zeroed', metrics: { inputTokens: 0, outputTokens: 0, cost: 0 } }, + ], + }) + + expect(await collect(tmpDir)).toHaveLength(0) + }) + + it('reads the co-located messages file when messages_path is stale', async () => { + await writeSession(tmpDir, 'sess-a', { + messagesPath: '/nonexistent/other-machine/sess-a.messages.json', + messages: [{ role: 'assistant', text: 'a', metrics: { inputTokens: 7, outputTokens: 1, cost: 0.1 } }], + }) + + const calls = await collect(tmpDir) + + expect(calls).toHaveLength(1) + expect(calls[0]?.inputTokens).toBe(7) + }) +}) + +describe('cline-cli provider - rollup fallback', () => { + it('falls back to the session rollup when no message carries metrics', async () => { + await writeSession(tmpDir, 'sess-a', { + omitMessagesFile: true, + usage: { inputTokens: 5483, outputTokens: 133, cacheReadTokens: 50, cacheWriteTokens: 0, totalCost: 0.0081984 }, + }) + + const calls = await collect(tmpDir) + + expect(calls).toHaveLength(1) + expect(calls[0]?.inputTokens).toBe(5483) + expect(calls[0]?.outputTokens).toBe(133) + expect(calls[0]?.cacheReadInputTokens).toBe(50) + expect(calls[0]?.costUSD).toBeCloseTo(0.0081984, 7) + expect(calls[0]?.costIsEstimated).toBe(false) + }) + + it('does not double count when per-message metrics already covered the session', async () => { + await writeSession(tmpDir, 'sess-a', { + usage: { inputTokens: 300, outputTokens: 30, totalCost: 0.03 }, + messages: [ + { role: 'assistant', text: 'a', metrics: { inputTokens: 100, outputTokens: 10, cost: 0.01 } }, + { role: 'assistant', text: 'b', metrics: { inputTokens: 200, outputTokens: 20, cost: 0.02 } }, + ], + }) + + const calls = await collect(tmpDir) + + expect(calls).toHaveLength(2) + expect(calls.reduce((sum, c) => sum + c.inputTokens, 0)).toBe(300) + }) + + it('does not fire the rollup when a duplicated session_id deduped every per-message call', async () => { + // A session directory copied on disk: two dirs sharing the same internal + // session_id and message ids, the second also carrying a metadata.usage + // rollup. The shared dedup suppresses the copy's per-message calls; the + // rollup must not then fire and re-count the session. Regression for #894. + for (const [dirName, withRollup] of [['aaa', false], ['bbb', true]] as const) { + const dir = join(tmpDir, dirName) + await mkdir(dir, { recursive: true }) + const metadata: Record = {} + if (withRollup) metadata['usage'] = { inputTokens: 100, outputTokens: 10, totalCost: 0.01 } + await writeFile(join(dir, `${dirName}.json`), JSON.stringify({ + version: 1, session_id: 'shared', source: 'cli', status: 'completed', + provider: 'cline-pass', model: 'z-ai/glm-5.2', + cwd: '/Users/dev/work/my-repo', workspace_root: '/Users/dev/work/my-repo', + started_at: '2026-08-02T20:04:18.628Z', ended_at: '2026-08-02T20:08:27.768Z', + metadata, messages_path: join(dir, `${dirName}.messages.json`), + })) + await writeFile(join(dir, `${dirName}.messages.json`), JSON.stringify({ + version: 1, sessionId: 'shared', messages: [{ + id: 'msg_0', role: 'assistant', content: [{ type: 'text', text: 'a' }], + ts: 1785701064304, metrics: { inputTokens: 100, outputTokens: 10, cost: 0.01 }, + modelInfo: { id: 'z-ai/glm-5.2', provider: 'cline-pass' }, + }], + })) + } + + const calls = await collect(tmpDir) + + // Exactly one call (the first copy's msg_0); the copy is deduped and its + // rollup declined, so cost stays $0.01 rather than doubling to $0.02. + expect(calls).toHaveLength(1) + expect(calls.some(c => c.deduplicationKey === 'cline-cli:shared:rollup')).toBe(false) + expect(calls.reduce((sum, c) => sum + c.costUSD, 0)).toBeCloseTo(0.01, 7) + }) + + it('keeps a metered $0 rollup reported instead of re-estimating it', async () => { + await writeSession(tmpDir, 'sess-a', { + omitMessagesFile: true, + usage: { inputTokens: 1000, outputTokens: 100, totalCost: 0 }, + }) + + const calls = await collect(tmpDir) + + expect(calls).toHaveLength(1) + expect(calls[0]?.costUSD).toBe(0) + expect(calls[0]?.costIsEstimated).toBe(false) + }) + + it('estimates a rollup that reports no cost at all', async () => { + await writeSession(tmpDir, 'sess-a', { + omitMessagesFile: true, + usage: { inputTokens: 1000, outputTokens: 100 }, + }) + + const calls = await collect(tmpDir) + + expect(calls).toHaveLength(1) + expect(calls[0]?.costIsEstimated).toBe(true) + expect(calls[0]?.costUSD).toBeGreaterThan(0) + }) + + it('keeps web-search requests out of the estimated-cost path', async () => { + const fetchTool = { name: 'fetch_web_content', input: { url: 'https://example.com' } } + await writeSession(tmpDir, 'sess-a', { + messages: [ + { role: 'user', text: 'go' }, + { role: 'assistant', text: 'ok', metrics: { inputTokens: 1000, outputTokens: 100 }, toolUse: fetchTool }, + ], + }) + await writeSession(tmpDir, 'sess-b', { + messages: [ + { role: 'user', text: 'go' }, + { role: 'assistant', text: 'ok', metrics: { inputTokens: 1000, outputTokens: 100, cost: 0.02 }, toolUse: fetchTool }, + ], + }) + + const calls = await collect(tmpDir) + const estimated = calls.find(c => c.costIsEstimated) + const metered = calls.find(c => !c.costIsEstimated) + + expect(estimated?.tools).toContain('WebFetch') + // Upstream prices this path with a hardcoded 0 requests; routing the real + // count into the pricing pass would bill $0.01 per fetch on top of tokens. + expect(estimated?.webSearchRequests).toBe(0) + expect(metered?.webSearchRequests).toBe(1) + }) + + it('emits nothing for a session with neither message metrics nor a rollup', async () => { + await writeSession(tmpDir, 'sess-a', { omitMessagesFile: true }) + + expect(await collect(tmpDir)).toHaveLength(0) + }) +}) diff --git a/packages/cli/tests/setup/env-isolation.ts b/packages/cli/tests/setup/env-isolation.ts index 525e9c520..42f2aebac 100644 --- a/packages/cli/tests/setup/env-isolation.ts +++ b/packages/cli/tests/setup/env-isolation.ts @@ -49,6 +49,9 @@ const CLEARED = [ // Provider session-discovery dirs 'CLAUDE_CONFIG_DIR', 'CLAUDE_CONFIG_DIRS', + 'CLINE_DIR', + 'CLINE_DATA_DIR', + 'CLINE_SESSION_DATA_DIR', 'CODEX_HOME', 'CODEWHALE_HOME', 'CRUSH_GLOBAL_DATA', diff --git a/packages/core/package.json b/packages/core/package.json index 6260aa8c5..827e911eb 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -41,6 +41,10 @@ "types": "./dist/providers/claude/index.d.ts", "import": "./dist/providers/claude/index.js" }, + "./providers/cline-cli": { + "types": "./dist/providers/cline-cli/index.d.ts", + "import": "./dist/providers/cline-cli/index.js" + }, "./providers/codebuff": { "types": "./dist/providers/codebuff/index.d.ts", "import": "./dist/providers/codebuff/index.js" diff --git a/packages/core/src/providers/cline-cli/decode.ts b/packages/core/src/providers/cline-cli/decode.ts new file mode 100644 index 000000000..4f6d695df --- /dev/null +++ b/packages/core/src/providers/cline-cli/decode.ts @@ -0,0 +1,355 @@ +// @codeburn/core Cline CLI decoder: pure decode over the composite +// { meta, messages } record the host hands in. No fs / env / clock — the host +// reads and JSON-parses the metadata + messages files and passes one record +// through. The rich output carries token buckets + the CLI's reported cost but +// NO pricing (cost leaves the decoder; the host prices via its measured / +// estimated seam) and NO bash base-name extraction (that, with its `strip-ansi` +// dependency, stays host-side). +// +// Cline CLI is a "simple file-based" provider: one session directory is one +// logical session, the host re-reads both files every run (no incremental +// cache), so the decoder is a single pass with no serializable resume state. +// The only cross-record memory it needs is the pending user message (threaded +// within the one pass) and the cross-file dedup set (threaded live by the host, +// exactly like qwen). + +import type { DecodeContext } from '../../contracts.js' +import type { RecordDiagnostic } from '../../diagnostics.js' +import type { ClineCliDecodedCall, ClineCliSessionRecords, ClineCliToolCall } from './types.js' + +export const PROVIDER_NAME = 'cline-cli' + +// Cline CLI tool names mapped to the canonical vocabulary. A name with no +// mapping passes through unchanged so a provider-native tool still shows up. +export const clineCliToolNameMap: Record = { + run_commands: 'Bash', + read_files: 'Read', + editor: 'Edit', + apply_patch: 'Edit', + search_codebase: 'Grep', + fetch_web_content: 'WebFetch', + skills: 'Skill', + spawn_agent: 'Agent', + team_spawn_teammate: 'Agent', + team_run_task: 'Agent', + ask_question: 'AskUser', +} + +function mapToolName(rawTool: string): string { + return clineCliToolNameMap[rawTool] ?? rawTool +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value) +} + +function nonEmptyString(value: unknown): string | undefined { + return typeof value === 'string' && value.trim() ? value.trim() : undefined +} + +function safeNonNegativeNumber(value: unknown): number { + return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : 0 +} + +function safeTokenCount(value: unknown): number { + return Math.floor(Math.min(safeNonNegativeNumber(value), Number.MAX_SAFE_INTEGER)) +} + +// A cost counts as metered only when it is actually present and non-negative. +// `0` is a legitimate metered value (a free/cached call) and must stay reported, +// so this is a presence check, not a truthiness check. +function isReportedCost(value: unknown): boolean { + return typeof value === 'number' && Number.isFinite(value) && value >= 0 +} + +type ParsedMetrics = { + inputTokens: number + outputTokens: number + cacheReadTokens: number + cacheWriteTokens: number + /** CLI-reported dollar cost, present only when actually metered (incl. $0). */ + reportedCost?: number +} + +function parseMetrics(value: unknown): ParsedMetrics | null { + if (!isRecord(value)) return null + const metrics: ParsedMetrics = { + inputTokens: safeTokenCount(value['inputTokens']), + outputTokens: safeTokenCount(value['outputTokens']), + cacheReadTokens: safeTokenCount(value['cacheReadTokens']), + cacheWriteTokens: safeTokenCount(value['cacheWriteTokens']), + // A negative cost is not a credit we can represent — treat it as absent and + // fall back to token pricing, rather than reporting a clamped $0 as metered. + ...(isReportedCost(value['cost']) ? { reportedCost: safeNonNegativeNumber(value['cost']) } : {}), + } + const hasTokens = metrics.inputTokens > 0 || metrics.outputTokens > 0 + || metrics.cacheReadTokens > 0 || metrics.cacheWriteTokens > 0 + return hasTokens || metrics.reportedCost !== undefined && metrics.reportedCost > 0 ? metrics : null +} + +// The CLI writes epoch milliseconds, but a seconds-resolution value would +// otherwise silently land in 1970. Promote it and reject what stays +// implausible, matching the guard kiro.ts uses on the same hazard. +const MIN_REASONABLE_TIMESTAMP_MS = 1_000_000_000_000 + +function isoTimestamp(value: unknown, fallback: string): string { + if (typeof value === 'number' && Number.isFinite(value) && value > 0) { + const ms = value < MIN_REASONABLE_TIMESTAMP_MS ? value * 1000 : value + const date = new Date(ms) + if (!Number.isNaN(date.getTime()) && date.getTime() >= MIN_REASONABLE_TIMESTAMP_MS) { + return date.toISOString() + } + } + const parsed = nonEmptyString(value) + if (parsed) { + const date = new Date(parsed) + if (!Number.isNaN(date.getTime())) return date.toISOString() + } + return fallback +} + +// `run_commands` carries its commands as a JSON-encoded array in a string +// field; anything else is treated as a single command line. +function commandsFrom(input: unknown): string[] { + if (!isRecord(input)) return [] + const raw = input['commands'] ?? input['command'] + if (Array.isArray(raw)) return raw.filter((c): c is string => typeof c === 'string') + const text = nonEmptyString(raw) + if (!text) return [] + if (text.startsWith('[')) { + try { + const parsed = JSON.parse(text) as unknown + if (Array.isArray(parsed)) return parsed.filter((c): c is string => typeof c === 'string') + } catch { + // Not JSON after all - fall through and treat the whole string as one command. + } + } + return [text] +} + +function firstString(input: unknown, keys: string[]): string | undefined { + if (!isRecord(input)) return undefined + for (const key of keys) { + const value = nonEmptyString(input[key]) + if (value) return value + } + return undefined +} + +type CollectedTools = { + tools: string[] + rawBashCommands: string[] + toolSequence: ClineCliToolCall[][] + skills: string[] + subagentTypes: string[] + webSearchRequests: number +} + +function collectTools(content: unknown): CollectedTools { + const collected: CollectedTools = { + tools: [], rawBashCommands: [], toolSequence: [], skills: [], subagentTypes: [], webSearchRequests: 0, + } + if (!Array.isArray(content)) return collected + + const turnTools: ClineCliToolCall[] = [] + for (const block of content) { + if (!isRecord(block) || block['type'] !== 'tool_use') continue + const rawName = nonEmptyString(block['name']) + if (!rawName) continue + const mapped = mapToolName(rawName) + const input = block['input'] + const toolCall: ClineCliToolCall = { tool: mapped } + + const file = firstString(input, ['path', 'file_path', 'paths', 'file']) + if (file) toolCall.file = file + + if (mapped === 'Bash') { + const commands = commandsFrom(input) + const [first] = commands + if (first) toolCall.command = first + // Raw command strings travel host-side; base-name extraction (with its + // `strip-ansi` dependency) stays in the CLI adapter's toProviderCall. + for (const command of commands) collected.rawBashCommands.push(command) + } + if (mapped === 'Skill') { + const skill = firstString(input, ['name', 'skill', 'skill_name']) + if (skill) collected.skills.push(skill) + } + if (mapped === 'Agent') { + const subagentType = firstString(input, ['agent', 'agent_type', 'type', 'name']) + if (subagentType) collected.subagentTypes.push(subagentType) + } + if (mapped === 'WebFetch') collected.webSearchRequests++ + + collected.tools.push(mapped) + turnTools.push(toolCall) + } + + if (turnTools.length > 0) collected.toolSequence.push(turnTools) + return collected +} + +function textFromContent(content: unknown): string { + if (typeof content === 'string') return content + if (!Array.isArray(content)) return '' + for (const block of content) { + if (!isRecord(block) || block['type'] !== 'text') continue + const text = nonEmptyString(block['text']) + if (text) return text + } + return '' +} + +function firstUserMessage(messages: unknown[]): string { + for (const message of messages) { + if (!isRecord(message) || message['role'] !== 'user') continue + const text = textFromContent(message['content']) + // Tool results come back as role:user too; they carry no text block. + if (text) return text + } + return '' +} + +export type ClineCliDecodeInput = { + records: unknown[] + context: DecodeContext + // Optional live dedup set the host mutates in place (its shared cross-file + // seenKeys). Threaded exactly like qwen's live set. Simple file-based + // providers never persist resume state, so there is no serialized fallback. + seenKeys?: Set +} + +export type ClineCliDecodeResult = { + calls: ClineCliDecodedCall[] + diagnostics: RecordDiagnostic[] +} + +/** + * Decode one Cline CLI session's composite record into rich, cost-free calls. + * Emits one call per assistant message carrying a metrics block; when no + * message carries metrics at all, falls back to the session rollup + * (`metadata.usage`, deliberately NOT `aggregateUsage`, which folds in spawned + * subagents that are themselves separate session directories). + * + * Dedup is keyed on `cline-cli::` (per-message) and + * `cline-cli::rollup` against the live `seenKeys` set (host-owned). + */ +// `context` is part of the decode contract but the rich layer never consumes it: +// minimization / fingerprinting happens in toObservations. +export function decodeClineCli({ records, seenKeys: liveSeen }: ClineCliDecodeInput): ClineCliDecodeResult { + const seen = liveSeen ?? new Set() + const calls: ClineCliDecodedCall[] = [] + const diagnostics: RecordDiagnostic[] = [] + + const envelope = records[0] + if (!isRecord(envelope)) return { calls, diagnostics } + const recordsShape = envelope as unknown as ClineCliSessionRecords + const meta = isRecord(recordsShape.meta) ? recordsShape.meta : null + if (!meta) return { calls, diagnostics } + const messages = Array.isArray(recordsShape.messages) ? recordsShape.messages : [] + + const sessionId = nonEmptyString(meta['session_id']) ?? '' + const metadata = isRecord(meta['metadata']) ? meta['metadata'] : {} + const workspace = nonEmptyString(meta['workspace_root']) ?? nonEmptyString(meta['cwd']) + const sessionModel = nonEmptyString(meta['model']) ?? 'unknown' + const startedAt = isoTimestamp(meta['started_at'], new Date(0).toISOString()) + // Always injected by the CLI adapter's readRecords (from the discovered + // source); falls back to the display name only if that ever changes. + const project = nonEmptyString(meta['project']) ?? 'Cline CLI' + const cwd = nonEmptyString(meta['cwd']) + + const userMessage = firstUserMessage(messages) + // Whether the session carried any per-message metrics at all. Set before + // the dedup check below so a session whose calls were all deduped (e.g. a + // duplicated session directory reusing a session_id) still declines the + // rollup fallback rather than double-counting its cost through it. + let hadMetrics = false + + for (const [index, message] of messages.entries()) { + if (!isRecord(message) || message['role'] !== 'assistant') continue + const metrics = parseMetrics(message['metrics']) + if (!metrics) continue + hadMetrics = true + + const modelInfo = isRecord(message['modelInfo']) ? message['modelInfo'] : {} + const model = nonEmptyString(modelInfo['id']) ?? sessionModel + const messageId = nonEmptyString(message['id']) ?? String(index) + const deduplicationKey = `${PROVIDER_NAME}:${sessionId}:${messageId}` + if (seen.has(deduplicationKey)) continue + seen.add(deduplicationKey) + + const { tools, rawBashCommands, toolSequence, skills, subagentTypes, webSearchRequests } + = collectTools(message['content']) + + calls.push({ + provider: PROVIDER_NAME, + model, + inputTokens: metrics.inputTokens, + outputTokens: metrics.outputTokens, + cacheCreationInputTokens: metrics.cacheWriteTokens, + cacheReadInputTokens: metrics.cacheReadTokens, + cachedInputTokens: 0, + reasoningTokens: 0, + webSearchRequests, + ...(metrics.reportedCost !== undefined ? { reportedCost: metrics.reportedCost } : {}), + tools, + rawBashCommands, + skills: skills.length > 0 ? skills : [], + subagentTypes: subagentTypes.length > 0 ? subagentTypes : [], + timestamp: isoTimestamp(message['ts'], startedAt), + speed: 'standard', + deduplicationKey, + turnId: `${sessionId}:${messageId}`, + toolSequence: toolSequence.length > 0 ? toolSequence : undefined, + userMessage, + sessionId, + project, + projectPath: workspace, + ...(cwd ? { workingDirectory: cwd } : {}), + }) + } + + if (hadMetrics) return { calls, diagnostics } + + // No per-message metrics: fall back to the session rollup so an interrupted + // or older session still reports its spend. + const rollup = parseMetrics(isRecord(metadata['usage']) ? metadata['usage'] : null) + if (!rollup) return { calls, diagnostics } + const deduplicationKey = `${PROVIDER_NAME}:${sessionId}:rollup` + if (seen.has(deduplicationKey)) return { calls, diagnostics } + seen.add(deduplicationKey) + + // Same presence-not-truthiness rule as the per-message path: a metered $0 + // rollup stays reported instead of being re-estimated from tokens. + const rawRollupCost = (isRecord(metadata['usage']) ? metadata['usage']['totalCost'] : undefined) + ?? metadata['totalCost'] + const rollupReportedCost = isReportedCost(rawRollupCost) ? safeNonNegativeNumber(rawRollupCost) : undefined + + calls.push({ + provider: PROVIDER_NAME, + model: sessionModel, + inputTokens: rollup.inputTokens, + outputTokens: rollup.outputTokens, + cacheCreationInputTokens: rollup.cacheWriteTokens, + cacheReadInputTokens: rollup.cacheReadTokens, + cachedInputTokens: 0, + reasoningTokens: 0, + webSearchRequests: 0, + ...(rollupReportedCost !== undefined ? { reportedCost: rollupReportedCost } : {}), + tools: [], + rawBashCommands: [], + skills: [], + subagentTypes: [], + timestamp: isoTimestamp(meta['ended_at'], startedAt), + speed: 'standard', + deduplicationKey, + turnId: `${sessionId}:rollup`, + userMessage, + sessionId, + project, + projectPath: workspace, + ...(cwd ? { workingDirectory: cwd } : {}), + }) + + return { calls, diagnostics } +} diff --git a/packages/core/src/providers/cline-cli/index.ts b/packages/core/src/providers/cline-cli/index.ts new file mode 100644 index 000000000..89a2deb71 --- /dev/null +++ b/packages/core/src/providers/cline-cli/index.ts @@ -0,0 +1,30 @@ +// @codeburn/core Cline CLI provider. +// +// Two layers: +// - Rich pure decode (`decodeClineCli`): host-facing, NOT part of the stable +// minimized surface. Pure over supplied records; carries content in-memory +// but no pricing (cost leaves the decoder) and no bash base-name extraction +// (that stays host-side with its `strip-ansi` dependency). +// - Minimizing transform (`toObservations`): maps the rich decode into the +// strict observation envelope; the content-smuggling guarantees bind here. + +export { + decodeClineCli, + clineCliToolNameMap, + PROVIDER_NAME, + type ClineCliDecodeInput, + type ClineCliDecodeResult, +} from './decode.js' + +export { + toObservations, + type RichClineCliSessionDecode, + type ClineCliToObservationsContext, +} from './observations.js' + +export type { + ClineCliDecodedCall, + ClineCliMetrics, + ClineCliSessionRecords, + ClineCliToolCall, +} from './types.js' diff --git a/packages/core/src/providers/cline-cli/observations.ts b/packages/core/src/providers/cline-cli/observations.ts new file mode 100644 index 000000000..beffedc53 --- /dev/null +++ b/packages/core/src/providers/cline-cli/observations.ts @@ -0,0 +1,98 @@ +// Minimizing transform: rich Cline CLI decode -> the strict observation +// envelope. +// +// Only opaque ids, fingerprints, enums, numbers, timestamps, and canonical tool +// names cross into the output. Project paths are fingerprinted; user messages, +// commands, and file paths stay behind. + +import { projectRef, sessionRef } from '../../fingerprint.js' +import type { RecordDiagnostic } from '../../diagnostics.js' +import type { CallObservation, SessionObservation } from '../../observations.js' +import { extractResourceRefs } from '../resource-refs.js' +import type { ClineCliDecodedCall } from './types.js' +import { normalizeModelIdentifier } from '../../schema.js' + +/** One Cline CLI session's rich decode, as the host holds it before minimization. */ +export interface RichClineCliSessionDecode { + sessionId: string + /** Absolute project path (the session workspace); fingerprinted, never emitted raw. */ + projectPath: string + /** Rich, cost-free calls in decode order (one per metered message, or the rollup). */ + calls: ClineCliDecodedCall[] +} + +export interface ClineCliToObservationsContext { + /** HMAC key that scopes every fingerprint. */ + privacyKey: string + /** Provider id stamped onto sessions/calls and folded into sessionRef. */ + provider?: string +} + +const CANONICAL_TOOL_NAME = /^[A-Za-z0-9_.-]{1,64}$/ + +function toCallObservation(call: ClineCliDecodedCall, turnIndex: number, privacyKey: string): CallObservation { + const measured = call.reportedCost !== undefined + return { + provider: call.provider, + model: normalizeModelIdentifier(call.model), + tokens: { + input: call.inputTokens, + output: call.outputTokens, + reasoning: call.reasoningTokens, + cacheRead: call.cacheReadInputTokens, + cacheCreate: call.cacheCreationInputTokens, + }, + webSearchRequests: call.webSearchRequests, + speed: call.speed, + // The CLI reports its own metered dollar cost when present (a metered $0 + // stays reported); otherwise the host prices from the token buckets. + costBasis: measured ? 'measured' : 'estimated', + ...(measured ? { measuredCostUSD: call.reportedCost } : {}), + timestamp: call.timestamp, + dedupKey: call.deduplicationKey, + toolNames: call.tools.filter(t => CANONICAL_TOOL_NAME.test(t)), + turnIndex, + ...extractResourceRefs(privacyKey, call.toolSequence), + } +} + +function toSessionObservation( + decode: RichClineCliSessionDecode, + ctx: ClineCliToObservationsContext, +): SessionObservation { + const provider = ctx.provider ?? 'cline-cli' + const calls: CallObservation[] = decode.calls.map((call, i) => toCallObservation(call, i, ctx.privacyKey)) + + const timestamps = calls.map(c => c.timestamp).filter(t => t.length > 0).sort() + const startedAt = timestamps[0] ?? '' + const endedAt = timestamps.length > 0 ? timestamps[timestamps.length - 1]! : '' + + const session: SessionObservation = { + sessionRef: sessionRef(ctx.privacyKey, provider, decode.sessionId), + projectRef: projectRef(ctx.privacyKey, decode.projectPath), + providerId: provider, + startedAt, + ...(endedAt ? { endedAt } : {}), + calls, + turnCount: calls.length, + } + return session +} + +/** + * Map a rich Cline CLI decode into the minimized observation layer. Returns the + * `sessions` array plus any per-record `diagnostics`. + * + * Content-smuggling guarantee: no free text (user message, cwd, project path, + * command, read/edited file path, tool argument) is ever copied into the result. + * Only fingerprints, enums, numbers, timestamps, dedup keys, and canonical tool + * names cross the boundary. + */ +export function toObservations( + decode: RichClineCliSessionDecode | RichClineCliSessionDecode[], + ctx: ClineCliToObservationsContext, +): { sessions: SessionObservation[]; diagnostics: RecordDiagnostic[] } { + const decodes = Array.isArray(decode) ? decode : [decode] + const sessions = decodes.map(d => toSessionObservation(d, ctx)) + return { sessions, diagnostics: [] } +} diff --git a/packages/core/src/providers/cline-cli/types.ts b/packages/core/src/providers/cline-cli/types.ts new file mode 100644 index 000000000..26fc4e176 --- /dev/null +++ b/packages/core/src/providers/cline-cli/types.ts @@ -0,0 +1,83 @@ +// Raw record + rich-decode types for the Cline CLI provider. +// +// The Cline CLI (npm `cline`, 3.x) stores sessions as +// //.json (metadata + rolled-up usage) plus a +// co-located .messages.json (per-message metrics). This is a +// different layout from the VS Code extension's tasks/ui_messages.json tree the +// `cline` provider reads, so it is kept as its own provider. +// +// The host reads + JSON-parses both files (I/O stays CLI-side, like codewhale) +// and hands ONE composite record to the pure decoder. The Decoded* types are +// the rich decode layer's output: pure over supplied records, carrying content +// in-memory but NO pricing (the host prices them). The CLI adapter maps +// ClineCliDecodedCall into its own ParsedProviderCall by adding +// `costBasis`/`costUSD` (measured when the CLI reported a cost, estimated +// otherwise) and running the pricing pass. + +export type ClineCliMetrics = { + inputTokens: number + outputTokens: number + cacheReadTokens: number + cacheWriteTokens: number +} + +// One tool invocation captured in a message's tool sequence. Mirrors the CLI's +// ToolCall so the host can consume it without a shape conversion; `file` and +// `command` are host-side only (fingerprinted before they can reach an +// observation). +export type ClineCliToolCall = { + tool: string + file?: string + command?: string +} + +/** + * The composite record the host hands the core decoder for one session: the + * parsed metadata file plus the parsed messages array. + * + * The CLI injects two host-side conveniences into `meta` before handing it + * over so the decoder stays path-free: + * - `session_id` when the file omits it (the session directory name, i.e. the + * metadata file's basename without `.json`); + * - `project` (the discovered source's project label). + */ +export type ClineCliSessionRecords = { + meta: Record + messages: unknown[] +} + +// The rich decode of one Cline CLI call (one assistant message with a metrics +// block, or the session rollup fallback), pre-pricing. Mirrors the host's +// ParsedProviderCall minus cost fields (the host adds those): cost leaves the +// decoder. `reportedCost` carries the CLI's own metered dollar figure when one +// was actually present and non-negative (a metered $0 stays reported); when +// absent the host prices from the token buckets. `rawBashCommands` are the +// un-split shell command strings from Bash-mapped tool calls; the CLI adapter +// runs its own base-name extraction on them to build the `bashCommands` field. +export type ClineCliDecodedCall = { + provider: 'cline-cli' + model: string + inputTokens: number + outputTokens: number + cacheCreationInputTokens: number + cacheReadInputTokens: number + cachedInputTokens: number + reasoningTokens: number + webSearchRequests: number + /** CLI-reported dollar cost, present only when actually metered (incl. $0). */ + reportedCost?: number + tools: string[] + rawBashCommands: string[] + skills: string[] + subagentTypes: string[] + toolSequence?: ClineCliToolCall[][] + timestamp: string + speed: 'standard' + deduplicationKey: string + turnId: string + userMessage: string + sessionId: string + project: string + projectPath?: string + workingDirectory?: string +} diff --git a/packages/core/tests/architecture-gate.test.ts b/packages/core/tests/architecture-gate.test.ts index 474618418..090c6187a 100644 --- a/packages/core/tests/architecture-gate.test.ts +++ b/packages/core/tests/architecture-gate.test.ts @@ -127,6 +127,8 @@ const CORRECTION_PHRASES = [ const USER_MESSAGE_ALLOWLIST = new Set([ 'src/providers/claude/decode.ts', 'src/providers/claude/types.ts', + 'src/providers/cline-cli/decode.ts', + 'src/providers/cline-cli/types.ts', 'src/providers/codebuff/decode.ts', 'src/providers/codebuff/types.ts', 'src/providers/codewhale/decode.ts', diff --git a/packages/core/tests/content-smuggling.test.ts b/packages/core/tests/content-smuggling.test.ts index 55e76d6ec..717cba586 100644 --- a/packages/core/tests/content-smuggling.test.ts +++ b/packages/core/tests/content-smuggling.test.ts @@ -20,6 +20,7 @@ import { } from '../src/providers/claude/index.js' import type { JournalEntry, ToolResultMeta } from '../src/providers/claude/index.js' import { decodeCodex, toObservations as toCodexObservations } from '../src/providers/codex/index.js' +import { decodeClineCli, toObservations as toClineCliObservations } from '../src/providers/cline-cli/index.js' import { decodeQwen, toObservations as toQwenObservations } from '../src/providers/qwen/index.js' import { decodeGrok, toObservations as toGrokObservations } from '../src/providers/grok/index.js' import { decodeKimi, toObservations as toKimiObservations } from '../src/providers/kimi/index.js' @@ -422,6 +423,85 @@ describe('content-smuggling guardrail: real qwen decode -> toObservations is sec }) }) +describe('content-smuggling guardrail: real cline-cli decode -> toObservations is secret-free', () => { + // A hostile Cline CLI session planting every secret in the free-text fields a + // real decode captures: the user prompt, a run_commands shell line, and a + // read_files path — plus a tool NAME carrying a command line. Decoding it + // fully and minimizing MUST surface none of them. + const clineCliContext: DecodeContext = { privacyKey: 'test-privacy-key', providerId: 'cline-cli', sourceRef: 'ref' } + + function decodeAndMinimize() { + const records = [{ + meta: { + version: 1, session_id: 'sess-hostile', source: 'cli', status: 'completed', + provider: 'cline-pass', model: 'z-ai/glm-5.2', + cwd: SECRETS.absPath, workspace_root: SECRETS.absPath, + started_at: '2026-08-02T20:04:18.628Z', ended_at: '2026-08-02T20:08:27.768Z', + metadata: {}, project: 'secret-plan', + }, + messages: [ + { + id: 'u1', role: 'user', ts: 1785701064304, + content: [{ type: 'text', text: `${SECRETS.prompt} ${SECRETS.apiKey} ${SECRETS.fileContent}` }], + }, + { + id: 'a1', role: 'assistant', ts: 1785701064305, + modelInfo: { id: 'z-ai/glm-5.2', provider: 'cline-pass' }, + metrics: { inputTokens: 500, outputTokens: 200, cacheReadTokens: 0, cacheWriteTokens: 0, cost: 0.01 }, + content: [ + { type: 'text', text: 'done' }, + { type: 'tool_use', id: 'call_0', name: 'run_commands', input: { commands: JSON.stringify([SECRETS.commandLine]) } }, + { type: 'tool_use', id: 'call_1', name: 'read_files', input: { path: SECRETS.absPath } }, + // A hostile tool NAME carrying a command line (spaces + slashes): it + // fails the canonical charset and must be dropped, not emitted. + { type: 'tool_use', id: 'call_2', name: SECRETS.commandLine, input: {} }, + ], + }, + ], + }] + const { calls } = decodeClineCli({ records, context: clineCliContext }) + const { sessions } = toClineCliObservations( + { sessionId: 'sess-hostile', projectPath: SECRETS.absPath, calls }, + { privacyKey: 'test-privacy-key', provider: 'cline-cli' }, + ) + return { + schemaVersion: OBSERVATION_SCHEMA_VERSION, + generator: { name: '@codeburn/core', version: '0.0.0-test' }, + sessions, + } + } + + it('produces a schema-valid envelope from the hostile chat', () => { + expect(ObservationEnvelope.safeParse(decodeAndMinimize()).success).toBe(true) + }) + + it('the serialized envelope contains none of the planted secrets', () => { + const serialized = JSON.stringify(decodeAndMinimize()) + for (const secret of ALL_SECRETS) { + expect(serialized).not.toContain(secret) + } + }) + + it('keeps canonical tool names (Bash/Read) and drops the argument-carrying name', () => { + const env = decodeAndMinimize() + const allToolNames = env.sessions.flatMap(s => s.calls.flatMap(c => c.toolNames)) + expect(allToolNames).toContain('Bash') + expect(allToolNames).toContain('Read') + expect(allToolNames).not.toContain(SECRETS.commandLine) + }) + + it('fingerprints the read_files path into a 16-hex resourceRead, never the raw path', () => { + const env = decodeAndMinimize() + const reads = env.sessions.flatMap(s => s.calls.flatMap(c => c.resourceReads ?? [])) + expect(reads.length).toBeGreaterThan(0) + for (const ref of reads) { + expect(ref.resourceId).toMatch(/^[0-9a-f]{16}$/) + expect(typeof ref.resourceClass).toBe('string') + } + expect(allStrings(reads)).not.toContain(SECRETS.absPath) + }) +}) + describe('content-smuggling guardrail: real vscode-cline decode -> toObservations is secret-free', () => { // A hostile vscode-cline task planting every secret in the free-text fields the // decode captures: the user message, the workspace path, and raw history text. diff --git a/packages/core/tests/providers/cline-cli-decode.test.ts b/packages/core/tests/providers/cline-cli-decode.test.ts new file mode 100644 index 000000000..fc3fbe06c --- /dev/null +++ b/packages/core/tests/providers/cline-cli-decode.test.ts @@ -0,0 +1,182 @@ +import { describe, expect, it } from 'vitest' + +import { decodeClineCli, toObservations } from '../../src/providers/cline-cli/index.js' +import { ObservationEnvelope } from '../../src/observations.js' +import { OBSERVATION_SCHEMA_VERSION } from '../../src/schema.js' +import type { DecodeContext } from '../../src/contracts.js' + +const context: DecodeContext = { privacyKey: 'k', providerId: 'cline-cli', sourceRef: 'ref' } + +type MessageSpec = { + id?: string + role: 'user' | 'assistant' + text?: string + metrics?: Record + model?: string + ts?: number + toolUse?: { name: string; input: Record } +} + +function session(meta: Record, messages: MessageSpec[]): unknown[] { + return [{ + meta, + messages: messages.map((spec, index) => { + const content: unknown[] = [] + if (spec.text) content.push({ type: 'text', text: spec.text }) + if (spec.toolUse) content.push({ type: 'tool_use', id: `call_${index}`, ...spec.toolUse }) + const message: Record = { + id: spec.id ?? `msg_${index}`, + role: spec.role, + content, + ts: spec.ts ?? 1785701064304 + index * 1000, + } + if (spec.metrics) message['metrics'] = spec.metrics + if (spec.model) message['modelInfo'] = { id: spec.model, provider: 'cline-pass' } + return message + }), + }] +} + +const DEFAULT_META: Record = { + version: 1, + session_id: 'sess-a', + source: 'cli', + status: 'completed', + provider: 'cline-pass', + model: 'z-ai/glm-5.2', + cwd: '/Users/dev/work/my-repo', + workspace_root: '/Users/dev/work/my-repo', + started_at: '2026-08-02T20:04:18.628Z', + ended_at: '2026-08-02T20:08:27.768Z', + metadata: {}, + project: 'my-repo', +} + +describe('cline-cli rich decode (moved to @codeburn/core)', () => { + it('decodes metered assistant messages into rich, cost-free calls', () => { + const records = session(DEFAULT_META, [ + { role: 'user', text: 'do the thing' }, + { role: 'assistant', text: 'ok', metrics: { inputTokens: 100, outputTokens: 10, cacheReadTokens: 5, cacheWriteTokens: 2, cost: 0.01 } }, + { role: 'assistant', text: 'done', metrics: { inputTokens: 200, outputTokens: 20, cost: 0.02 } }, + ]) + const { calls } = decodeClineCli({ records, context }) + + expect(calls).toHaveLength(2) + // No pricing crosses into the decode layer. + expect(calls[0]).not.toHaveProperty('costUSD') + expect(calls[0]).not.toHaveProperty('costBasis') + + expect(calls[0]!.inputTokens).toBe(100) + expect(calls[0]!.outputTokens).toBe(10) + expect(calls[0]!.cacheReadInputTokens).toBe(5) + expect(calls[0]!.cacheCreationInputTokens).toBe(2) + expect(calls[0]!.reportedCost).toBe(0.01) + expect(calls[0]!.sessionId).toBe('sess-a') + expect(calls[0]!.project).toBe('my-repo') + expect(calls[0]!.projectPath).toBe('/Users/dev/work/my-repo') + expect(calls[0]!.workingDirectory).toBe('/Users/dev/work/my-repo') + expect(calls[0]!.deduplicationKey).toBe('cline-cli:sess-a:msg_1') + expect(calls[0]!.turnId).toBe('sess-a:msg_1') + expect(calls[0]!.userMessage).toBe('do the thing') + expect(calls[1]!.reportedCost).toBe(0.02) + }) + + it('keeps a metered $0 reported and treats a negative cost as absent', () => { + const records = session(DEFAULT_META, [ + { role: 'assistant', text: 'zero', metrics: { inputTokens: 10, outputTokens: 10, cost: 0 } }, + { role: 'assistant', text: 'negative', metrics: { inputTokens: 10, outputTokens: 10, cost: -5 } }, + ]) + const { calls } = decodeClineCli({ records, context }) + + expect(calls).toHaveLength(2) + expect(calls[0]!.reportedCost).toBe(0) + expect(calls[1]!.reportedCost).toBeUndefined() + }) + + it('maps tools to canonical names and carries raw bash commands host-side', () => { + const records = session(DEFAULT_META, [{ + role: 'assistant', text: 'running', + metrics: { inputTokens: 1, outputTokens: 1, cost: 0.1 }, + toolUse: { name: 'run_commands', input: { commands: JSON.stringify(['git status', 'ls -la']) } }, + }]) + const { calls } = decodeClineCli({ records, context }) + + expect(calls[0]!.tools).toEqual(['Bash']) + // Raw command strings survive host-side; base-name extraction is the CLI's job. + expect(calls[0]!.rawBashCommands).toEqual(['git status', 'ls -la']) + expect(calls[0]!.toolSequence?.[0]?.[0]).toEqual({ tool: 'Bash', command: 'git status' }) + }) + + it('threads a live seenKeys set so a repeated message id across passes drops', () => { + const records = session(DEFAULT_META, [ + { role: 'assistant', text: 'a', metrics: { inputTokens: 1, outputTokens: 1, cost: 0.1 } }, + ]) + const seen = new Set() + expect(decodeClineCli({ records, context, seenKeys: seen }).calls).toHaveLength(1) + // Re-decoding the same records with the shared set yields nothing. + expect(decodeClineCli({ records, context, seenKeys: seen }).calls).toEqual([]) + }) + + it('promotes a seconds-resolution timestamp instead of landing in 1970', () => { + const seconds = Math.floor(Date.parse('2026-08-02T20:04:18.000Z') / 1000) + const records = session(DEFAULT_META, [ + { role: 'assistant', text: 'a', ts: seconds, metrics: { inputTokens: 1, outputTokens: 1, cost: 0.1 } }, + ]) + const { calls } = decodeClineCli({ records, context }) + expect(calls[0]!.timestamp).toBe('2026-08-02T20:04:18.000Z') + }) + + it('falls back to the session rollup when no message carries metrics', () => { + const meta = { ...DEFAULT_META, metadata: { usage: { inputTokens: 5483, outputTokens: 133, cacheReadTokens: 50, cacheWriteTokens: 0, totalCost: 0.0081984 } } } + const records = session(meta, []) + const { calls } = decodeClineCli({ records, context }) + + expect(calls).toHaveLength(1) + expect(calls[0]!.inputTokens).toBe(5483) + expect(calls[0]!.reportedCost).toBeCloseTo(0.0081984, 7) + expect(calls[0]!.deduplicationKey).toBe('cline-cli:sess-a:rollup') + }) + + it('declines the rollup when per-message calls were all deduped (hadMetrics gate)', () => { + // A duplicated session directory: the shared dedup already owns the message + // id, so every per-message call is suppressed — the rollup must not then + // fire and double-count the session (regression for #894). + const meta = { ...DEFAULT_META, session_id: 'shared', metadata: { usage: { inputTokens: 100, outputTokens: 10, totalCost: 0.01 } } } + const records = session(meta, [ + { id: 'msg_0', role: 'assistant', text: 'a', metrics: { inputTokens: 100, outputTokens: 10, cost: 0.01 } }, + ]) + const seen = new Set(['cline-cli:shared:msg_0']) + const { calls } = decodeClineCli({ records, context, seenKeys: seen }) + + expect(calls).toEqual([]) + }) + + it('toObservations produces a schema-valid, content-free envelope', () => { + const records = session(DEFAULT_META, [ + { role: 'user', text: 'read the file' }, + { + role: 'assistant', text: 'ok', metrics: { inputTokens: 100, outputTokens: 10, cost: 0.01 }, + toolUse: { name: 'read_files', input: { path: '/Users/dev/work/my-repo/src/a.ts' } }, + }, + ]) + const { calls } = decodeClineCli({ records, context }) + const { sessions } = toObservations( + { sessionId: 'sess-a', projectPath: '/Users/dev/work/my-repo', calls }, + { privacyKey: 'test-privacy-key', provider: 'cline-cli' }, + ) + const envelope = { + schemaVersion: OBSERVATION_SCHEMA_VERSION, + generator: { name: '@codeburn/core', version: '0.0.0-test' }, + sessions, + } + expect(ObservationEnvelope.safeParse(envelope).success).toBe(true) + // The metered cost crosses as measuredCostUSD (the observation carries the + // provider-reported figure the host would otherwise re-price). + expect(sessions[0]!.calls[0]!.costBasis).toBe('measured') + expect(sessions[0]!.calls[0]!.measuredCostUSD).toBe(0.01) + // The read_file path is fingerprinted into a resourceRead, never emitted raw. + const reads = sessions.flatMap(s => s.calls.flatMap(c => c.resourceReads ?? [])) + expect(reads.length).toBeGreaterThan(0) + for (const ref of reads) expect(ref.resourceId).toMatch(/^[0-9a-f]{16}$/) + }) +}) diff --git a/packages/core/tsup.config.ts b/packages/core/tsup.config.ts index 8af7b2db8..abd69a078 100644 --- a/packages/core/tsup.config.ts +++ b/packages/core/tsup.config.ts @@ -12,6 +12,7 @@ export default defineConfig({ 'src/contracts.ts', 'src/detectors/index.ts', 'src/providers/claude/index.ts', + 'src/providers/cline-cli/index.ts', 'src/providers/codebuff/index.ts', 'src/providers/codewhale/index.ts', 'src/providers/codex/index.ts', From 8ec2753cbdc9d604e4721b6970a0b31cdcab3123 Mon Sep 17 00:00:00 2001 From: iamtoruk Date: Fri, 21 Aug 2026 13:37:22 -0700 Subject: [PATCH 2/2] feat(cli): rehome codex tool-excluded active throughput MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The second half of #940 (@ozymandiashh): upstream `main` measures Codex throughput — a task's wall time minus its recorded tool wait, divided across the task's calls by generated tokens — and none of it exists on this branch. Added on one side only, so a `main` merge would land `src/codex-throughput.ts` at a path npm workspaces does not build. Rehomed against main as it ships today, with one deliberate divergence from #940, which the maintainer decided: main's resume design wins. - Timing state is captured ONLY at a `task_started` boundary, where every per-task accumulator is provably empty, and a task's calls are buffered until its window is known. No recorded call is ever mutated after it has been handed to the host. #940's alternative — threading the open task window through the serialized state and back-patching earlier-pass calls via applyCodexTimingPatches — is dropped in full. - The branch's Phase-4 token-decode resume is untouched: it stays any-offset and round-trip proven. Marrying the two needed one adaptation, since core decodes records and never sees bytes: the decoder now reports its last task_started as a `checkpoint` (record index + call count + state), and the CLI turns that index into a byte offset and replays only the calls before it, letting the still-open task re-derive. A pass that crosses no boundary keeps the previous one; a cold decode of a file with no task_started at all falls back to end-of-file with `taskOpen: false`, so a task_complete whose window this pass never saw attributes nothing rather than spreading a whole task's active time over part of its tokens. Restores the three fad84662 review fixes that #940 reverted: the discovery fast path already short-circuits on cachedProject before isValidCodexSession (unchanged here, verified); payload-level `duration_ms` outranks any nested one (`timingDuration ?? timingNumber('duration_ms')`), so a duration buried in an oversized mcp_tool_call_end's invocation.arguments can no longer inflate tool wait; and MIN_WIDE stays 90 with the Tok/s column behind a showTps gate rather than jumping to 130 and costing 90-129 column terminals their two-column dashboard. Also ports the fork-suppressed-task_started regression test and the depth-1 payloadString helper (main 1d36f444/497f6556), which the branch lacked. Scope discipline: main's codex pricing work (billableOutputTokens, #1078) is NOT dragged along — that is #1083 — and neither are its unported parser changes (custom-tool transport, exact token counts and MCP names on oversized lines), so cost, calls and tokens are untouched. Verified on a 1326-session real corpus: codex totals byte-identical to the base branch, with 1302 of 1328 model slices now carrying timing (36.5 Tok/s on GPT-5.5). CODEX_CACHE_VERSION takes 12, clear of main's ladder (11 as of #1078) so a cache written by either line can never be read as current by the other, and the codex parse version bumps in lockstep so session-cache.json cannot keep serving timing-less turns without invoking the parser. --- CHANGELOG.md | 1 + packages/cli/src/codex-cache.ts | 19 +- packages/cli/src/codex-throughput.ts | 521 ++++++++++++++++++ packages/cli/src/dashboard.tsx | 20 +- packages/cli/src/main.ts | 107 +++- packages/cli/src/model-breakdown.ts | 5 + packages/cli/src/parser.ts | 24 + packages/cli/src/providers/codex.ts | 35 +- packages/cli/src/providers/types.ts | 8 + packages/cli/src/session-cache.ts | 14 +- packages/cli/src/types.ts | 9 +- packages/cli/tests/cli-codex-tps.test.ts | 46 ++ packages/cli/tests/codex-resume.test.ts | 48 ++ ...odex-throughput-cache-invalidation.test.ts | 110 ++++ .../codex-throughput-cache-roundtrip.test.ts | 160 ++++++ packages/cli/tests/codex-throughput.test.ts | 124 +++++ packages/cli/tests/providers/codex.test.ts | 231 ++++++++ packages/core/src/providers/codex/decode.ts | 284 +++++++++- packages/core/src/providers/codex/index.ts | 1 + packages/core/src/providers/codex/types.ts | 23 + .../core/tests/providers/codex-decode.test.ts | 65 +++ 21 files changed, 1833 insertions(+), 22 deletions(-) create mode 100644 packages/cli/src/codex-throughput.ts create mode 100644 packages/cli/tests/cli-codex-tps.test.ts create mode 100644 packages/cli/tests/codex-throughput-cache-invalidation.test.ts create mode 100644 packages/cli/tests/codex-throughput-cache-roundtrip.test.ts create mode 100644 packages/cli/tests/codex-throughput.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 178c8ebbb..98a606d7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Added (CLI) - `codeburn sync push --attribution` (opt-in): sends git attribution spans — the session→commit correlation from `codeburn yield` (`codeburn.session.attribution` and `codeburn.commit` span types with normalized repo remote, commit SHAs, merged/reverted state, and PR links). Nothing new is sent without the flag; local-only repos and Windows filesystem paths are never emitted as repo identities, and sessions whose project path no longer resolves never inherit the push-time working directory's repo. See docs/sync/README.md "Git attribution". +- **Codex throughput tracking**: per-model Tok/s in the dashboard (active time excludes recorded tool wait), plus `codeburn codex-tps` for a retrospective generated-tokens/sec estimate off a rollout's checkpoints. Timing is attributed per task, from `task_started` to `task_complete`, and split across the task's calls by generated tokens; a task's calls are emitted only once that window is known, so no recorded call is ever revised after the fact. The codex results cache takes v12 and the codex parse version bumps in lockstep, so already-cached sessions re-derive once and pick the fields up. (#940, thanks @ozymandiashh) - **Cline CLI provider** — the standalone Cline command-line agent (npm `cline`, 3.x), separate from the VS Code extension's [Cline](docs/providers/cline.md) provider. Reads session metadata and rolled-up usage from `~/.cline/data/sessions//.json`, and per-message `metrics` (input, output, cache read/write, cost) from the co-located `.messages.json`; cost is read per message rather than estimated. (#940, thanks @ozymandiashh) ### Changed (@codeburn/core — breaking; version bump deferred to the next release, which must take at least a minor under 0.x) diff --git a/packages/cli/src/codex-cache.ts b/packages/cli/src/codex-cache.ts index 23175aa50..49e98f22d 100644 --- a/packages/cli/src/codex-cache.ts +++ b/packages/cli/src/codex-cache.ts @@ -32,7 +32,17 @@ import type { ParsedProviderCall } from './providers/types.js' // re-parse of multi-GB rollout corpora over that is a bad trade — deliberately // NOT bumped. The daily-cache bump alone propagates the discovery widening: // newly-eligible files aren't in this cache yet and parse fresh regardless. -const CODEX_CACHE_VERSION = 8 +// +// v12: tool-excluded active timing. Every cached call can now carry +// activeDurationMs / activeGeneratedTokens / toolWaitMs, the stored `state` can +// carry the task-boundary window flag, and `callCount` marks how much of +// `calls` a resumed decode may replay (the rest re-derives from the last +// task_started). Cached entries have none of that, so bump once and let +// unchanged sessions re-decode. This takes 12 rather than 9: main's own ladder +// has since reached 11 (#1078), and a shared version number on two different +// payload shapes would let a cache written by either line be read as current by +// the other. +const CODEX_CACHE_VERSION = 12 const CACHE_FILE = 'codex-results.json' type FileFingerprint = { mtimeMs: number; sizeBytes: number } @@ -41,8 +51,15 @@ type FileEntry = { mtimeMs: number sizeBytes: number project: string + // Resume point: the byte offset after the last `task_started` line this file + // decoded past, with `state` snapshotted there and `callCount` counting the + // calls emitted before it. An appended tail resumes from that boundary and + // re-derives the calls beyond it, so a task whose task_complete lands in the + // appended region gets its timing from a whole re-read window instead of a + // patch applied to calls already served. byteOffset: number state: CodexDecodeState + callCount: number calls: ParsedProviderCall[] } diff --git a/packages/cli/src/codex-throughput.ts b/packages/cli/src/codex-throughput.ts new file mode 100644 index 000000000..4796206d9 --- /dev/null +++ b/packages/cli/src/codex-throughput.ts @@ -0,0 +1,521 @@ +import { open, stat } from 'node:fs/promises' +import { StringDecoder } from 'node:string_decoder' + +export type CodexThroughputPoint = { + timestamp: string + model?: string + outputTokens: number + reasoningTokens: number + generatedTokens: number + taskGeneratedTokens?: number + elapsedSeconds?: number + generatedTokensPerSecond?: number + activeDurationSeconds?: number + activeGeneratedTokensPerSecond?: number + toolWaitSeconds?: number +} + +type TokenUsage = { + output_tokens?: number + reasoning_output_tokens?: number + total_tokens?: number +} + +type RolloutLine = { + type?: string + timestamp?: string + payload?: { + type?: string + turn_id?: string + call_id?: string + started_at?: number + duration_ms?: number + duration?: { secs?: number; nanos?: number } | string + model?: string + forked_from_id?: string + info?: { + last_token_usage?: TokenUsage + total_token_usage?: TokenUsage + } + } +} + +const CHUNK_BYTES = 64 * 1024 +const MAX_PENDING_LINE_CHARS = 4 * 1024 * 1024 +const TRUNCATION_MARKER = '__CODEBURN_TRUNCATED_LINE__' + +function rawString(source: string, field: string): string | undefined { + const match = new RegExp(`"${field}"\\s*:\\s*"((?:\\\\.|[^"\\\\])*)"`).exec(source) + if (!match) return undefined + try { return JSON.parse(`"${match[1]}"`) as string } catch { return undefined } +} + +function rawNumber(source: string, field: string): number | undefined { + const match = new RegExp(`"${field}"\\s*:\\s*(-?\\d+(?:\\.\\d+)?)`).exec(source) + if (!match) return undefined + const value = Number(match[1]) + return Number.isFinite(value) ? value : undefined +} + +function compactUsage(source: string, field: 'last_token_usage' | 'total_token_usage'): TokenUsage | undefined { + const index = source.indexOf(`"${field}"`) + if (index < 0) return undefined + const body = source.slice(index, index + 4096) + return { + output_tokens: rawNumber(body, 'output_tokens'), + reasoning_output_tokens: rawNumber(body, 'reasoning_output_tokens'), + total_tokens: rawNumber(body, 'total_tokens'), + } +} + +function parseRawDurationValue(value: string): number | undefined { + const objectMatch = /^\s*\{\s*"secs"\s*:\s*(-?\d+(?:\.\d+)?)\s*,\s*"nanos"\s*:\s*(-?\d+(?:\.\d+)?)\s*\}/.exec(value) + if (objectMatch) { + const seconds = Number(objectMatch[1]) + const nanos = Number(objectMatch[2]) + if (Number.isFinite(seconds) && Number.isFinite(nanos)) return seconds * 1000 + nanos / 1e6 + } + const stringMatch = /^\s*"(\d+(?:\.\d+)?)(ms|s)?"/.exec(value) + if (stringMatch) { + const parsed = Number(stringMatch[1]) + if (Number.isFinite(parsed)) return parsed * (stringMatch[2] === 's' ? 1000 : 1) + } + const numberMatch = /^\s*(-?\d+(?:\.\d+)?)/.exec(value) + if (numberMatch) { + const parsed = Number(numberMatch[1]) + if (Number.isFinite(parsed)) return parsed + } + return undefined +} + +function durationMs(payload: RolloutLine['payload']): number | undefined { + if (!payload) return undefined + if (typeof payload.duration_ms === 'number' && Number.isFinite(payload.duration_ms)) return payload.duration_ms + if (typeof payload.duration === 'object' && payload.duration) { + const seconds = payload.duration.secs + const nanos = payload.duration.nanos + if (typeof seconds === 'number' && typeof nanos === 'number' && Number.isFinite(seconds) && Number.isFinite(nanos)) { + return seconds * 1000 + nanos / 1e6 + } + } + if (typeof payload.duration === 'string') { + const match = /^(\d+(?:\.\d+)?)(ms|s)?$/.exec(payload.duration.trim()) + if (match) return Number(match[1]) * (match[2] === 's' ? 1000 : 1) + } + return undefined +} + +function mergeToolIntervals(intervals: Array<[number, number]>, durationMs: number, taskStartedAt?: number, taskCompletedAt?: number): number { + const windowStart = taskStartedAt ?? (taskCompletedAt !== undefined ? taskCompletedAt - durationMs : undefined) + const windowEnd = windowStart !== undefined ? windowStart + durationMs : undefined + const clipped = intervals.map(([start, end]) => [ + windowStart !== undefined ? Math.max(start, windowStart) : start, + windowEnd !== undefined ? Math.min(end, windowEnd) : end, + ] as [number, number]).filter(([start, end]) => end > start) + const merged = clipped.sort((a, b) => a[0] - b[0]).reduce>((result, interval) => { + const previous = result.at(-1) + if (previous && interval[0] <= previous[1]) previous[1] = Math.max(previous[1], interval[1]) + else result.push([...interval]) + return result + }, []) + return Math.min(durationMs, merged.reduce((total, [start, end]) => total + end - start, 0)) +} + +function parseLine(line: string): RolloutLine | null { + const payloadStart = line.indexOf('"payload"') + const payloadHead = payloadStart >= 0 ? line.slice(payloadStart) : line + if (line.length > 256 * 1024 || line.startsWith(TRUNCATION_MARKER)) { + const payloadType = rawString(payloadHead, 'type') + const infoStart = payloadHead.indexOf('"info"') + const info = infoStart >= 0 ? payloadHead.slice(infoStart) : '' + return { + type: rawString(line, 'type'), + timestamp: rawString(line, 'timestamp'), + payload: { + type: payloadType, + turn_id: rawString(payloadHead, 'turn_id'), + call_id: rawString(payloadHead, 'call_id'), + started_at: rawNumber(payloadHead, 'started_at'), + duration_ms: rawNumber(payloadHead, 'duration_ms'), + duration: rawString(payloadHead, 'duration') ?? (rawNumber(payloadHead, 'secs') !== undefined + ? { secs: rawNumber(payloadHead, 'secs'), nanos: rawNumber(payloadHead, 'nanos') } + : undefined), + model: rawString(payloadHead, 'model'), + forked_from_id: rawString(payloadHead, 'forked_from_id'), + info: { + last_token_usage: compactUsage(info, 'last_token_usage'), + total_token_usage: compactUsage(info, 'total_token_usage'), + }, + }, + } + } + try { + return JSON.parse(line) as RolloutLine + } catch { + return null + } +} + +/** + * Estimate generated tokens/sec from a Codex rollout's persisted checkpoints. + * Codex JSONL has no per-token timestamps, so this is deliberately a + * checkpoint-to-checkpoint estimate, not live decode speed. + */ +type ThroughputState = { + model?: string + previousTotal?: number + previousOutput: number + previousReasoning: number + previousTimestamp?: number + currentTaskGenerated: number + currentTaskToolIntervals: Array<[number, number]> + currentTaskStartedAt?: number + toolStarts: Map + latestPoint?: CodexThroughputPoint + points: CodexThroughputPoint[] + forkCutoffMs?: number +} + +function newThroughputState(): ThroughputState { + return { + previousOutput: 0, + previousReasoning: 0, + currentTaskGenerated: 0, + currentTaskToolIntervals: [], + toolStarts: new Map(), + points: [], + } +} + +/** + * Incrementally parses a rollout. Watch mode feeds only newly appended bytes + * to this reader, so a growing JSONL file is not reparsed from byte zero. + */ +export class CodexThroughputReader { + private offset = 0 + private pending = '' + private decoder = new StringDecoder('utf8') + private pendingDurationMs: number | undefined + private scanDepth = 0 + private scanPayloadDepth: number | undefined + private scanInString = false + private scanEscape = false + private scanString = '' + private scanLastString = '' + private scanAwaitingColon = false + private scanCurrentKey: string | undefined + private scanCapture: { mode: 'string' | 'object' | 'primitive'; text: string; depth: number } | undefined + private state = newThroughputState() + + reset(): void { + this.offset = 0 + this.pending = '' + this.decoder = new StringDecoder('utf8') + this.pendingDurationMs = undefined + this.scanDepth = 0 + this.scanPayloadDepth = undefined + this.scanInString = false + this.scanEscape = false + this.scanString = '' + this.scanLastString = '' + this.scanAwaitingColon = false + this.scanCurrentKey = undefined + this.scanCapture = undefined + this.state = newThroughputState() + } + + private finishDurationCapture(): void { + if (!this.scanCapture) return + const value = this.scanCapture.mode === 'string' ? `"${this.scanCapture.text}"` : this.scanCapture.text + const parsed = parseRawDurationValue(value) + if (parsed !== undefined && this.pendingDurationMs === undefined) this.pendingDurationMs = parsed + this.scanCapture = undefined + } + + private scanDurationSegment(source: string): void { + for (let i = 0; i < source.length; i++) { + const char = source[i]! + if (this.scanInString) { + if (this.scanEscape) { + this.scanEscape = false + if (this.scanCapture?.mode === 'object') this.scanCapture.text += char + else if (this.scanCapture?.mode === 'string') this.scanCapture.text += char + else this.scanString += char + continue + } + if (char === '\\') { + this.scanEscape = true + if (this.scanCapture?.mode === 'object' || this.scanCapture?.mode === 'string') this.scanCapture.text += char + continue + } + if (char === '"') { + if (this.scanCapture?.mode === 'object') this.scanCapture.text += char + this.scanInString = false + if (this.scanCapture?.mode === 'string') this.finishDurationCapture() + else if (this.scanCapture?.mode === 'object') { + this.scanAwaitingColon = false + this.scanCurrentKey = undefined + } else { + this.scanLastString = this.scanString + this.scanAwaitingColon = true + } + continue + } + if (this.scanCapture?.mode === 'object' || this.scanCapture?.mode === 'string') this.scanCapture.text += char + else this.scanString += char + continue + } + + if (this.scanCapture?.mode === 'primitive') { + if (char === ',' || char === '}' || char === ']') this.finishDurationCapture() + else { this.scanCapture.text += char; continue } + } + if (this.scanAwaitingColon) { + if (/\s/.test(char)) continue + if (char === ':') { + this.scanCurrentKey = this.scanLastString + this.scanAwaitingColon = false + continue + } + this.scanAwaitingColon = false + } + if (char === '"') { + this.scanString = '' + if (this.scanCapture?.mode === 'object') this.scanCapture.text += char + if (this.scanCurrentKey === 'duration' && this.scanPayloadDepth === this.scanDepth) { + this.scanCapture = { mode: 'string', text: '', depth: this.scanDepth } + this.scanCurrentKey = undefined + } + this.scanInString = true + continue + } + if (char === '{' || char === '[') { + if (this.scanCurrentKey === 'payload' && char === '{' && this.scanPayloadDepth === undefined) { + this.scanPayloadDepth = this.scanDepth + 1 + } + if (this.scanCurrentKey === 'duration' && this.scanPayloadDepth === this.scanDepth) { + this.scanCapture = { mode: 'object', text: char, depth: this.scanDepth + 1 } + this.scanCurrentKey = undefined + } else if (this.scanCapture?.mode === 'object') { + this.scanCapture.text += char + } + this.scanDepth++ + continue + } + if (char === '}' || char === ']') { + if (this.scanCapture?.mode === 'object') this.scanCapture.text += char + this.scanDepth = Math.max(0, this.scanDepth - 1) + if (this.scanCapture?.mode === 'object' && this.scanDepth < this.scanCapture.depth) this.finishDurationCapture() + continue + } + if (this.scanCurrentKey === 'duration' && this.scanPayloadDepth === this.scanDepth && !/\s/.test(char)) { + this.scanCapture = { mode: 'primitive', text: char, depth: this.scanDepth } + this.scanCurrentKey = undefined + continue + } + if (this.scanCapture?.mode === 'object') this.scanCapture.text += char + } + } + + private processLine(line: string, durationOverride?: number): void { + const entry = parseLine(line) + if (!entry) return + if (durationOverride !== undefined && (line.startsWith(TRUNCATION_MARKER) || line.length > 256 * 1024) && entry.type === 'event_msg' && (entry.payload?.type === 'mcp_tool_call_end' || entry.payload?.type === 'task_complete')) { + entry.payload = { ...entry.payload, duration_ms: durationOverride } + } + const state = this.state + if (entry.type === 'session_meta') { + if (entry.payload?.model) state.model = entry.payload.model + if (entry.payload?.forked_from_id && entry.timestamp) { + const timestamp = Date.parse(entry.timestamp) + if (Number.isFinite(timestamp)) state.forkCutoffMs = timestamp + 5000 + } + return + } + if (entry.type === 'turn_context' && entry.payload?.model) state.model = entry.payload.model + const entryTimestamp = entry.timestamp ? Date.parse(entry.timestamp) : NaN + const isForkReplay = state.forkCutoffMs !== undefined && Number.isFinite(entryTimestamp) && entryTimestamp < state.forkCutoffMs + if (isForkReplay && ( + entry.payload?.type === 'task_started' || + entry.payload?.type === 'task_complete' || + entry.payload?.type === 'function_call' || + entry.payload?.type === 'function_call_output' || + entry.payload?.type === 'custom_tool_call' || + entry.payload?.type === 'custom_tool_call_output' || + entry.payload?.type === 'mcp_tool_call_end' || + entry.payload?.type === 'patch_apply_end' || + entry.payload?.type === 'token_count' + )) return + if (entry.type === 'event_msg' && entry.payload?.type === 'task_started') { + state.currentTaskGenerated = 0 + state.currentTaskToolIntervals = [] + const startedAt = entry.timestamp ? Date.parse(entry.timestamp) : NaN + state.currentTaskStartedAt = Number.isFinite(startedAt) ? startedAt : undefined + state.toolStarts.clear() + } + if (entry.type === 'response_item' && (entry.payload?.type === 'function_call' || entry.payload?.type === 'custom_tool_call') && entry.payload.call_id && entry.timestamp) { + const started = Date.parse(entry.timestamp) + if (Number.isFinite(started)) state.toolStarts.set(entry.payload.call_id, started) + } + if (entry.type === 'response_item' && (entry.payload?.type === 'function_call_output' || entry.payload?.type === 'custom_tool_call_output') && entry.payload.call_id && entry.timestamp) { + const ended = Date.parse(entry.timestamp) + const started = state.toolStarts.get(entry.payload.call_id) + if (started !== undefined && Number.isFinite(ended) && ended > started) state.currentTaskToolIntervals.push([started, ended]) + state.toolStarts.delete(entry.payload.call_id) + } + if (entry.type === 'event_msg' && entry.payload?.type === 'mcp_tool_call_end' && entry.timestamp) { + const ended = Date.parse(entry.timestamp) + const elapsed = durationMs(entry.payload) + if (Number.isFinite(ended) && elapsed !== undefined && elapsed > 0) state.currentTaskToolIntervals.push([ended - elapsed, ended]) + } + if (entry.type === 'event_msg' && entry.payload?.type === 'task_complete') { + const taskDurationMs = durationMs(entry.payload) + if (state.latestPoint && typeof taskDurationMs === 'number' && taskDurationMs > 0 && state.currentTaskGenerated > 0) { + state.latestPoint.taskGeneratedTokens = state.currentTaskGenerated + const completedAt = entry.timestamp ? Date.parse(entry.timestamp) : undefined + const toolWaitMs = mergeToolIntervals(state.currentTaskToolIntervals, taskDurationMs, state.currentTaskStartedAt, Number.isFinite(completedAt) ? completedAt : undefined) + const activeMs = taskDurationMs - toolWaitMs + if (activeMs > 0) { + state.latestPoint.activeDurationSeconds = activeMs / 1000 + state.latestPoint.toolWaitSeconds = toolWaitMs / 1000 + state.latestPoint.activeGeneratedTokensPerSecond = state.currentTaskGenerated / (activeMs / 1000) + } + } + } + if (entry.type !== 'event_msg' || entry.payload?.type !== 'token_count') return + const info = entry.payload.info + if (!info || !entry.timestamp) return + const last = info.last_token_usage + const total = info.total_token_usage + const cumulative = total?.total_tokens + if (cumulative !== undefined && cumulative === state.previousTotal) return + let outputTokens = last?.output_tokens ?? 0 + let reasoningTokens = last?.reasoning_output_tokens ?? 0 + if (!last && total && cumulative !== undefined && state.previousTotal !== undefined) { + outputTokens = Math.max(0, (total.output_tokens ?? 0) - state.previousOutput) + reasoningTokens = Math.max(0, (total.reasoning_output_tokens ?? 0) - state.previousReasoning) + } + if (cumulative !== undefined) { + state.previousTotal = cumulative + state.previousOutput = total?.output_tokens ?? state.previousOutput + state.previousReasoning = total?.reasoning_output_tokens ?? state.previousReasoning + } + const generatedTokens = outputTokens + reasoningTokens + if (generatedTokens <= 0) return + const timestampMs = Date.parse(entry.timestamp) + if (!Number.isFinite(timestampMs)) return + const point: CodexThroughputPoint = { + timestamp: entry.timestamp, + model: state.model, + outputTokens, + reasoningTokens, + generatedTokens, + } + state.currentTaskGenerated += generatedTokens + state.latestPoint = point + if (state.previousTimestamp !== undefined && timestampMs > state.previousTimestamp) { + const elapsedSeconds = (timestampMs - state.previousTimestamp) / 1000 + point.elapsedSeconds = elapsedSeconds + point.generatedTokensPerSecond = generatedTokens / elapsedSeconds + } + state.previousTimestamp = timestampMs + state.points.push(point) + if (state.points.length > 10000) state.points.splice(0, state.points.length - 10000) + } + + async update(filePath: string, limit = 10, finalize = false): Promise { + const info = await stat(filePath) + if (info.size < this.offset) this.reset() + const bytesToRead = info.size - this.offset + if (bytesToRead > 0) { + const file = await open(filePath, 'r') + try { + let position = this.offset + while (position < info.size) { + const buffer = Buffer.allocUnsafe(Math.min(CHUNK_BYTES, info.size - position)) + const { bytesRead } = await file.read(buffer, 0, buffer.length, position) + if (bytesRead === 0) break + position += bytesRead + this.offset = position + let chunk = this.decoder.write(buffer.subarray(0, bytesRead)) + while (chunk.length > 0) { + const newlineIndex = chunk.search(/\r?\n/) + const segment = newlineIndex >= 0 ? chunk.slice(0, newlineIndex) : chunk + this.pending += segment + this.scanDurationSegment(segment) + if (newlineIndex < 0) break + const newlineLength = chunk[newlineIndex] === '\r' ? 2 : 1 + const line = this.pending + const durationOverride = this.pendingDurationMs + this.pending = '' + this.pendingDurationMs = undefined + this.scanDepth = 0 + this.scanPayloadDepth = undefined + this.scanInString = false + this.scanEscape = false + this.scanString = '' + this.scanLastString = '' + this.scanAwaitingColon = false + this.scanCurrentKey = undefined + this.scanCapture = undefined + this.processLine(line, durationOverride) + chunk = chunk.slice(newlineIndex + newlineLength) + } + if (this.pending.length > MAX_PENDING_LINE_CHARS) { + const body = this.pending.startsWith(TRUNCATION_MARKER) + ? this.pending.slice(TRUNCATION_MARKER.length) + : this.pending + this.pending = TRUNCATION_MARKER + body.slice(0, 256 * 1024) + body.slice(-256 * 1024) + } + } + } finally { + await file.close() + } + } + if (finalize && this.pending) { + this.processLine(this.pending, this.pendingDurationMs) + this.pending = '' + this.pendingDurationMs = undefined + } + return limit > 0 ? this.state.points.slice(-limit) : this.state.points.slice() + } +} + +export async function readCodexThroughput(filePath: string, limit = 10): Promise { + return new CodexThroughputReader().update(filePath, limit, true) +} + +export async function newestCodexSession(sessions: Array<{ path: string }>): Promise { + let newest: { path: string; mtimeMs: number } | undefined + for (const session of sessions) { + try { + const info = await stat(session.path) + if (!newest || info.mtimeMs > newest.mtimeMs) newest = { path: session.path, mtimeMs: info.mtimeMs } + } catch { + // A session can disappear while Codex rotates or archives it. + } + } + return newest?.path +} + +export function renderCodexThroughput(points: CodexThroughputPoint[], filePath: string): string { + const latest = points.at(-1) + if (!latest) return `No token_count checkpoints found in ${filePath}.` + const lines = [ + 'CodeBurn Codex throughput estimate', + `Session: ${filePath}`, + `Latest checkpoint: ${latest.timestamp}`, + `Latest checkpoint tokens: ${latest.generatedTokens.toLocaleString()} (${latest.outputTokens.toLocaleString()} output + ${latest.reasoningTokens.toLocaleString()} reasoning)`, + ] + if (latest.taskGeneratedTokens !== undefined) lines.push(`Completed task total: ${latest.taskGeneratedTokens.toLocaleString()} generated tokens`) + if (latest.activeGeneratedTokensPerSecond !== undefined) { + lines.push(`Active throughput: ${latest.activeGeneratedTokensPerSecond.toFixed(1)} generated tokens/sec over ${latest.activeDurationSeconds!.toFixed(1)}s`) + lines.push(`Excluded tool wait: ${latest.toolWaitSeconds!.toFixed(1)}s`) + } else if (latest.generatedTokensPerSecond !== undefined) { + lines.push(`Checkpoint estimate: ${latest.generatedTokensPerSecond.toFixed(1)} generated tokens/sec over ${latest.elapsedSeconds!.toFixed(1)}s`) + } else { + lines.push('Throughput: unavailable (waiting for a completed turn)') + } + lines.push('Note: offline JSONL estimate; tool intervals are removed, but server/prompt latency may remain.') + return lines.join('\n') +} diff --git a/packages/cli/src/dashboard.tsx b/packages/cli/src/dashboard.tsx index 1b68a2228..305be3373 100644 --- a/packages/cli/src/dashboard.tsx +++ b/packages/cli/src/dashboard.tsx @@ -46,6 +46,8 @@ export function showEmptyState(projectCount: number, scrollableHistory: boolean, return historyProjectCount === 0 && !historyLoading } +// The By Model panel drops the Tok/s column when the panel is too narrow, so +// the wider two-column layout still activates at ordinary terminal widths. const MIN_WIDE = 90 const ORANGE = '#FF8C42' const DIM = '#555555' @@ -196,9 +198,9 @@ function nextTick(): Promise { return new Promise(resolve => setImmediate(resolve)) } -type Layout = { dashWidth: number; wide: boolean; halfWidth: number; barWidth: number } +export type Layout = { dashWidth: number; wide: boolean; halfWidth: number; barWidth: number } -function getLayout(columns?: number): Layout { +export function getLayout(columns?: number): Layout { const termWidth = columns || parseInt(process.env['COLUMNS'] ?? '') || 80 const dashWidth = Math.min(160, termWidth) const wide = dashWidth >= MIN_WIDE @@ -440,6 +442,7 @@ const MODEL_COL_COST = 8 const MODEL_COL_CACHE = 7 const MODEL_COL_CALLS = 7 const MODEL_COL_ONESHOT = 7 +const MODEL_COL_TPS = 7 const MODEL_NAME_WIDTH = 14 const MIN_EDIT_TURNS_FOR_RATE = 5 @@ -449,6 +452,10 @@ function ModelBreakdown({ projects, pw, bw }: { projects: ProjectSummary[]; pw: const modelTotals = aggregateModelTotals(projects) const modelEfficiency = aggregateModelEfficiency(projects) const anyEstimated = Object.values(modelTotals).some(d => d.estimatedCostUSD > 0) + const anyActiveTiming = Object.values(modelTotals).some(d => d.activeDurationMs > 0 && d.activeGeneratedTokens > 0) + // The Tok/s column needs 61 inner columns for the full row; hide it on narrower + // panels and when no model has timing data (non-Codex users get no dead column). + const showTps = pw - PANEL_CHROME >= 61 && anyActiveTiming const sorted = Object.entries(modelTotals).sort(([, a], [, b]) => b.costUSD - a.costUSD) const maxCost = sorted[0]?.[1]?.costUSD ?? 0 const unpriced = findUnpricedModels(Object.entries(modelTotals).map(([model, d]) => ({ @@ -460,7 +467,7 @@ function ModelBreakdown({ projects, pw, bw }: { projects: ProjectSummary[]; pw: return ( - {''.padEnd(bw + 1 + MODEL_NAME_WIDTH)}{'cost'.padStart(MODEL_COL_COST)}{'cache'.padStart(MODEL_COL_CACHE)}{'calls'.padStart(MODEL_COL_CALLS)}{'1-shot'.padStart(MODEL_COL_ONESHOT)} + {''.padEnd(bw + 1 + MODEL_NAME_WIDTH)}{'cost'.padStart(MODEL_COL_COST)}{'cache'.padStart(MODEL_COL_CACHE)}{'calls'.padStart(MODEL_COL_CALLS)}{'1-shot'.padStart(MODEL_COL_ONESHOT)}{showTps ? 'Tok/s'.padStart(MODEL_COL_TPS) : ''} {sorted.map(([model, data], i) => { const totalInput = data.freshInput + data.cacheRead + data.cacheWrite const cacheHit = totalInput > 0 ? (data.cacheRead / totalInput) * 100 : 0 @@ -469,6 +476,9 @@ function ModelBreakdown({ projects, pw, bw }: { projects: ProjectSummary[]; pw: const oneShotLabel = efficiency && efficiency.editTurns >= MIN_EDIT_TURNS_FOR_RATE && efficiency.oneShotRate !== null ? `${efficiency.oneShotRate.toFixed(1)}%` : '-' + const tpsLabel = data.activeDurationMs > 0 && data.activeGeneratedTokens > 0 + ? (data.activeGeneratedTokens / (data.activeDurationMs / 1000)).toFixed(1) + : '-' return ( @@ -477,6 +487,7 @@ function ModelBreakdown({ projects, pw, bw }: { projects: ProjectSummary[]; pw: {cacheLabel.padStart(MODEL_COL_CACHE)} {String(data.calls).padStart(MODEL_COL_CALLS)} {oneShotLabel.padStart(MODEL_COL_ONESHOT)} + {showTps && {tpsLabel.padStart(MODEL_COL_TPS)}} ) })} @@ -488,6 +499,9 @@ function ModelBreakdown({ projects, pw, bw }: { projects: ProjectSummary[]; pw: {anyEstimated && ( ~ estimated cost (priced from estimated tokens) )} + {showTps && ( + ~ Tok/s: generated tokens / active time; tool wait excluded + )} ) } diff --git a/packages/cli/src/main.ts b/packages/cli/src/main.ts index ae6867cf2..36e4beaac 100644 --- a/packages/cli/src/main.ts +++ b/packages/cli/src/main.ts @@ -4,7 +4,7 @@ import { installMenubarApp } from './menubar-installer.js' import { exportCsv, exportJson, type PeriodExport } from './export.js' import { findUnpricedModels, loadPricing, setModelAliases, setPriceOverrides, setLocalModelSavings, setProxyPaths, normalizeProxyPath } from './models.js' import { parseAllSessions, filterProjectsByName, filterProjectsByDateRange, clearSessionCache, setInteractiveScanUI } from './parser.js' -import { allProviderNames, getAllProviders } from './providers/index.js' +import { allProviderNames, getAllProviders, getProvider } from './providers/index.js' import { convertCost, formatCost } from './currency.js' import { renderStatusBar } from './format.js' import { toDateString } from './daily-cache.js' @@ -46,6 +46,7 @@ import { createRequire } from 'node:module' const require = createRequire(import.meta.url) const { version } = require('../package.json') import { loadCurrency, getCurrency, isValidCurrencyCode } from './currency.js' +import { CodexThroughputReader, newestCodexSession, renderCodexThroughput } from './codex-throughput.js' // A downstream reader that closes the pipe early (`| head`, quitting `less`, or // a missing command) makes stdout writes fail with EPIPE. Exit cleanly rather @@ -68,6 +69,22 @@ function parseInteger(value: string): number { return parseInt(value, 10) } +function parseCodexTpsLimit(value: string): number { + const parsed = Number(value) + if (!Number.isFinite(parsed) || !Number.isInteger(parsed) || parsed < 1 || parsed > 10000) { + throw new Error('limit must be an integer from 1 to 10000') + } + return parsed +} + +function parseCodexTpsWatch(value: string): number { + const parsed = Number(value) + if (!Number.isFinite(parsed) || parsed < 0 || (parsed > 0 && parsed < 1) || parsed > 3600) { + throw new Error('watch must be 0 or at least 1 second (up to 3600 seconds)') + } + return parsed +} + type PriceOverrideConfig = NonNullable[string] type PriceOverrideOptions = { @@ -1843,6 +1860,94 @@ program await runContextCommand(session, opts) }) +program + .command('codex-tps [session]') + .description('Retrospective Codex generated-tokens/sec estimate from rollout checkpoints (not live decode speed)') + .option('--json', 'JSON output') + .option('--limit ', 'Number of recent checkpoints to scan', parseCodexTpsLimit, 10) + .option('--watch ', 'Refresh continuously while Codex writes checkpoints', parseCodexTpsWatch, 0) + .action(async (session: string | undefined, opts: { json?: boolean; limit: number; watch: number }) => { + const intervalMs = Math.max(0, opts.watch) * 1000 + if (opts.json && intervalMs > 0) { + process.stderr.write('codeburn codex-tps: --json cannot be combined with --watch; use text watch output or one-shot JSON.\n') + process.exitCode = 2 + return + } + const provider = await getProvider('codex') + if (!provider) { + process.stderr.write('codeburn codex-tps: Codex provider is unavailable.\n') + process.exitCode = 1 + return + } + let cachedPath: string | undefined = session + let throughputReader: CodexThroughputReader | undefined + let lastFileState: { size: number; mtimeMs: number } | undefined + let lastDiscoveryMs = 0 + let refreshInFlight = false + const render = async (): Promise => { + if (refreshInFlight) return + refreshInFlight = true + try { + let filePath = session ?? cachedPath + // Keep an idle watcher on its chosen rollout. A full active+archive + // discovery can be hundreds of milliseconds on large histories, so + // only re-scan slowly to notice rotation; disappearance still triggers + // an immediate discovery on the next tick. + if (!session && (!filePath || Date.now() - lastDiscoveryMs >= 60_000)) { + lastDiscoveryMs = Date.now() + filePath = await newestCodexSession(await provider.discoverSessions()) + } + if (!filePath) { + process.stderr.write('codeburn codex-tps: no Codex rollout sessions found.\n') + if (intervalMs === 0) process.exitCode = 1 + return + } + const previousPath = cachedPath + cachedPath = filePath + if (previousPath !== filePath || !throughputReader) throughputReader = new CodexThroughputReader() + const fileInfo = await import('node:fs/promises').then(fs => fs.stat(filePath)).catch(() => null) + if (!fileInfo) { + process.stderr.write(`codeburn codex-tps: session file not found: ${filePath}\n`) + if (intervalMs === 0) process.exitCode = 1 + if (!session) cachedPath = undefined + return + } + if (intervalMs > 0 && lastFileState && fileInfo.size === lastFileState.size && fileInfo.mtimeMs === lastFileState.mtimeMs) return + lastFileState = { size: fileInfo.size, mtimeMs: fileInfo.mtimeMs } + const points = await throughputReader!.update(filePath, opts.limit, intervalMs === 0) + if (opts.json) { + process.stdout.write(JSON.stringify({ session: filePath, points, live: intervalMs > 0 }, null, 2) + '\n') + } else { + if (intervalMs > 0) process.stdout.write('\x1b[2J\x1b[H') + process.stdout.write(renderCodexThroughput(points, filePath) + (intervalMs > 0 ? '\nWatching for new Codex checkpoints... (Ctrl-C to stop)\n' : '\n')) + } + } finally { + refreshInFlight = false + } + } + try { + await render() + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + process.stderr.write(`codeburn codex-tps: refresh failed: ${message}\n`) + if (intervalMs === 0) { + process.exitCode = 1 + return + } + } + if (intervalMs > 0) { + await new Promise((resolve) => { + const timer = setInterval(() => { + void render().catch(error => { + const message = error instanceof Error ? error.message : String(error) + process.stderr.write(`codeburn codex-tps: refresh failed: ${message}\n`) + }) + }, intervalMs) + process.once('SIGINT', () => { clearInterval(timer); resolve() }) + }) + } + }) + program .command('compare') .description('Compare two AI models side-by-side') diff --git a/packages/cli/src/model-breakdown.ts b/packages/cli/src/model-breakdown.ts index 5801be803..799338e8b 100644 --- a/packages/cli/src/model-breakdown.ts +++ b/packages/cli/src/model-breakdown.ts @@ -8,6 +8,8 @@ export interface ModelTotals { freshInput: number cacheRead: number cacheWrite: number + activeDurationMs: number + activeGeneratedTokens: number } /// Aggregate per-model usage across every session, keyed by the friendly display @@ -24,6 +26,7 @@ export function aggregateModelTotals(projects: ProjectSummary[]): Record): SessionPars if (resume && cached) { startByteOffset = cached.byteOffset initialState = cached.state - priorCalls = cached.calls + // Only the calls emitted BEFORE the cached task boundary are replayed. + // The rest belong to the task that was still open there and re-derive + // from this pass, which reads that task's records from its task_started + // and can therefore stamp its active timing. + priorCalls = cached.calls.slice(0, cached.callCount) for (const c of priorCalls) seenKeys.add(c.deduplicationKey) } // Stream raw lines (only the appended tail when resuming). Buffers for huge // lines pass straight into the decoder without a full string conversion. const records: (string | Buffer)[] = [] + // Byte offset AFTER each streamed record, parallel to `records`. The + // decoder reports its resume checkpoint as a record index (core never sees + // bytes); this is what turns it back into a file offset. + const offsetAfter: number[] = [] const tracker = { lastCompleteLineOffset: startByteOffset } let sawAnyLine = false for await (const rawLine of readSessionLines(source.path, undefined, { @@ -261,6 +272,7 @@ function createParser(source: SessionSource, seenKeys: Set): SessionPars })) { sawAnyLine = true records.push(rawLine) + offsetAfter.push(tracker.lastCompleteLineOffset) } // A cold decode that streamed nothing means the file was unreadable, @@ -268,7 +280,7 @@ function createParser(source: SessionSource, seenKeys: Set): SessionPars // pin an empty result set (mirrors the pre-phase-4 sawAnyLine guard). if (!sawAnyLine && !resume) return - const { calls: richCalls, diagnostics, state: newState } = decodeCodex({ + const { calls: richCalls, diagnostics, state: newState, checkpoint } = decodeCodex({ records, context: { privacyKey: '', providerId: 'codex', sourceRef: source.path }, state: initialState, @@ -290,16 +302,29 @@ function createParser(source: SessionSource, seenKeys: Set): SessionPars const newPriced = richCalls.map(toPricedProviderCall) const allCalls = resume ? [...priorCalls, ...newPriced] : newPriced - // Persist the state blob + host-priced calls + resume offset. seenKeys is + // Persist the state blob + host-priced calls + resume point. seenKeys is // stripped from the stored state (cross-file dedup is reconstructed each // run from the session cache, as the pre-phase-4 shared set was). - const storedState: CodexDecodeState = { ...newState, seenKeys: [] } + // + // The resume point is the decoder's last `task_started` checkpoint, the + // only offset where the per-task timing accumulators are provably empty. + // A pass that crossed no task boundary leaves the previous entry's + // boundary standing (nothing better exists) and, on a cold decode of a + // file with no task_started at all, falls back to end-of-file — a state + // that declares no open window, so nothing is attributed a partial one. + const resumePoint = checkpoint + ? { byteOffset: offsetAfter[checkpoint.recordIndex]!, state: checkpoint.state, callCount: priorCalls.length + checkpoint.callCount } + : resume && cached + ? { byteOffset: cached.byteOffset, state: cached.state, callCount: cached.callCount } + : { byteOffset: tracker.lastCompleteLineOffset, state: newState, callCount: allCalls.length } + const storedState: CodexDecodeState = { ...resumePoint.state, seenKeys: [] } await writeCodexCacheEntry(source.path, { mtimeMs: currentFp.mtimeMs, sizeBytes: currentFp.sizeBytes, project: source.project, - byteOffset: tracker.lastCompleteLineOffset, + byteOffset: resumePoint.byteOffset, state: storedState, + callCount: resumePoint.callCount, calls: allCalls, }) diff --git a/packages/cli/src/providers/types.ts b/packages/cli/src/providers/types.ts index 0cb3a6f0b..63912de60 100644 --- a/packages/cli/src/providers/types.ts +++ b/packages/cli/src/providers/types.ts @@ -76,6 +76,14 @@ export type ParsedProviderCall = { // Exact provider-recorded cwd, kept separately because projectPath may later // canonicalize a linked worktree to its main repository. workingDirectory?: string + // Tool-excluded active throughput: `activeDurationMs` is the enclosing task's + // duration minus its recorded tool-wait intervals, `activeGeneratedTokens` the + // task's generated tokens, both attributed to this call proportionally (Codex + // only). `toolWaitMs` is the excluded wait share. Present only when the task + // recorded both timing and generated tokens. + activeDurationMs?: number + activeGeneratedTokens?: number + toolWaitMs?: number } // A directory or database file that a provider's discoverSessions() scans. diff --git a/packages/cli/src/session-cache.ts b/packages/cli/src/session-cache.ts index 53f2e88ac..d1a4d521e 100644 --- a/packages/cli/src/session-cache.ts +++ b/packages/cli/src/session-cache.ts @@ -52,6 +52,10 @@ export type CachedCall = { toolErrors?: number // Codex: count of this call's patch applications with success === false. editFailed?: number + // Tool-excluded active throughput (Codex only), attributed per call. + activeDurationMs?: number + activeGeneratedTokens?: number + toolWaitMs?: number } export type CachedTurn = { @@ -290,7 +294,12 @@ export const PROVIDER_PARSE_VERSIONS: Record = { // rich-session-capture-v1: per-call LOC deltas + editFailed from // patch_apply_end. (The codex-results.json CODEX_CACHE_VERSION is bumped in // lockstep so the pre-session-cache layer re-parses too.) - codex: 'mcp-attribution-v2-est-cost-rich-capture-v1-cross-provider-pr-v1', + // active-timing-v1: calls now carry activeDurationMs / activeGeneratedTokens + // / toolWaitMs, and the codex-results resume point moved to the last + // task_started boundary (see codex-cache.ts v12). Cached turns hold neither, + // so bump in lockstep with that cache or session-cache.json keeps serving + // timing-less turns without ever invoking the parser. + codex: 'mcp-attribution-v2-est-cost-rich-capture-v1-cross-provider-pr-v1-active-timing-v1', cursor: 'composer-anchored-crediting-v1-est-cost', 'cursor-agent': 'workspaceless-transcript-v1', copilot: 'cli-shutdown-cost-v1-skills-dedup-key-hmac-v1', @@ -432,6 +441,9 @@ function validateCall(c: unknown): c is CachedCall { && (o['speed'] === 'standard' || o['speed'] === 'fast') && isOptionalNum(o['costUSD']) && isOptionalBool(o['isEstimated']) + && isOptionalNum(o['activeDurationMs']) + && isOptionalNum(o['activeGeneratedTokens']) + && isOptionalNum(o['toolWaitMs']) && isStringArray(o['tools']) && isStringArray(o['bashCommands']) && isStringArray(o['skills']) diff --git a/packages/cli/src/types.ts b/packages/cli/src/types.ts index 84727dce0..c325c3fc8 100644 --- a/packages/cli/src/types.ts +++ b/packages/cli/src/types.ts @@ -82,6 +82,13 @@ export type ParsedApiCall = { /// Count of this call's tool results flagged `is_error` (Claude tool_result /// blocks). Bash stderr alone is NOT counted (warnings go there). Omitted at 0. toolErrors?: number + /// Tool-excluded active throughput (Codex only): the task duration minus its + /// recorded tool-wait intervals, the task's generated tokens, and the excluded + /// wait share, attributed to this call proportionally. Omitted when the task + /// recorded no timing. + activeDurationMs?: number + activeGeneratedTokens?: number + toolWaitMs?: number } export type TaskCategory = @@ -191,7 +198,7 @@ export type SessionSummary = { /// from a provider that never captures branches (→ contributes nothing). /// Claude only; absent otherwise. everHadBranch?: boolean - modelBreakdown: Record + modelBreakdown: Record toolBreakdown: Record mcpBreakdown: Record bashBreakdown: Record diff --git a/packages/cli/tests/cli-codex-tps.test.ts b/packages/cli/tests/cli-codex-tps.test.ts new file mode 100644 index 000000000..4b39027ac --- /dev/null +++ b/packages/cli/tests/cli-codex-tps.test.ts @@ -0,0 +1,46 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { spawnSync } from 'node:child_process' +import { afterEach, describe, expect, it } from 'vitest' + +const homes: string[] = [] + +afterEach(async () => { + while (homes.length) await rm(homes.pop()!, { recursive: true, force: true }) +}) + +function runCli(args: string[], home: string) { + return spawnSync(process.execPath, ['--import', 'tsx', 'src/cli.ts', ...args], { + cwd: process.cwd(), + env: { ...process.env, HOME: home, CODEX_HOME: join(home, '.codex'), TZ: 'UTC' }, + encoding: 'utf-8', + timeout: 30_000, + }) +} + +describe('codex-tps CLI validation', () => { + it('rejects sub-second watch intervals', async () => { + const home = await mkdtemp(join(tmpdir(), 'codeburn-tps-cli-')) + homes.push(home) + const result = runCli(['codex-tps', '--watch', '0.1'], home) + expect(result.status).toBe(1) + expect(result.stderr).toContain('watch must be 0 or at least 1 second') + }) + + it('rejects JSON watch output instead of concatenating invalid JSON documents', async () => { + const home = await mkdtemp(join(tmpdir(), 'codeburn-tps-cli-')) + homes.push(home) + const result = runCli(['codex-tps', '--json', '--watch', '1'], home) + expect(result.status).toBe(2) + expect(result.stderr).toContain('--json cannot be combined with --watch') + }) + + it('returns a nonzero status for a missing explicit rollout', async () => { + const home = await mkdtemp(join(tmpdir(), 'codeburn-tps-cli-')) + homes.push(home) + const result = runCli(['codex-tps', join(home, 'missing.jsonl')], home) + expect(result.status).toBe(1) + expect(result.stderr).toContain('session file not found') + }) +}) diff --git a/packages/cli/tests/codex-resume.test.ts b/packages/cli/tests/codex-resume.test.ts index a5f90dbb2..fa01e9804 100644 --- a/packages/cli/tests/codex-resume.test.ts +++ b/packages/cli/tests/codex-resume.test.ts @@ -116,4 +116,52 @@ describe('codex append-resume through the CLI cache', () => { expect(v2.map(c => c.model)).toContain('SENTINEL-MODEL') }) + + it('attributes active timing across the append boundary (mid-task cut then task_complete)', async () => { + // The live-session cut: run 1 parses a rollout that ends mid-task + // (token_count emitted, task_complete not yet written), so its call has no + // timing. Run 2 appends the task_complete. The resume point is the task's + // own task_started, so run 2 re-reads the whole window and emits the call + // WITH timing — no already-emitted call is ever patched. + const TIMING_PREFIX = [ + JSON.stringify({ type: 'session_meta', timestamp: '2026-04-14T10:00:00Z', payload: { session_id: 'sess-timing', model: 'gpt-5.5', cwd: '/Users/t/p', originator: 'codex-cli' } }), + JSON.stringify({ type: 'event_msg', timestamp: '2026-04-14T10:00:00Z', payload: { type: 'task_started', turn_id: 'turn-1' } }), + userMessage('run the tool', '2026-04-14T10:00:01Z'), + JSON.stringify({ type: 'response_item', timestamp: '2026-04-14T10:00:02Z', payload: { type: 'function_call', call_id: 'call-1', name: 'exec_command' } }), + JSON.stringify({ type: 'response_item', timestamp: '2026-04-14T10:00:05Z', payload: { type: 'function_call_output', call_id: 'call-1', output: 'done' } }), + JSON.stringify({ type: 'event_msg', timestamp: '2026-04-14T10:01:10Z', payload: { type: 'token_count', info: { last_token_usage: { input_tokens: 100, output_tokens: 100, reasoning_output_tokens: 20, total_tokens: 220 }, total_token_usage: { total_tokens: 220 } } } }), + ] + const TIMING_COMPLETE = [ + JSON.stringify({ type: 'event_msg', timestamp: '2026-04-14T10:01:11Z', payload: { type: 'task_complete', duration_ms: 10_000 } }), + ] + + const filePath = await writeAt(tmpDir, 'rollout-timing-grow.jsonl', TIMING_PREFIX) + + // Run 1: cold decode, ends mid-task — no timing yet. + const v1 = await parseFile(filePath) + expect(v1).toHaveLength(1) + expect(v1[0]!.activeDurationMs).toBeUndefined() + + // The resume point rewound to the task_started rather than end-of-file: + // none of the open task's calls may be replayed, or run 2 would have to + // patch a call it already served. Revert the checkpoint to the end-of-file + // offset and callCount becomes 1 and the timing below never arrives. + const midEntry = await readCodexCacheEntry(filePath) + expect(midEntry!.callCount).toBe(0) + expect(midEntry!.state.taskOpen).toBe(true) + expect(midEntry!.calls).toHaveLength(1) + + // Run 2: the file grows by the task_complete; the codex-results cache + // resumes from the persisted task boundary. + await appendFile(filePath, TIMING_COMPLETE.join('\n') + '\n') + const v2 = await parseFile(filePath) + + // Cold decode of the full grown file: the resumed output must equal it. + const coldPath = await writeAt(tmpDir, 'rollout-timing-cold.jsonl', [...TIMING_PREFIX, ...TIMING_COMPLETE]) + const cold = await parseFile(coldPath) + expect(cold).toHaveLength(1) + expect(cold[0]).toMatchObject({ activeDurationMs: 7000, activeGeneratedTokens: 120, toolWaitMs: 3000 }) + + expect(v2).toEqual(cold) + }) }) diff --git a/packages/cli/tests/codex-throughput-cache-invalidation.test.ts b/packages/cli/tests/codex-throughput-cache-invalidation.test.ts new file mode 100644 index 000000000..287737a6f --- /dev/null +++ b/packages/cli/tests/codex-throughput-cache-invalidation.test.ts @@ -0,0 +1,110 @@ +// Regression for the stale-cache path of the codex active-timing port (same +// class as #478/#618). Two caches serve a codex session without ever invoking +// the decoder: session-cache.json (per provider, gated by envFingerprint) and +// codex-results.json (per file, gated by CODEX_CACHE_VERSION). A user upgrading +// into this change has both warm and both timing-less, so unless BOTH gates +// move, the dashboard's Tok/s column stays empty on every unchanged session +// forever. This drives the full parseAllSessions pipeline against caches seeded +// exactly as the pre-change release left them. +// +// Revert-proof: drop the `-active-timing-v1` suffix from the codex entry in +// PROVIDER_PARSE_VERSIONS and the seeded fingerprint matches again, the stale +// section is served, and the assertion below fails. + +import { describe, it, expect, beforeEach, afterAll, vi } from 'vitest' +import { mkdir, rm, readFile, writeFile } from 'fs/promises' +import { createHash } from 'crypto' +import { join } from 'path' + +import { clearSessionCache, parseAllSessions } from '../src/parser.js' +import { sessionCachePath } from '../src/session-cache.js' + +const testRoot = vi.hoisted(() => { + const root = `${process.env['TMPDIR'] || '/tmp'}/codex-timing-stale-${process.pid}-${Date.now()}` + process.env['HOME'] = `${root}/home` + process.env['USERPROFILE'] = `${root}/home` + process.env['CODEX_HOME'] = `${root}/codex` + return root +}) + +const CODEX_HOME = join(testRoot, 'codex') +const CACHE_DIR = join(testRoot, 'cache') + +// computeEnvFingerprint('codex') as the pre-change release computed it: the +// same CODEX_HOME, the same parse version minus this change's suffix. +const PRE_CHANGE_PARSE_VERSION = 'mcp-attribution-v2-est-cost-rich-capture-v1-cross-provider-pr-v1' + +function preChangeFingerprint(): string { + const parts = [`CODEX_HOME=${CODEX_HOME}`, `parser=${PRE_CHANGE_PARSE_VERSION}`] + return createHash('sha256').update(parts.join('\0')).digest('hex').slice(0, 16) +} + +beforeEach(() => { + process.env['HOME'] = join(testRoot, 'home') + process.env['USERPROFILE'] = join(testRoot, 'home') + process.env['CODEX_HOME'] = CODEX_HOME + process.env['CODEBURN_CACHE_DIR'] = CACHE_DIR +}) + +afterAll(async () => { + await rm(testRoot, { recursive: true, force: true }) +}) + +function timingTotals(projects: Awaited>): number[] { + return projects.flatMap(p => p.sessions.flatMap(s => Object.values(s.modelBreakdown).map(m => m.activeDurationMs ?? 0))) +} + +describe('codex active-timing invalidates both stale caches', () => { + it('re-parses an unchanged codex file cached by the pre-change release', async () => { + const sessionDir = join(CODEX_HOME, 'sessions', '2026', '04', '14') + await mkdir(sessionDir, { recursive: true }) + await mkdir(CACHE_DIR, { recursive: true }) + const lines = [ + JSON.stringify({ type: 'session_meta', timestamp: '2026-04-14T10:00:00Z', payload: { session_id: 'sess-timing-stale', model: 'gpt-5.5', cwd: '/Users/test/proj', originator: 'codex_cli_rs' } }), + JSON.stringify({ type: 'event_msg', timestamp: '2026-04-14T10:00:00Z', payload: { type: 'task_started' } }), + JSON.stringify({ type: 'response_item', timestamp: '2026-04-14T10:00:01Z', payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'run it' }] } }), + JSON.stringify({ type: 'event_msg', timestamp: '2026-04-14T10:00:08Z', payload: { type: 'token_count', info: { last_token_usage: { input_tokens: 300, output_tokens: 100 }, total_token_usage: { total_tokens: 400 } } } }), + JSON.stringify({ type: 'event_msg', timestamp: '2026-04-14T10:00:10Z', payload: { type: 'task_complete', duration_ms: 10_000 } }), + ] + await writeFile(join(sessionDir, 'rollout-timing-stale.jsonl'), lines.join('\n') + '\n') + + // Run 1: cold cache, current code. Timing present (sanity). + clearSessionCache() + const fresh = await parseAllSessions(undefined, 'codex') + expect(timingTotals(fresh)).toEqual([10_000]) + + // Rewrite both caches as the pre-change release left them: the old provider + // envFingerprint, the old codex-results version, and cached calls/turns + // with no timing at all. The rollout file itself is untouched, so nothing + // but the two version gates can trigger a re-parse. + const cachePath = sessionCachePath() + const cache = JSON.parse(await readFile(cachePath, 'utf8')) + cache.providers.codex.envFingerprint = preChangeFingerprint() + for (const f of Object.values(cache.providers.codex.files) as any[]) { + for (const turn of f.turns) { + for (const call of turn.calls) { + delete call.activeDurationMs + delete call.activeGeneratedTokens + delete call.toolWaitMs + } + } + } + await writeFile(cachePath, JSON.stringify(cache)) + + const codexCachePath = join(CACHE_DIR, 'codex-results.json') + const codexCache = JSON.parse(await readFile(codexCachePath, 'utf8')) + codexCache.version = 8 + for (const f of Object.values(codexCache.files) as any[]) { + for (const call of f.calls ?? []) { + delete call.activeDurationMs + delete call.activeGeneratedTokens + delete call.toolWaitMs + } + } + await writeFile(codexCachePath, JSON.stringify(codexCache)) + + clearSessionCache() + const second = await parseAllSessions(undefined, 'codex') + expect(timingTotals(second)).toEqual([10_000]) + }) +}) diff --git a/packages/cli/tests/codex-throughput-cache-roundtrip.test.ts b/packages/cli/tests/codex-throughput-cache-roundtrip.test.ts new file mode 100644 index 000000000..a1b5429a1 --- /dev/null +++ b/packages/cli/tests/codex-throughput-cache-roundtrip.test.ts @@ -0,0 +1,160 @@ +// End-to-end regression for the dashboard Tok/s column (activeDurationMs / +// activeGeneratedTokens / toolWaitMs). providerCallToCachedCall used to drop +// the three throughput fields when converting a parsed codex call into a +// cached turn, so a mapper-level unit test passed while the aggregated +// modelBreakdown (and with it the dashboard column) stayed empty — the exact +// failure this test guards against. It drives the full parseAllSessions +// pipeline twice: +// +// 1. cold: the rollout is parsed and written to session-cache.json through +// providerCallToCachedCall (the write hop); +// 2. warm: the file is byte-identical, so runParse serves the unchanged +// file's turns from the on-disk cache via cachedCallToApiCall (the read +// hop) without ever invoking the provider parser again. +// +// The fields must survive both hops to show up in modelBreakdown, which is the +// shape the dashboard aggregates (aggregateModelTotals) into its Tok/s column. + +import { describe, it, expect, beforeEach, afterAll, vi } from 'vitest' +import { mkdir, rm, writeFile, appendFile } from 'fs/promises' +import { join } from 'path' + +import { clearSessionCache, parseAllSessions } from '../src/parser.js' +import { aggregateModelTotals } from '../src/model-breakdown.js' + +const testRoot = vi.hoisted(() => { + const root = `${process.env['TMPDIR'] || '/tmp'}/codex-tps-roundtrip-${process.pid}-${Date.now()}` + process.env['HOME'] = `${root}/home` + process.env['USERPROFILE'] = `${root}/home` + process.env['CODEX_HOME'] = `${root}/codex` + return root +}) + +const CODEX_HOME = join(testRoot, 'codex') +const CACHE_DIR = join(testRoot, 'cache') + +beforeEach(() => { + process.env['HOME'] = join(testRoot, 'home') + process.env['USERPROFILE'] = join(testRoot, 'home') + process.env['CODEX_HOME'] = CODEX_HOME + process.env['CODEBURN_CACHE_DIR'] = CACHE_DIR +}) + +afterAll(async () => { + await rm(testRoot, { recursive: true, force: true }) +}) + +// The single codex session in a parseAllSessions result, and its only +// modelBreakdown entry (keyed by the friendly short name, whatever it resolves +// to for the fixture model). +function firstModelEntry(projects: Awaited>) { + expect(projects).toHaveLength(1) + const sessions = projects[0]!.sessions + expect(sessions).toHaveLength(1) + const entries = Object.entries(sessions[0]!.modelBreakdown) + expect(entries).toHaveLength(1) + return entries[0]![1] +} + +describe('codex active-throughput fields survive the session-cache round trip', () => { + it('reaches modelBreakdown on a cold parse AND a warm cache read', async () => { + const sessionDir = join(CODEX_HOME, 'sessions', '2026', '04', '14') + await mkdir(sessionDir, { recursive: true }) + await mkdir(CACHE_DIR, { recursive: true }) + + // Same shape as the fixture in tests/providers/codex.test.ts that yields + // activeDurationMs 7000 / activeGeneratedTokens 120 / toolWaitMs 3000: + // task_started -> 3s tool call (excluded as tool wait) -> + // token_count (100 output + 20 reasoning) -> task_complete duration_ms 10s. + const lines = [ + JSON.stringify({ type: 'session_meta', timestamp: '2026-04-14T10:00:00Z', payload: { session_id: 'sess-tps', model: 'gpt-5.5', cwd: '/Users/test/proj', originator: 'codex-cli' } }), + JSON.stringify({ type: 'event_msg', timestamp: '2026-04-14T10:00:00Z', payload: { type: 'task_started', turn_id: 'turn-1' } }), + JSON.stringify({ type: 'response_item', timestamp: '2026-04-14T10:00:00Z', payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'run the tool' }] } }), + JSON.stringify({ type: 'response_item', timestamp: '2026-04-14T10:00:02Z', payload: { type: 'function_call', call_id: 'call-1', name: 'exec_command' } }), + JSON.stringify({ type: 'response_item', timestamp: '2026-04-14T10:00:05Z', payload: { type: 'function_call_output', call_id: 'call-1', output: 'done' } }), + JSON.stringify({ type: 'event_msg', timestamp: '2026-04-14T10:01:10Z', payload: { type: 'token_count', info: { last_token_usage: { input_tokens: 100, output_tokens: 100, reasoning_output_tokens: 20, total_tokens: 220 }, total_token_usage: { total_tokens: 220 } } } }), + JSON.stringify({ type: 'event_msg', timestamp: '2026-04-14T10:01:11Z', payload: { type: 'task_complete', duration_ms: 10_000 } }), + ] + await writeFile(join(sessionDir, 'rollout-tps.jsonl'), lines.join('\n') + '\n') + + // Run 1: cold cache. The fresh parse is immediately converted to cached + // turns, so even this run crosses the providerCallToCachedCall write hop + // before the query-time aggregation reads the cached turns back. + clearSessionCache() + const fresh = await parseAllSessions(undefined, 'codex') + expect(firstModelEntry(fresh)).toMatchObject({ + activeDurationMs: 7000, + activeGeneratedTokens: 120, + toolWaitMs: 3000, + }) + + // Dashboard shape: aggregateModelTotals feeds the Tok/s column + // (activeGeneratedTokens / (activeDurationMs / 1000)). + const totals = aggregateModelTotals(fresh) + expect(Object.values(totals)).toHaveLength(1) + expect(Object.values(totals)[0]!).toMatchObject({ activeDurationMs: 7000, activeGeneratedTokens: 120 }) + + // Run 2: warm cache. The rollout is byte-identical, so the unchanged file + // is served straight from session-cache.json — the provider parser never + // runs. These fields only exist if run 1 actually wrote them. + clearSessionCache() + const warm = await parseAllSessions(undefined, 'codex') + expect(firstModelEntry(warm)).toMatchObject({ + activeDurationMs: 7000, + activeGeneratedTokens: 120, + toolWaitMs: 3000, + }) + }) + + it('reaches modelBreakdown on the append-resume path (mid-task cut then task_complete)', async () => { + // The live-session case the cold/warm round trip does NOT cover: run 1 + // parses while a task is still running (token_count emitted, task_complete + // not yet written), so the call is cached without timing. Run 2 re-parses + // the GROWN file incrementally — the codex-results cache resumes from its + // persisted state + byte offset and the carried task window attributes the + // three throughput fields to the earlier-pass call. Without that window, + // the fields stay missing exactly like they did before the mapper repair. + const sessionDir = join(CODEX_HOME, 'sessions', '2026', '04', '15') + // Hermetic: this `it` runs against whatever the previous one left behind + // (its rollout file, both caches, and the in-memory result cache). + await rm(join(CODEX_HOME, 'sessions'), { recursive: true, force: true }) + await mkdir(sessionDir, { recursive: true }) + await rm(CACHE_DIR, { recursive: true, force: true }) + await mkdir(CACHE_DIR, { recursive: true }) + + const midTaskLines = [ + JSON.stringify({ type: 'session_meta', timestamp: '2026-04-14T10:00:00Z', payload: { session_id: 'sess-append', model: 'gpt-5.5', cwd: '/Users/test/proj', originator: 'codex-cli' } }), + JSON.stringify({ type: 'event_msg', timestamp: '2026-04-14T10:00:00Z', payload: { type: 'task_started', turn_id: 'turn-1' } }), + JSON.stringify({ type: 'response_item', timestamp: '2026-04-14T10:00:00Z', payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'run the tool' }] } }), + JSON.stringify({ type: 'response_item', timestamp: '2026-04-14T10:00:02Z', payload: { type: 'function_call', call_id: 'call-1', name: 'exec_command' } }), + JSON.stringify({ type: 'response_item', timestamp: '2026-04-14T10:00:05Z', payload: { type: 'function_call_output', call_id: 'call-1', output: 'done' } }), + // NOTE: no task_complete yet — the rollout ends mid-task. + JSON.stringify({ type: 'event_msg', timestamp: '2026-04-14T10:01:10Z', payload: { type: 'token_count', info: { last_token_usage: { input_tokens: 100, output_tokens: 100, reasoning_output_tokens: 20, total_tokens: 220 }, total_token_usage: { total_tokens: 220 } } } }), + ] + const filePath = join(sessionDir, 'rollout-append.jsonl') + await writeFile(filePath, midTaskLines.join('\n') + '\n') + + // Run 1: mid-task parse — the call exists but has no timing yet. + clearSessionCache() + const midTask = await parseAllSessions(undefined, 'codex') + expect(firstModelEntry(midTask)).not.toHaveProperty('activeDurationMs') + + // The task completes: Codex appends the task_complete line. + await appendFile(filePath, JSON.stringify({ type: 'event_msg', timestamp: '2026-04-14T10:01:11Z', payload: { type: 'task_complete', duration_ms: 10_000 } }) + '\n') + + // Run 2: the grown file re-parses through the append-resume path; the + // fields must now survive into modelBreakdown. + clearSessionCache() + const appended = await parseAllSessions(undefined, 'codex') + expect(firstModelEntry(appended)).toMatchObject({ + activeDurationMs: 7000, + activeGeneratedTokens: 120, + toolWaitMs: 3000, + }) + + // Dashboard shape: the Tok/s column aggregates the same fields. + const totals = aggregateModelTotals(appended) + expect(Object.values(totals)).toHaveLength(1) + expect(Object.values(totals)[0]!).toMatchObject({ activeDurationMs: 7000, activeGeneratedTokens: 120 }) + }) +}) diff --git a/packages/cli/tests/codex-throughput.test.ts b/packages/cli/tests/codex-throughput.test.ts new file mode 100644 index 000000000..1e0fd4da7 --- /dev/null +++ b/packages/cli/tests/codex-throughput.test.ts @@ -0,0 +1,124 @@ +import { appendFile, mkdtemp, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { CodexThroughputReader, readCodexThroughput, renderCodexThroughput } from '../src/codex-throughput.js' + +describe('Codex throughput prototype', () => { + it('estimates generated tokens/sec between token_count checkpoints', async () => { + const dir = await mkdtemp(join(tmpdir(), 'codeburn-tps-')) + const path = join(dir, 'rollout.jsonl') + await writeFile(path, [ + JSON.stringify({ type: 'session_meta', timestamp: '2026-07-25T00:00:00.000Z', payload: { model: 'gpt-5.6-sol' } }), + JSON.stringify({ type: 'event_msg', timestamp: '2026-07-25T00:00:00.000Z', payload: { type: 'task_started' } }), + JSON.stringify({ type: 'response_item', timestamp: '2026-07-25T00:00:02.000Z', payload: { type: 'function_call', call_id: 'tool-1' } }), + JSON.stringify({ type: 'response_item', timestamp: '2026-07-25T00:00:05.000Z', payload: { type: 'function_call_output', call_id: 'tool-1' } }), + JSON.stringify({ type: 'event_msg', timestamp: '2026-07-25T00:00:05.000Z', payload: { type: 'mcp_tool_call_end', duration: { secs: 3, nanos: 0 } } }), + JSON.stringify({ type: 'event_msg', timestamp: '2026-07-25T00:00:10.000Z', payload: { type: 'token_count', info: { last_token_usage: { output_tokens: 80, reasoning_output_tokens: 20 }, total_token_usage: { total_tokens: 100, output_tokens: 80, reasoning_output_tokens: 20 } } } }), + JSON.stringify({ type: 'event_msg', timestamp: '2026-07-25T00:00:15.000Z', payload: { type: 'token_count', info: { last_token_usage: { output_tokens: 40, reasoning_output_tokens: 10 }, total_token_usage: { total_tokens: 150, output_tokens: 120, reasoning_output_tokens: 30 } } } }), + JSON.stringify({ type: 'event_msg', timestamp: '2026-07-25T00:00:16.000Z', payload: { type: 'task_complete', duration_ms: 10000 } }), + ].join('\n')) + + const points = await readCodexThroughput(path) + expect(points).toHaveLength(2) + expect(points[1]).toMatchObject({ generatedTokens: 50, elapsedSeconds: 5, generatedTokensPerSecond: 10, activeDurationSeconds: 7, activeGeneratedTokensPerSecond: 21.428571428571427, toolWaitSeconds: 3, model: 'gpt-5.6-sol' }) + expect(renderCodexThroughput(points, path)).toContain('21.4 generated tokens/sec') + }) + + it('parses only appended complete lines while watching a growing rollout', async () => { + const dir = await mkdtemp(join(tmpdir(), 'codeburn-tps-watch-')) + const path = join(dir, 'rollout.jsonl') + const first = JSON.stringify({ type: 'event_msg', timestamp: '2026-07-25T00:00:00.000Z', payload: { type: 'token_count', info: { last_token_usage: { output_tokens: 8, reasoning_output_tokens: 2 }, total_token_usage: { total_tokens: 10, output_tokens: 8, reasoning_output_tokens: 2 } } } }) + const second = JSON.stringify({ type: 'event_msg', timestamp: '2026-07-25T00:00:01.000Z', payload: { type: 'token_count', info: { last_token_usage: { output_tokens: 4, reasoning_output_tokens: 1 }, total_token_usage: { total_tokens: 15, output_tokens: 12, reasoning_output_tokens: 3 } } } }) + await writeFile(path, first.slice(0, 40)) + const reader = new CodexThroughputReader() + expect(await reader.update(path)).toEqual([]) + await appendFile(path, first.slice(40) + '\n' + second + '\n') + const points = await reader.update(path) + expect(points).toHaveLength(2) + expect(points[1]).toMatchObject({ generatedTokens: 5, generatedTokensPerSecond: 5 }) + }) + + it('ignores replayed pre-fork checkpoints before estimating new work', async () => { + const dir = await mkdtemp(join(tmpdir(), 'codeburn-tps-fork-')) + const path = join(dir, 'rollout.jsonl') + const line = (timestamp: string, payload: Record) => JSON.stringify({ type: 'event_msg', timestamp, payload }) + await writeFile(path, [ + JSON.stringify({ type: 'session_meta', timestamp: '2026-07-25T00:00:00.000Z', payload: { model: 'gpt-5.6-sol', forked_from_id: 'parent' } }), + line('2026-07-25T00:00:01.000Z', { type: 'task_started' }), + line('2026-07-25T00:00:02.000Z', { type: 'token_count', info: { last_token_usage: { output_tokens: 100 }, total_token_usage: { total_tokens: 100, output_tokens: 100 } } }), + line('2026-07-25T00:00:03.000Z', { type: 'task_complete', duration_ms: 1000 }), + line('2026-07-25T00:00:06.000Z', { type: 'task_started' }), + line('2026-07-25T00:00:08.000Z', { type: 'token_count', info: { last_token_usage: { output_tokens: 20 }, total_token_usage: { total_tokens: 20, output_tokens: 20 } } }), + line('2026-07-25T00:00:10.000Z', { type: 'task_complete', duration_ms: 4000 }), + ].join('\n')) + + const points = await readCodexThroughput(path) + expect(points).toHaveLength(1) + expect(points[0]).toMatchObject({ generatedTokens: 20, activeGeneratedTokensPerSecond: 5 }) + }) + + it('keeps oversized rollout lines bounded while extracting token usage', async () => { + const dir = await mkdtemp(join(tmpdir(), 'codeburn-tps-large-')) + const path = join(dir, 'rollout.jsonl') + const largeResult = JSON.stringify({ + type: 'event_msg', + timestamp: '2026-07-25T00:00:01.000Z', + payload: { + type: 'token_count', + info: { last_token_usage: { output_tokens: 12 }, total_token_usage: { total_tokens: 12, output_tokens: 12 } }, + result: 'x'.repeat(5 * 1024 * 1024), + }, + }) + await writeFile(path, largeResult) + const points = await readCodexThroughput(path) + expect(points).toHaveLength(1) + expect(points[0]?.generatedTokens).toBe(12) + }) + + it('keeps MCP duration when arguments and result surround the middle field', async () => { + const dir = await mkdtemp(join(tmpdir(), 'codeburn-tps-mcp-large-')) + const path = join(dir, 'rollout.jsonl') + const mcp = JSON.stringify({ + type: 'event_msg', timestamp: '2026-07-25T00:00:05.000Z', + payload: { + type: 'mcp_tool_call_end', + invocation: { server: 'github', tool: 'get_issue', arguments: { body: 'x'.repeat(5 * 1024 * 1024) } }, + duration: { secs: 3, nanos: 0 }, + result: { duration: '9s', text: 'x'.repeat(5 * 1024 * 1024) }, + }, + }) + await writeFile(path, [ + JSON.stringify({ type: 'session_meta', timestamp: '2026-07-25T00:00:00.000Z', payload: { model: 'gpt-5.6-sol' } }), + JSON.stringify({ type: 'event_msg', timestamp: '2026-07-25T00:00:00.000Z', payload: { type: 'task_started' } }), + mcp, + JSON.stringify({ type: 'event_msg', timestamp: '2026-07-25T00:00:08.000Z', payload: { type: 'token_count', info: { last_token_usage: { output_tokens: 100 }, total_token_usage: { total_tokens: 100, output_tokens: 100 } } } }), + JSON.stringify({ type: 'event_msg', timestamp: '2026-07-25T00:00:10.000Z', payload: { type: 'task_complete', duration_ms: 10000 } }), + ].join('\n')) + const points = await readCodexThroughput(path) + expect(points[0]).toMatchObject({ toolWaitSeconds: 3, activeDurationSeconds: 7, activeGeneratedTokensPerSecond: 100 / 7 }) + }) + + it('keeps a streamed string MCP duration when arguments and result surround the middle field', async () => { + const dir = await mkdtemp(join(tmpdir(), 'codeburn-tps-mcp-string-large-')) + const path = join(dir, 'rollout.jsonl') + const mcp = JSON.stringify({ + type: 'event_msg', timestamp: '2026-07-25T00:00:05.000Z', + payload: { + type: 'mcp_tool_call_end', + invocation: { server: 'github', tool: 'get_issue', arguments: { body: 'x'.repeat(5 * 1024 * 1024) } }, + duration: '3s', + result: { duration: '9s', text: 'x'.repeat(5 * 1024 * 1024) }, + }, + }) + await writeFile(path, [ + JSON.stringify({ type: 'session_meta', timestamp: '2026-07-25T00:00:00.000Z', payload: { model: 'gpt-5.6-sol' } }), + JSON.stringify({ type: 'event_msg', timestamp: '2026-07-25T00:00:00.000Z', payload: { type: 'task_started' } }), + mcp, + JSON.stringify({ type: 'event_msg', timestamp: '2026-07-25T00:00:08.000Z', payload: { type: 'token_count', info: { last_token_usage: { output_tokens: 100 }, total_token_usage: { total_tokens: 100, output_tokens: 100 } } } }), + JSON.stringify({ type: 'event_msg', timestamp: '2026-07-25T00:00:10.000Z', payload: { type: 'task_complete', duration_ms: 10000 } }), + ].join('\n')) + const points = await readCodexThroughput(path) + expect(points[0]).toMatchObject({ toolWaitSeconds: 3, activeDurationSeconds: 7, activeGeneratedTokensPerSecond: 100 / 7 }) + }) +}) diff --git a/packages/cli/tests/providers/codex.test.ts b/packages/cli/tests/providers/codex.test.ts index 012abc4e8..ddf277d1b 100644 --- a/packages/cli/tests/providers/codex.test.ts +++ b/packages/cli/tests/providers/codex.test.ts @@ -874,4 +874,235 @@ describe('codex provider - token_count timestamps cannot poison the day aggregat expect(days.map(d => d.date)).toEqual(['2026-04-14']) expect(days[0]!.calls).toBe(1) }) + + it('does not treat a nested session_meta model as the active turn model', async () => { + const largeSessionMeta = JSON.stringify({ + type: 'session_meta', + timestamp: '2026-04-14T10:00:00Z', + payload: { + cwd: '/Users/test/model-switch', + originator: 'codex-cli', + session_id: 'sess-model-switch', + base_instructions: { + provenance: { type: 'model', model: 'gpt-5.6-sol' }, + text: 'x'.repeat(40_000), + }, + }, + }) + const turnContext = JSON.stringify({ + type: 'turn_context', + timestamp: '2026-04-14T10:00:01Z', + payload: { model: 'gpt-5.6-luna' }, + }) + const filePath = await writeSession(tmpDir, '2026-04-14', 'rollout-model-switch.jsonl', [ + largeSessionMeta, + turnContext, + tokenCount({ timestamp: '2026-04-14T10:00:02Z', last: { input: 100, output: 50 }, total: { total: 150 } }), + largeSessionMeta, + tokenCount({ timestamp: '2026-04-14T10:00:03Z', last: { input: 200, output: 100 }, total: { total: 450 } }), + ]) + + const provider = createCodexProvider(tmpDir) + const source = { path: filePath, project: 'test', provider: 'codex' } + const calls: ParsedProviderCall[] = [] + for await (const call of provider.createSessionParser(source, new Set()).parse()) calls.push(call) + + expect(calls.map(call => call.model)).toEqual(['gpt-5.6-luna', 'gpt-5.6-luna']) + }) + + it('reads session_meta cwd/session_id/originator at payload depth 1, not the first nested same-name key', async () => { + const largeSessionMeta = JSON.stringify({ + type: 'session_meta', + timestamp: '2026-04-14T10:00:00Z', + payload: { + dynamic_tools: [{ + name: 'shadow-tool', + cwd: '/shadow/cwd', + originator: 'shadow-originator', + session_id: 'shadow-session', + forked_from_id: 'shadow-fork', + model_provider: 'shadow-provider', + }], + base_instructions: { text: 'x'.repeat(40_000) }, + cwd: '/Users/test/real-project', + originator: 'codex-cli', + session_id: 'sess-real', + model: 'gpt-5.6-luna', + model_provider: 'openai', + name: 'real-session-name', + }, + }) + const filePath = await writeSession(tmpDir, '2026-04-14', 'rollout-nested-keys.jsonl', [ + largeSessionMeta, + functionCall('exec_command'), + tokenCount({ timestamp: '2026-04-14T10:01:00Z', last: { input: 100, output: 50 }, total: { total: 150 } }), + ]) + + const provider = createCodexProvider(tmpDir) + const source = { path: filePath, project: 'test', provider: 'codex' } + const calls: ParsedProviderCall[] = [] + for await (const call of provider.createSessionParser(source, new Set()).parse()) calls.push(call) + + expect(calls).toHaveLength(1) + expect(calls[0]!.sessionId).toBe('sess-real') + expect(calls[0]!.workingDirectory).toBe('/Users/test/real-project') + expect(calls[0]!.projectPath).toBe('/Users/test/real-project') + expect(calls[0]!.model).toBe('gpt-5.6-luna') + expect(calls[0]!.tools).toEqual(['Bash']) + }) + + it('parses large rollout lines and computes active timing across a tool call', async () => { + // The oversized line under test is task_complete: it appends the final + // assistant message BEFORE its duration fields, so a head-only scan would + // lose the timing entirely. + const largeCompleteLine = JSON.stringify({ + type: 'event_msg', + timestamp: '2026-04-14T10:01:11Z', + payload: { type: 'task_complete', last_agent_message: 'x'.repeat(40_000), duration_ms: 10_000 }, + }) + const filePath = await writeSession(tmpDir, '2026-04-14', 'rollout-timing.jsonl', [ + sessionMeta({ session_id: 'sess-timing', model: 'gpt-5.5' }), + JSON.stringify({ type: 'event_msg', timestamp: '2026-04-14T10:00:00Z', payload: { type: 'task_started', turn_id: 'turn-1' } }), + userMessage('run the tool'), + JSON.stringify({ type: 'response_item', timestamp: '2026-04-14T10:00:02Z', payload: { type: 'function_call', call_id: 'call-1', name: 'exec_command' } }), + JSON.stringify({ type: 'response_item', timestamp: '2026-04-14T10:00:05Z', payload: { type: 'function_call_output', call_id: 'call-1', output: 'done' } }), + tokenCount({ timestamp: '2026-04-14T10:01:10Z', last: { input: 100, output: 100, reasoning: 20 }, total: { input: 100, output: 100, reasoning: 20, total: 220 } }), + largeCompleteLine, + ]) + + const provider = createCodexProvider(tmpDir) + const source = { path: filePath, project: 'test', provider: 'codex' } + const calls: ParsedProviderCall[] = [] + for await (const call of provider.createSessionParser(source, new Set()).parse()) calls.push(call) + + expect(calls).toHaveLength(1) + expect(calls[0]).toMatchObject({ + outputTokens: 100, + reasoningTokens: 20, + tools: ['Bash'], + activeDurationMs: 7000, + activeGeneratedTokens: 120, + toolWaitMs: 3000, + }) + }) + + it('subtracts native MCP wait time from active timing', async () => { + const filePath = await writeSession(tmpDir, '2026-04-14', 'rollout-mcp-timing.jsonl', [ + sessionMeta({ session_id: 'sess-mcp-timing', model: 'gpt-5.5' }), + JSON.stringify({ type: 'event_msg', timestamp: '2026-04-14T10:00:00Z', payload: { type: 'task_started' } }), + userMessage('look up the issue'), + JSON.stringify({ + type: 'event_msg', + timestamp: '2026-04-14T10:00:05Z', + payload: { + type: 'mcp_tool_call_end', + call_id: 'mcp-1', + invocation: { server: 'github', tool: 'get_issue', arguments: {} }, + duration: { secs: 3, nanos: 0 }, + }, + }), + tokenCount({ + timestamp: '2026-04-14T10:00:08Z', + last: { input: 300, output: 100 }, + total: { total: 400 }, + }), + JSON.stringify({ type: 'event_msg', timestamp: '2026-04-14T10:00:10Z', payload: { type: 'task_complete', duration_ms: 10_000 } }), + ]) + + const provider = createCodexProvider(tmpDir) + const source = { path: filePath, project: 'test', provider: 'codex' } + const calls: ParsedProviderCall[] = [] + for await (const call of provider.createSessionParser(source, new Set()).parse()) calls.push(call) + + expect(calls).toHaveLength(1) + expect(calls[0]).toMatchObject({ activeDurationMs: 7000, toolWaitMs: 3000 }) + }) + + it('prefers payload-level duration over a nested duration_ms in large mcp_tool_call_end lines', async () => { + // Regression guard: a naive first-match regex would pick up the + // `duration_ms: 9999` inside invocation.arguments instead of the payload-level + // `duration: { secs: 3 }`. The depth-aware payload scan must win. + const largeMcpLine = JSON.stringify({ + type: 'event_msg', + timestamp: '2026-04-14T10:00:05Z', + payload: { + type: 'mcp_tool_call_end', + call_id: 'mcp-duration-collision', + invocation: { server: 'github', tool: 'get_issue', arguments: { duration_ms: 9999, body: 'x'.repeat(40_000) } }, + duration: { secs: 3, nanos: 0 }, + result: { Ok: { content: [{ type: 'text', text: 'x'.repeat(40_000) }] } }, + }, + }) + const filePath = await writeSession(tmpDir, '2026-04-14', 'rollout-mcp-duration-collision.jsonl', [ + sessionMeta({ session_id: 'sess-mcp-duration-collision', model: 'gpt-5.5' }), + JSON.stringify({ type: 'event_msg', timestamp: '2026-04-14T10:00:00Z', payload: { type: 'task_started' } }), + userMessage('look up the issue'), + largeMcpLine, + tokenCount({ timestamp: '2026-04-14T10:00:08Z', last: { input: 300, output: 100 }, total: { total: 400 } }), + JSON.stringify({ type: 'event_msg', timestamp: '2026-04-14T10:00:10Z', payload: { type: 'task_complete', duration_ms: 10_000 } }), + ]) + + const provider = createCodexProvider(tmpDir) + const source = { path: filePath, project: 'test', provider: 'codex' } + const calls: ParsedProviderCall[] = [] + for await (const call of provider.createSessionParser(source, new Set()).parse()) calls.push(call) + + expect(calls).toHaveLength(1) + // 3s of the 10s task is MCP wait: the payload-level `duration` won over the + // 9999 buried in invocation.arguments (which would have zeroed active time). + expect(calls[0]).toMatchObject({ activeDurationMs: 7000, toolWaitMs: 3000 }) + }) + + it('attributes a task_complete over everything since the last task_started, even across a suppressed one', async () => { + // A mid-file session_meta carrying forked_from_id re-arms the fork-replay + // cutoff, which swallows the task_started right behind it while its + // task_complete lands past the cutoff. Attribution then has to span both + // turns, exactly as it did before calls were buffered per task. + const filePath = await writeSession(tmpDir, '2026-04-14', 'rollout-suppressed-task-start.jsonl', [ + sessionMeta({ session_id: 'sess-suppressed-start', model: 'gpt-5.5' }), + JSON.stringify({ type: 'event_msg', timestamp: '2026-04-14T10:00:00Z', payload: { type: 'task_started' } }), + userMessage('first ask'), + tokenCount({ timestamp: '2026-04-14T10:00:05Z', last: { input: 300, output: 100 }, total: { total: 400 } }), + JSON.stringify({ type: 'event_msg', timestamp: '2026-04-14T10:00:10Z', payload: { type: 'task_complete', duration_ms: 10_000 } }), + sessionMeta({ timestamp: '2026-04-14T10:00:11Z', session_id: 'sess-suppressed-start', model: 'gpt-5.5', forked_from_id: 'sess-parent' }), + JSON.stringify({ type: 'event_msg', timestamp: '2026-04-14T10:00:12Z', payload: { type: 'task_started' } }), + userMessage('second ask', '2026-04-14T10:00:18Z'), + tokenCount({ timestamp: '2026-04-14T10:00:20Z', last: { input: 300, output: 300 }, total: { total: 1000 } }), + JSON.stringify({ type: 'event_msg', timestamp: '2026-04-14T10:00:25Z', payload: { type: 'task_complete', duration_ms: 5_000 } }), + ]) + + const provider = createCodexProvider(tmpDir) + const source = { path: filePath, project: 'test', provider: 'codex' } + const calls: ParsedProviderCall[] = [] + for await (const call of provider.createSessionParser(source, new Set()).parse()) calls.push(call) + + expect(calls).toHaveLength(2) + // The second task_complete re-attributes the first turn too, so the 5s + // window is split across both by generated tokens rather than leaving the + // first turn pinned to its own 10s window. + expect(calls[0]!.activeDurationMs).toBeCloseTo(1250, 6) + expect(calls[1]!.activeDurationMs).toBeCloseTo(3750, 6) + expect(calls[0]!.activeDurationMs! + calls[1]!.activeDurationMs!).toBeCloseTo(5000, 6) + }) + + it('omits active timing when recorded tool wait consumes the task duration', async () => { + const filePath = await writeSession(tmpDir, '2026-04-14', 'rollout-degenerate-timing.jsonl', [ + sessionMeta({ session_id: 'sess-degenerate-timing', model: 'gpt-5.5' }), + JSON.stringify({ type: 'event_msg', timestamp: '2026-04-14T10:00:00Z', payload: { type: 'task_started' } }), + userMessage('wait for the tool'), + JSON.stringify({ type: 'response_item', timestamp: '2026-04-14T10:00:00Z', payload: { type: 'function_call', call_id: 'call-1', name: 'exec_command' } }), + JSON.stringify({ type: 'response_item', timestamp: '2026-04-14T10:00:10Z', payload: { type: 'function_call_output', call_id: 'call-1', output: 'done' } }), + tokenCount({ timestamp: '2026-04-14T10:00:12Z', last: { input: 300, output: 100 }, total: { total: 400 } }), + JSON.stringify({ type: 'event_msg', timestamp: '2026-04-14T10:00:13Z', payload: { type: 'task_complete', duration_ms: 10_000 } }), + ]) + + const provider = createCodexProvider(tmpDir) + const source = { path: filePath, project: 'test', provider: 'codex' } + const calls: ParsedProviderCall[] = [] + for await (const call of provider.createSessionParser(source, new Set()).parse()) calls.push(call) + + expect(calls).toHaveLength(1) + expect(calls[0]!.activeDurationMs).toBeUndefined() + expect(calls[0]!.toolWaitMs).toBeUndefined() + }) }) diff --git a/packages/core/src/providers/codex/decode.ts b/packages/core/src/providers/codex/decode.ts index c441ae732..7cd064fb1 100644 --- a/packages/core/src/providers/codex/decode.ts +++ b/packages/core/src/providers/codex/decode.ts @@ -100,6 +100,106 @@ function payloadHead(head: string): string { return idx === -1 ? head : head.slice(idx) } +function getRawJsonNumberField(head: string, field: string): number | undefined { + const match = new RegExp(`"${field}"\\s*:\\s*(-?\\d+(?:\\.\\d+)?)`).exec(head) + if (!match) return undefined + const value = Number(match[1]) + return Number.isFinite(value) ? value : undefined +} + +// Return a small window of `source` starting at the DEPTH-1 payload key `field`, +// i.e. the payload's own key rather than the first same-named key anywhere +// inside it. A compact head scan takes the first match it sees, which on a +// nested object (session_meta embeds base_instructions / dynamic_tools; an MCP +// record embeds invocation.arguments) is the wrong one. +function getRawPayloadFieldWindow(source: Buffer, field: string, windowBytes = 4096): string | undefined { + const payloadKey = Buffer.from('"payload"') + const payloadIndex = source.indexOf(payloadKey) + if (payloadIndex < 0) return undefined + const payloadStart = source.indexOf(0x7b, payloadIndex + payloadKey.length) // { + if (payloadStart < 0) return undefined + + let depth = 0 + let inString = false + let escaped = false + for (let i = payloadStart; i < source.length; i++) { + const byte = source[i]! + if (inString) { + if (escaped) escaped = false + else if (byte === 0x5c) escaped = true // backslash + else if (byte === 0x22) inString = false // " + continue + } + if (byte === 0x22) { + const keyStart = i + 1 + let keyEnd = keyStart + let keyEscaped = false + for (; keyEnd < source.length; keyEnd++) { + const keyByte = source[keyEnd]! + if (keyEscaped) { keyEscaped = false; continue } + if (keyByte === 0x5c) { keyEscaped = true; continue } + if (keyByte === 0x22) break + } + if (depth === 1 && keyEnd < source.length) { + const key = source.subarray(keyStart, keyEnd).toString('utf-8') + let valueStart = keyEnd + 1 + while (valueStart < source.length && (source[valueStart] === 0x20 || source[valueStart] === 0x09 || source[valueStart] === 0x0a || source[valueStart] === 0x0d)) valueStart++ + if (source[valueStart] === 0x3a && key === field) { + return source.subarray(i, Math.min(source.length, i + windowBytes)).toString('utf-8') + } + } + i = keyEnd + inString = false + continue + } + if (byte === 0x22) inString = true + else if (byte === 0x7b || byte === 0x5b) depth++ // { or [ + else if (byte === 0x7d || byte === 0x5d) depth-- // } or ] + if (depth < 0) break + } + return undefined +} + +function getRawDurationMs(head: string): number | undefined { + const objectMatch = /"duration"\s*:\s*\{\s*"secs"\s*:\s*(-?\d+(?:\.\d+)?)\s*,\s*"nanos"\s*:\s*(-?\d+(?:\.\d+)?)\s*\}/.exec(head) + if (objectMatch) { + const seconds = Number(objectMatch[1]) + const nanos = Number(objectMatch[2]) + if (Number.isFinite(seconds) && Number.isFinite(nanos)) return seconds * 1000 + nanos / 1e6 + } + const text = getRawJsonStringField(head, 'duration') + if (text) { + const match = /^(\d+(?:\.\d+)?)(ms|s)?$/.exec(text.trim()) + if (match) { + const value = Number(match[1]) + if (Number.isFinite(value)) return value * (match[2] === 's' ? 1000 : 1) + } + } + return undefined +} + +// The parsed (small-line) form of the same field: `duration` is a number of ms, +// a Rust Duration object, or a "1500ms"/"1.5s" string depending on the writer. +function durationValueMs(value: unknown): number | undefined { + if (typeof value === 'number' && Number.isFinite(value)) return value + if (typeof value === 'object' && value) { + const record = value as Record + const seconds = record['secs'] + const nanos = record['nanos'] + if (typeof seconds === 'number' && typeof nanos === 'number' && Number.isFinite(seconds) && Number.isFinite(nanos)) { + return seconds * 1000 + nanos / 1e6 + } + } + if (typeof value === 'string') { + const match = /^(\d+(?:\.\d+)?)(ms|s)?$/.exec(value.trim()) + if (match) { + const parsed = Number(match[1]) + if (Number.isFinite(parsed)) return parsed * (match[2] === 's' ? 1000 : 1) + } + } + return undefined +} + function countJsonStringBytes(source: Buffer, valueStart: number): number { let count = 0 for (let i = valueStart; i < source.length; i++) { @@ -172,6 +272,32 @@ export function parseCodexLine(line: string | Buffer): CodexEntry | null { const pHead = payloadHead(head) const payloadType = getRawJsonStringField(pHead, 'type') const role = getRawJsonStringField(pHead, 'role') + // task_complete appends the potentially huge final assistant message before + // its duration fields. Fall back to the full Buffer only for this event so + // timing metadata is not lost when the compact head stops early. + const needsTimingTail = type === 'event_msg' && (payloadType === 'task_complete' || payloadType === 'mcp_tool_call_end') + const timingTail = needsTimingTail && line.length > RAW_HEAD_BYTES + ? line.subarray(Math.max(0, line.length - 16 * 1024)).toString('utf-8') + : pHead + const timingNumber = (field: string): number | undefined => + getRawJsonNumberField(pHead, field) ?? getRawJsonNumberField(timingTail, field) + // MCP records can place a large invocation.arguments object before duration + // and a large result after it. Searching a small window around the field + // avoids materializing the middle of the Buffer while still preserving wait + // timing for those records. + const payloadDuration = payloadType === 'mcp_tool_call_end' + ? getRawDurationMs(getRawPayloadFieldWindow(line, 'duration') ?? '') + : undefined + const timingDuration = payloadDuration ?? getRawDurationMs(pHead) ?? getRawDurationMs(timingTail) + // session_meta can embed same-name keys under base_instructions / + // dynamic_tools (including provenance.model). A depth-agnostic scan of the + // compact head steals the first nested hit and can overwrite turn_context. + // Restrict every session_meta string field to payload depth 1. Other event + // types keep the cheap first-match scan. + const payloadString = (field: string): string | undefined => + type === 'session_meta' + ? getRawJsonStringField(getRawPayloadFieldWindow(line, field) ?? '', field) + : getRawJsonStringField(pHead, field) const entry: CodexEntry = { type, @@ -179,13 +305,21 @@ export function parseCodexLine(line: string | Buffer): CodexEntry | null { payload: { type: payloadType, role, - cwd: getRawJsonStringField(pHead, 'cwd'), - model_provider: getRawJsonStringField(pHead, 'model_provider'), - originator: getRawJsonStringField(pHead, 'originator'), - session_id: getRawJsonStringField(pHead, 'session_id'), - forked_from_id: getRawJsonStringField(pHead, 'forked_from_id'), - model: getRawJsonStringField(pHead, 'model'), - name: getRawJsonStringField(pHead, 'name'), + cwd: payloadString('cwd'), + model_provider: payloadString('model_provider'), + originator: payloadString('originator'), + session_id: payloadString('session_id'), + forked_from_id: payloadString('forked_from_id'), + model: payloadString('model'), + name: payloadString('name'), + call_id: getRawJsonStringField(pHead, 'call_id'), + turn_id: getRawJsonStringField(pHead, 'turn_id'), + // On mcp_tool_call_end a coincidental `duration_ms` inside the large + // invocation.arguments object can shadow the payload-level duration, so the + // depth-aware value wins. The naive scan stays as the fallback for + // task_complete, which records duration_ms at the payload level directly. + duration_ms: timingDuration ?? timingNumber('duration_ms'), + started_at: timingNumber('started_at'), }, } @@ -297,10 +431,26 @@ export type CodexDecodeInput = { sessionIdFallback?: string } +/** + * The last `task_started` boundary this pass crossed: the only point where a + * later pass may restart and still rebuild a whole task-timing window, because + * every per-task accumulator is empty there. `recordIndex` is the index of the + * task_started record (the host maps it to the byte offset AFTER that line); + * `callCount` is how many calls had been emitted at that point, so the host can + * replay exactly those and let the rest re-decode. Absent when the pass saw no + * task_started at all. + */ +export type CodexResumeCheckpoint = { + recordIndex: number + callCount: number + state: CodexDecodeState +} + export type CodexDecodeResult = { calls: CodexDecodedCall[] diagnostics: RecordDiagnostic[] state: CodexDecodeState + checkpoint?: CodexResumeCheckpoint } /** @@ -318,10 +468,56 @@ export function decodeCodex({ records, state: prevState, seenKeys: liveSeen, ses const calls: CodexDecodedCall[] = [] const diagnostics: RecordDiagnostic[] = [] + // Tool-excluded active timing. Calls decoded since the last task_started are + // held back here so task_complete can stamp them BEFORE they are appended to + // `calls`: emitting a task only once its timing is known keeps single-pass and + // resumed decodes in agreement instead of back-patching a call already handed + // to the host. Bounded by one task's calls; flushed at the next task_started + // and at end of input. + let pendingTaskCalls: CodexDecodedCall[] = [] + let taskGeneratedTokens = 0 + let taskToolIntervals: Array<[number, number]> = [] + // `taskOpen` is true only when THIS pass owns the whole current task window: + // either it saw the task_started, or it resumed from a state snapshotted at + // one. A task_complete without it attributes nothing, because a partial window + // would spread the task's active time over only part of its tokens. + let taskOpen = prevState?.taskOpen === true + let taskStartedAt: number | undefined = prevState?.taskStartedAt + const openToolStarts = new Map() + let checkpoint: CodexResumeCheckpoint | undefined + for (const [index, rawLine] of records.entries()) { const entry = parseCodexLine(rawLine as string | Buffer) if (!entry) continue + // Forked sessions replay the parent's event history clustered at the fork + // creation time. The token_count path has always skipped those replays; the + // timing branches must too, or a replayed task_started would reset a live + // window. Deliberately scoped to the timing branches: every other record + // type keeps the attribution this decoder already produces. + const isForkReplay = Boolean(s.forkCutoff && entry.timestamp && entry.timestamp < s.forkCutoff) + + if (entry.type === 'event_msg' && entry.payload?.type === 'task_started') { + if (isForkReplay) continue + // Emit the previous task. If it never reached task_complete its timing + // fields simply stay unset, matching the un-buffered behaviour. + calls.push(...pendingTaskCalls) + pendingTaskCalls = [] + taskGeneratedTokens = 0 + taskToolIntervals = [] + const startedAt = entry.timestamp ? Date.parse(entry.timestamp) : NaN + taskStartedAt = Number.isFinite(startedAt) ? startedAt : undefined + taskOpen = true + openToolStarts.clear() + // Everything decoded so far is in `calls` and every per-task accumulator + // is empty: the clean restart point for an appended tail. + const snapshot = cloneState(s) + snapshot.taskOpen = true + snapshot.taskStartedAt = taskStartedAt + checkpoint = { recordIndex: index, callCount: calls.length, state: snapshot } + continue + } + if (entry.type === 'session_meta') { // Update in place — do NOT reset the running counters. A single rollout // file can carry more than one session_meta (Codex re-emits it on resume / @@ -376,10 +572,26 @@ export function decodeCodex({ records, state: prevState, seenKeys: liveSeen, ses s.pendingToolSequence.push([{ tool: mcpTool }]) } } + const callId = entry.payload.call_id + const started = entry.timestamp ? Date.parse(entry.timestamp) : NaN + if (!isForkReplay && callId && Number.isFinite(started)) openToolStarts.set(callId, started) s.pendingToolSequence.push([call]) continue } + // Closes a tool-wait interval opened by the matching function_call. Tool + // names and files were already collected there, so this branch is timing + // only. + if (entry.type === 'response_item' && entry.payload?.type === 'function_call_output') { + if (isForkReplay) continue + const callId = entry.payload.call_id + const ended = entry.timestamp ? Date.parse(entry.timestamp) : NaN + const started = callId ? openToolStarts.get(callId) : undefined + if (started !== undefined && Number.isFinite(ended) && ended > started) taskToolIntervals.push([started, ended]) + if (callId) openToolStarts.delete(callId) + continue + } + if (entry.type === 'event_msg' && entry.payload?.type === 'patch_apply_end') { s.pendingTools.push('Edit') const p = entry.payload as Record @@ -402,6 +614,13 @@ export function decodeCodex({ records, state: prevState, seenKeys: liveSeen, ses } if (entry.type === 'event_msg' && entry.payload?.type === 'mcp_tool_call_end') { + // An MCP call records no start event, only its own duration on the end + // event, so the interval is reconstructed backwards from the end. + const endedAt = entry.timestamp ? Date.parse(entry.timestamp) : NaN + const mcpDurationMs = entry.payload.duration_ms ?? durationValueMs(entry.payload.duration) + if (!isForkReplay && typeof mcpDurationMs === 'number' && mcpDurationMs > 0 && Number.isFinite(endedAt)) { + taskToolIntervals.push([endedAt - mcpDurationMs, endedAt]) + } const inv = (entry.payload as Record)['invocation'] as Record | undefined const server = typeof inv?.['server'] === 'string' ? inv['server'] as string : '' const tool = typeof inv?.['tool'] === 'string' ? inv['tool'] as string : '' @@ -413,6 +632,37 @@ export function decodeCodex({ records, state: prevState, seenKeys: liveSeen, ses continue } + if (entry.type === 'event_msg' && entry.payload?.type === 'task_complete') { + if (isForkReplay) continue + const durationMs = entry.payload.duration_ms + if (taskOpen && typeof durationMs === 'number' && durationMs > 0 && taskGeneratedTokens > 0 && pendingTaskCalls.length > 0) { + const completedAt = entry.timestamp ? Date.parse(entry.timestamp) : NaN + const windowStart = taskStartedAt ?? (Number.isFinite(completedAt) ? completedAt - durationMs : undefined) + const windowEnd = windowStart !== undefined ? windowStart + durationMs : undefined + const clipped = taskToolIntervals.map(([start, end]) => [ + windowStart !== undefined ? Math.max(start, windowStart) : start, + windowEnd !== undefined ? Math.min(end, windowEnd) : end, + ] as [number, number]).filter(([start, end]) => end > start) + const merged = clipped.sort((a, b) => a[0] - b[0]).reduce>((acc, interval) => { + const previous = acc.at(-1) + if (previous && interval[0] <= previous[1]) previous[1] = Math.max(previous[1], interval[1]) + else acc.push([...interval]) + return acc + }, []) + const toolWaitMs = Math.min(durationMs, merged.reduce((sum, interval) => sum + interval[1] - interval[0], 0)) + const activeMs = durationMs - toolWaitMs + if (activeMs <= 0) continue + for (const call of pendingTaskCalls) { + const generated = call.outputTokens + call.reasoningTokens + if (generated <= 0) continue + call.activeGeneratedTokens = generated + call.activeDurationMs = activeMs * (generated / taskGeneratedTokens) + call.toolWaitMs = toolWaitMs * (generated / taskGeneratedTokens) + } + } + continue + } + if (entry.type === 'response_item' && entry.payload?.type === 'message' && entry.payload?.role === 'user') { const texts = normalizeContentBlocks(entry.payload.content) .filter(c => c.type === 'input_text') @@ -461,7 +711,7 @@ export function decodeCodex({ records, state: prevState, seenKeys: liveSeen, ses if (seen.has(dedupKey)) { clearPending(s); continue } seen.add(dedupKey) - calls.push({ + pendingTaskCalls.push({ provider: 'codex', model, inputTokens: estInput, @@ -486,6 +736,7 @@ export function decodeCodex({ records, state: prevState, seenKeys: liveSeen, ses ...(s.pendingEditFailed ? { editFailed: s.pendingEditFailed } : {}), }) + taskGeneratedTokens += estOutput clearPending(s) continue } @@ -546,7 +797,7 @@ export function decodeCodex({ records, state: prevState, seenKeys: liveSeen, ses if (seen.has(dedupKey)) continue seen.add(dedupKey) - calls.push({ + pendingTaskCalls.push({ provider: 'codex', model, inputTokens: uncachedInputTokens, @@ -570,10 +821,23 @@ export function decodeCodex({ records, state: prevState, seenKeys: liveSeen, ses ...(s.pendingEditFailed ? { editFailed: s.pendingEditFailed } : {}), }) + taskGeneratedTokens += outputTokens + reasoningTokens clearPending(s) } } + // Flush the final task, which has no following task_started to trigger it. A + // task still open here keeps its timing fields unset; the host resumes from + // `checkpoint` (the task's own task_started) so the next pass rebuilds the + // whole window and emits these same calls WITH timing, rather than patching + // calls it has already handed out. + calls.push(...pendingTaskCalls) + + // The end-of-input state is only ever used as a resume point when this pass + // crossed no task boundary at all, so it must never claim an open window. + s.taskOpen = false + s.taskStartedAt = undefined + s.seenKeys = liveSeen ? [] : [...seen] - return { calls, diagnostics, state: s } + return { calls, diagnostics, state: s, ...(checkpoint ? { checkpoint } : {}) } } diff --git a/packages/core/src/providers/codex/index.ts b/packages/core/src/providers/codex/index.ts index a5eae1d86..6232d4b24 100644 --- a/packages/core/src/providers/codex/index.ts +++ b/packages/core/src/providers/codex/index.ts @@ -16,6 +16,7 @@ export { codexToolNameMap, type CodexDecodeInput, type CodexDecodeResult, + type CodexResumeCheckpoint, } from './decode.js' export { diff --git a/packages/core/src/providers/codex/types.ts b/packages/core/src/providers/codex/types.ts index b361cdaef..b242e93a7 100644 --- a/packages/core/src/providers/codex/types.ts +++ b/packages/core/src/providers/codex/types.ts @@ -28,6 +28,11 @@ export type CodexEntry = { forked_from_id?: string model?: string name?: string + turn_id?: string + call_id?: string + started_at?: number + duration_ms?: number + duration?: { secs?: number; nanos?: number } | string content?: Array<{ type?: string; text?: string }> info?: { model?: string @@ -75,6 +80,15 @@ export type CodexDecodedCall = { locRemoved?: number editFailed?: number costIsEstimated?: boolean + // Tool-excluded active timing, attributed from the enclosing task's + // task_started/task_complete window (see decode.ts). `activeDurationMs` is the + // task duration minus recorded tool-wait intervals, split across the task's + // calls proportionally to their generated tokens; `toolWaitMs` is the excluded + // wait share. Present only when the task recorded both timing and generated + // tokens. + activeDurationMs?: number + activeGeneratedTokens?: number + toolWaitMs?: number } /** @@ -131,4 +145,13 @@ export type CodexDecodeState = { turnCounter: number currentTurnId: string seenKeys: string[] + // Tool-excluded active timing. These two are written ONLY into a state + // snapshotted at a `task_started` boundary (see CodexDecodeResult.checkpoint), + // where every per-task accumulator is provably empty: a pass resuming from + // such a state starts INSIDE that task having decoded none of its records, so + // it rebuilds the whole window itself. `taskOpen` is what tells it so; a state + // captured anywhere else leaves it false and a task_complete whose window this + // pass never saw attributes nothing rather than attributing a partial window. + taskOpen?: boolean + taskStartedAt?: number } diff --git a/packages/core/tests/providers/codex-decode.test.ts b/packages/core/tests/providers/codex-decode.test.ts index 392a85eba..515e12152 100644 --- a/packages/core/tests/providers/codex-decode.test.ts +++ b/packages/core/tests/providers/codex-decode.test.ts @@ -353,3 +353,68 @@ describe('codex decoder — untrusted fields from structurally-valid rollouts', expect(calls[0]!.deduplicationKey).not.toContain('object') }) }) + +// Tool-excluded active timing resumes ONLY from a task_started boundary: that is +// the one point where every per-task accumulator is empty, so a later pass can +// rebuild the whole window instead of patching calls it already emitted. +const TASK_CORPUS: string[] = [ + sessionMeta({ session_id: 'sess-task', timestamp: '2026-04-14T10:00:00Z' }), + // Task 1: 10s wall, 3s of it tool wait. + JSON.stringify({ type: 'event_msg', timestamp: '2026-04-14T10:00:00Z', payload: { type: 'task_started' } }), + userMessage('run the tool', '2026-04-14T10:00:01Z'), + JSON.stringify({ type: 'response_item', timestamp: '2026-04-14T10:00:02Z', payload: { type: 'function_call', call_id: 'c1', name: 'exec_command' } }), + JSON.stringify({ type: 'response_item', timestamp: '2026-04-14T10:00:05Z', payload: { type: 'function_call_output', call_id: 'c1', output: 'ok' } }), + tokenCount({ timestamp: '2026-04-14T10:00:08Z', last: { input: 300, output: 100 }, total: { input: 300, output: 100, total: 400 } }), + JSON.stringify({ type: 'event_msg', timestamp: '2026-04-14T10:00:10Z', payload: { type: 'task_complete', duration_ms: 10_000 } }), + // Task 2: 5s wall, no tool wait. + JSON.stringify({ type: 'event_msg', timestamp: '2026-04-14T10:00:11Z', payload: { type: 'task_started' } }), + userMessage('now the tests', '2026-04-14T10:00:12Z'), + tokenCount({ timestamp: '2026-04-14T10:00:14Z', last: { input: 200, output: 200 }, total: { input: 500, output: 300, total: 900 } }), + JSON.stringify({ type: 'event_msg', timestamp: '2026-04-14T10:00:16Z', payload: { type: 'task_complete', duration_ms: 5_000 } }), +] +const TASK2_STARTED_INDEX = 7 +const MID_TASK2_INDEX = 9 // task 2's token_count; its task_complete is the next record + +describe('codex decoder — task-timing checkpoint', () => { + it('attributes each task its own tool-excluded active window', () => { + const cold = decodeCold(TASK_CORPUS) + expect(cold).toHaveLength(2) + expect(cold[0]).toMatchObject({ activeDurationMs: 7000, activeGeneratedTokens: 100, toolWaitMs: 3000 }) + expect(cold[1]).toMatchObject({ activeDurationMs: 5000, activeGeneratedTokens: 200, toolWaitMs: 0 }) + }) + + it('reports the last task_started as the resume checkpoint, with the calls before it', () => { + const { checkpoint } = decodeCodex({ records: TASK_CORPUS, context }) + expect(checkpoint?.recordIndex).toBe(TASK2_STARTED_INDEX) + // Task 1's call is complete and replayable; task 2's is not, because a + // later pass re-derives it from this very boundary. + expect(checkpoint?.callCount).toBe(1) + expect(checkpoint?.state.taskOpen).toBe(true) + }) + + it('resuming at the checkpoint deep-equals a cold decode, timing included', () => { + const first = decodeCodex({ records: TASK_CORPUS.slice(0, MID_TASK2_INDEX + 1), context }) + const checkpoint = first.checkpoint! + // What the host persists: the calls before the boundary, and the boundary + // state as plain JSON. + const replayed = first.calls.slice(0, checkpoint.callCount) + const serialized: CodexDecodeState = JSON.parse(JSON.stringify(checkpoint.state)) + const second = decodeCodex({ records: TASK_CORPUS.slice(checkpoint.recordIndex + 1), context, state: serialized }) + expect([...replayed, ...second.calls]).toEqual(decodeCold(TASK_CORPUS)) + }) + + it('a mid-task split that resumes from end-of-file attributes no timing rather than a partial window', () => { + // The end-of-input state deliberately declares no open task, so a + // task_complete whose window this pass never saw stamps nothing. Wrong + // timing (a whole task's active time spread over part of its tokens) would + // be worse than none. + const first = decodeCodex({ records: TASK_CORPUS.slice(0, MID_TASK2_INDEX + 1), context }) + const serialized: CodexDecodeState = JSON.parse(JSON.stringify(first.state)) + expect(serialized.taskOpen).toBe(false) + const second = decodeCodex({ records: TASK_CORPUS.slice(MID_TASK2_INDEX + 1), context, state: serialized }) + const calls = [...first.calls, ...second.calls] + expect(calls).toHaveLength(2) + expect(calls[1]!.activeDurationMs).toBeUndefined() + expect(calls[1]!.toolWaitMs).toBeUndefined() + }) +})