From 16d81c8d211e7e333ab91bf1f374eb52f85d9cd5 Mon Sep 17 00:00:00 2001 From: ktwu1997 <136313490+ktwu1997@users.noreply.github.com> Date: Mon, 16 Nov 2026 14:30:22 +0800 Subject: [PATCH 1/4] fix(insights): stop recounting a Codex fork's copied parent history A forked Codex thread (a subagent) starts its rollout file with a copy of the parent's records. The first copied token_count had no previous total, so the parent's whole cumulative usage was counted again in every fork, and copied records before any turn_context landed under model "unknown". In a forked session, records before the child's first own turn now only set the cumulative baseline. Codex turn ids are UUIDv7, so a turn older than the forked thread id is copied history; copied turn_context records can precede the copied counters, so the first turn_context alone is not a safe boundary. Non-fork sessions are unchanged. The Codex parser version is bumped so stored scans are recomputed. Co-Authored-By: Claude Opus 5.5 --- .../mods/insights/core/adapters.mjs | 35 ++++++++++++++++++- herdr-insights/mods/insights/core/scan.mjs | 2 +- .../collector-adapters.test.mjs | 26 ++++++++++++++ .../fixtures/codex-fork-replay-bare.jsonl | 7 ++++ .../fixtures/codex-fork-replay-context.jsonl | 10 ++++++ .../fixtures/codex-nonfork-prefix.jsonl | 7 ++++ 6 files changed, 85 insertions(+), 2 deletions(-) create mode 100644 tests/herdr-insights/fixtures/codex-fork-replay-bare.jsonl create mode 100644 tests/herdr-insights/fixtures/codex-fork-replay-context.jsonl create mode 100644 tests/herdr-insights/fixtures/codex-nonfork-prefix.jsonl diff --git a/herdr-insights/mods/insights/core/adapters.mjs b/herdr-insights/mods/insights/core/adapters.mjs index 8645405..044fa64 100644 --- a/herdr-insights/mods/insights/core/adapters.mjs +++ b/herdr-insights/mods/insights/core/adapters.mjs @@ -458,6 +458,24 @@ function codexRecordSession(state, ctx, payload) { return codexSession(state, ctx); } +function codexUuidV7Ms(id) { + const match = /^([0-9a-f]{8})-([0-9a-f]{4})-7/i.exec(id); + return match ? Number.parseInt(match[1] + match[2], 16) : null; +} + +// A forked thread's file starts with a copy of its parent's history. Its own +// work begins with the first turn created after the fork; Codex turn ids are +// UUIDv7, so a turn older than the forked thread id is copied history. +function codexEndsForkReplay(state, record, payload) { + if (record.type !== "turn_context" && !(record.type === "event_msg" && payload.type === "task_started")) { + return false; + } + const turnId = asString(payload.turn_id ?? payload.root_turn_id, ""); + if (!turnId) return false; + const turnMs = codexUuidV7Ms(turnId); + return turnMs === null || state.forkStartMs === null || turnMs >= state.forkStartMs; +} + function codexContext(state, turnId) { if (!isPlainObject(state.turns)) state.turns = {}; if (turnId && isPlainObject(state.turns[turnId])) return state.turns[turnId]; @@ -647,6 +665,7 @@ function parseCodex(record, ctx) { const payload = isPlainObject(record.payload) ? record.payload : {}; const timestamp = parseTimestamp(record.timestamp) ?? state.timestamp ?? null; + if (state.forkReplay && codexEndsForkReplay(state, record, payload)) state.forkReplay = false; if (record.type === "session_meta") { const threadId = asString( @@ -654,7 +673,13 @@ function parseCodex(record, ctx) { "", ); const firstHeader = !state.session; - if (threadId && firstHeader) state.session = threadId; + if (threadId && firstHeader) { + state.session = threadId; + if (asString(payload.forked_from_id ?? payload.parent_thread_id, "")) { + state.forkReplay = true; + state.forkStartMs = codexUuidV7Ms(threadId); + } + } if (firstHeader || !state.project) { state.project = asString(payload.cwd, state.project ?? "unknown"); } @@ -691,6 +716,8 @@ function parseCodex(record, ctx) { } if (record.type === "token_usage_record") { + // Copied parent usage is already counted from the parent's own file. + if (state.forkReplay) return; state.modern = true; if (timestamp === null) { ctx.warn("Codex token usage record has no usable timestamp; skipped"); @@ -803,6 +830,11 @@ function parseCodex(record, ctx) { output_tokens: toNonNegInt(totals.output_tokens), reasoning_output_tokens: toNonNegInt(totals.reasoning_output_tokens), }; + if (state.forkReplay) { + // The child's counters continue from the parent's last total. + state.legacyTotals = current; + return; + } const previous = isPlainObject(state.legacyTotals) ? state.legacyTotals : null; @@ -824,6 +856,7 @@ function parseCodex(record, ctx) { current.reasoning_output_tokens, ]; } else if (isPlainObject(info.last_token_usage)) { + if (state.forkReplay) return; raw = info.last_token_usage; identityTail = [ "legacy-last", diff --git a/herdr-insights/mods/insights/core/scan.mjs b/herdr-insights/mods/insights/core/scan.mjs index d489d95..da10fdf 100644 --- a/herdr-insights/mods/insights/core/scan.mjs +++ b/herdr-insights/mods/insights/core/scan.mjs @@ -32,7 +32,7 @@ import { const CHUNK_BYTES = 256 * 1024; const MAX_WARNINGS = 50; const MAX_TRAVERSAL_ENTRIES = 250_000; -const PARSER_VERSIONS = { pi: 3, codex: 5, claude: 2, antigravity: 2 }; +const PARSER_VERSIONS = { pi: 3, codex: 6, claude: 2, antigravity: 2 }; const PROGRESS_CHUNK_INTERVAL = 64; export class ScanStopError extends Error { diff --git a/tests/herdr-insights/collector-adapters.test.mjs b/tests/herdr-insights/collector-adapters.test.mjs index f6260c2..e4c73b0 100644 --- a/tests/herdr-insights/collector-adapters.test.mjs +++ b/tests/herdr-insights/collector-adapters.test.mjs @@ -663,3 +663,29 @@ test("Snapshot rows never include transcript bodies", async () => { assert.equal(storeText.includes("synthetic prompt"), false); assert.equal(storeText.includes("synthetic failure"), false); }); + +async function codexRows(fixture) { + const context = await env(); + await placeFixture(context, `.codex/sessions/2026/03/08/${fixture}`, fixture); + const snapshot = await configure(context, [ + { id: "codex", label: "Codex", adapter: "codex", enabled: true, roots: ["~/.codex/sessions"] }, + ]); + assert.equal(snapshot.stats.errors, 0); + return snapshot.rows; +} + +for (const fixture of ["codex-fork-replay-context.jsonl", "codex-fork-replay-bare.jsonl"]) { + test(`Codex fork counts only the child's own usage, not the copied parent history (${fixture})`, async () => { + const rows = await codexRows(fixture); + assert.deepEqual(rows.map((row) => row.model), ["child-model"], "no unknown or parent-model rows"); + assert.equal(rows[0].session, "codex:019f0000-1000-7000-8000-000000000001"); + assert.equal(rows[0].tokens.total, 250, "2750 child total minus the 2500 parent baseline"); + }); +} + +test("Codex non-fork session still counts its first cumulative total in full", async () => { + const rows = await codexRows("codex-nonfork-prefix.jsonl"); + const total = rows.reduce((sum, row) => sum + row.tokens.total, 0); + assert.equal(total, 2750); + assert.ok(rows.some((row) => row.model === "unknown"), "records before any turn_context keep today's behavior"); +}); diff --git a/tests/herdr-insights/fixtures/codex-fork-replay-bare.jsonl b/tests/herdr-insights/fixtures/codex-fork-replay-bare.jsonl new file mode 100644 index 0000000..d85aa50 --- /dev/null +++ b/tests/herdr-insights/fixtures/codex-fork-replay-bare.jsonl @@ -0,0 +1,7 @@ +{"type":"session_meta","timestamp":"2026-03-08T08:00:00.000Z","payload":{"id":"019f0000-1000-7000-8000-000000000001","cwd":"/repo/fork-replay","model_provider":"openai","timestamp":"2026-03-08T08:00:00.000Z","forked_from_id":"019f0000-0000-7000-8000-000000000002","parent_thread_id":"019f0000-0000-7000-8000-000000000002","thread_source":"subagent","source":{"subagent":{"thread_spawn":{"parent_thread_id":"019f0000-0000-7000-8000-000000000002","depth":1}}}}} +{"type":"event_msg","timestamp":"2026-03-08T08:00:00.001Z","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":800,"cached_input_tokens":0,"output_tokens":200,"reasoning_output_tokens":0,"total_tokens":1000}}}} +{"type":"event_msg","timestamp":"2026-03-08T08:00:00.001Z","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":2000,"cached_input_tokens":0,"output_tokens":500,"reasoning_output_tokens":0,"total_tokens":2500}}}} +{"type":"event_msg","timestamp":"2026-03-08T08:00:01.000Z","payload":{"type":"task_started","turn_id":"019f0000-2000-7000-8000-000000000004"}} +{"type":"turn_context","timestamp":"2026-03-08T08:00:02.000Z","payload":{"turn_id":"019f0000-2000-7000-8000-000000000004","model":"child-model","cwd":"/repo/fork-replay"}} +{"type":"event_msg","timestamp":"2026-03-08T08:00:08.000Z","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":2080,"cached_input_tokens":0,"output_tokens":520,"reasoning_output_tokens":0,"total_tokens":2600}}}} +{"type":"event_msg","timestamp":"2026-03-08T08:00:12.000Z","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":2200,"cached_input_tokens":0,"output_tokens":550,"reasoning_output_tokens":0,"total_tokens":2750}}}} diff --git a/tests/herdr-insights/fixtures/codex-fork-replay-context.jsonl b/tests/herdr-insights/fixtures/codex-fork-replay-context.jsonl new file mode 100644 index 0000000..5b58b8e --- /dev/null +++ b/tests/herdr-insights/fixtures/codex-fork-replay-context.jsonl @@ -0,0 +1,10 @@ +{"type":"session_meta","timestamp":"2026-03-08T08:00:00.000Z","payload":{"id":"019f0000-1000-7000-8000-000000000001","cwd":"/repo/fork-replay","model_provider":"openai","timestamp":"2026-03-08T08:00:00.000Z","forked_from_id":"019f0000-0000-7000-8000-000000000002","parent_thread_id":"019f0000-0000-7000-8000-000000000002","thread_source":"subagent","source":{"subagent":{"thread_spawn":{"parent_thread_id":"019f0000-0000-7000-8000-000000000002","depth":1}}}}} +{"type":"session_meta","timestamp":"2026-03-08T08:00:00.000Z","payload":{"id":"019f0000-0000-7000-8000-000000000002","cwd":"/repo/fork-replay","model_provider":"openai","timestamp":"2026-03-08T08:00:00.000Z"}} +{"type":"event_msg","timestamp":"2026-03-08T08:00:00.001Z","payload":{"type":"task_started","turn_id":"019f0000-0001-7000-8000-000000000003"}} +{"type":"turn_context","timestamp":"2026-03-08T08:00:00.001Z","payload":{"turn_id":"019f0000-0001-7000-8000-000000000003","model":"parent-model","cwd":"/repo/fork-replay"}} +{"type":"event_msg","timestamp":"2026-03-08T08:00:00.001Z","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":800,"cached_input_tokens":0,"output_tokens":200,"reasoning_output_tokens":0,"total_tokens":1000}}}} +{"type":"event_msg","timestamp":"2026-03-08T08:00:00.001Z","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":2000,"cached_input_tokens":0,"output_tokens":500,"reasoning_output_tokens":0,"total_tokens":2500}}}} +{"type":"event_msg","timestamp":"2026-03-08T08:00:01.000Z","payload":{"type":"task_started","turn_id":"019f0000-2000-7000-8000-000000000004"}} +{"type":"turn_context","timestamp":"2026-03-08T08:00:02.000Z","payload":{"turn_id":"019f0000-2000-7000-8000-000000000004","model":"child-model","cwd":"/repo/fork-replay"}} +{"type":"event_msg","timestamp":"2026-03-08T08:00:08.000Z","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":2080,"cached_input_tokens":0,"output_tokens":520,"reasoning_output_tokens":0,"total_tokens":2600}}}} +{"type":"event_msg","timestamp":"2026-03-08T08:00:12.000Z","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":2200,"cached_input_tokens":0,"output_tokens":550,"reasoning_output_tokens":0,"total_tokens":2750}}}} diff --git a/tests/herdr-insights/fixtures/codex-nonfork-prefix.jsonl b/tests/herdr-insights/fixtures/codex-nonfork-prefix.jsonl new file mode 100644 index 0000000..88b6581 --- /dev/null +++ b/tests/herdr-insights/fixtures/codex-nonfork-prefix.jsonl @@ -0,0 +1,7 @@ +{"type":"session_meta","timestamp":"2026-03-08T08:00:00.000Z","payload":{"id":"019f0000-1000-7000-8000-000000000001","cwd":"/repo/fork-replay","model_provider":"openai","timestamp":"2026-03-08T08:00:00.000Z"}} +{"type":"event_msg","timestamp":"2026-03-08T08:00:00.001Z","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":800,"cached_input_tokens":0,"output_tokens":200,"reasoning_output_tokens":0,"total_tokens":1000}}}} +{"type":"event_msg","timestamp":"2026-03-08T08:00:00.001Z","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":2000,"cached_input_tokens":0,"output_tokens":500,"reasoning_output_tokens":0,"total_tokens":2500}}}} +{"type":"event_msg","timestamp":"2026-03-08T08:00:01.000Z","payload":{"type":"task_started","turn_id":"019f0000-2000-7000-8000-000000000004"}} +{"type":"turn_context","timestamp":"2026-03-08T08:00:02.000Z","payload":{"turn_id":"019f0000-2000-7000-8000-000000000004","model":"child-model","cwd":"/repo/fork-replay"}} +{"type":"event_msg","timestamp":"2026-03-08T08:00:08.000Z","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":2080,"cached_input_tokens":0,"output_tokens":520,"reasoning_output_tokens":0,"total_tokens":2600}}}} +{"type":"event_msg","timestamp":"2026-03-08T08:00:12.000Z","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":2200,"cached_input_tokens":0,"output_tokens":550,"reasoning_output_tokens":0,"total_tokens":2750}}}} From fda4836fc1e488051a68aabed2f675975fea0783 Mon Sep 17 00:00:00 2001 From: ktwu1997 <136313490+ktwu1997@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:11:26 +0800 Subject: [PATCH 2/4] fix(insights): end a Codex fork's copied history only at a UUIDv7 turn The fork boundary treated any turn id that was not UUIDv7 as the child's own turn. Copied parent history can contain goal-continuation turns with UUIDv4 ids, so the copy ended early and most of the parent's usage was still counted in every fork. Only a UUIDv7 turn created after the forked thread now ends the copy; a fork whose own thread id is not UUIDv7 keeps the previous behavior. The Codex parser version is bumped again so scans made with the earlier boundary are recomputed. Co-Authored-By: Claude Opus 5.5 --- herdr-insights/mods/insights/core/adapters.mjs | 14 +++++++++----- herdr-insights/mods/insights/core/scan.mjs | 2 +- tests/herdr-insights/collector-adapters.test.mjs | 2 +- .../fixtures/codex-fork-replay-v4-turn.jsonl | 9 +++++++++ 4 files changed, 20 insertions(+), 7 deletions(-) create mode 100644 tests/herdr-insights/fixtures/codex-fork-replay-v4-turn.jsonl diff --git a/herdr-insights/mods/insights/core/adapters.mjs b/herdr-insights/mods/insights/core/adapters.mjs index 044fa64..bc9130c 100644 --- a/herdr-insights/mods/insights/core/adapters.mjs +++ b/herdr-insights/mods/insights/core/adapters.mjs @@ -464,8 +464,9 @@ function codexUuidV7Ms(id) { } // A forked thread's file starts with a copy of its parent's history. Its own -// work begins with the first turn created after the fork; Codex turn ids are -// UUIDv7, so a turn older than the forked thread id is copied history. +// work begins with the first UUIDv7 turn created after the fork. Copied +// history can hold turns with other id versions (goal continuations use +// UUIDv4), and their bits are not a time, so they never end the copy. function codexEndsForkReplay(state, record, payload) { if (record.type !== "turn_context" && !(record.type === "event_msg" && payload.type === "task_started")) { return false; @@ -473,7 +474,7 @@ function codexEndsForkReplay(state, record, payload) { const turnId = asString(payload.turn_id ?? payload.root_turn_id, ""); if (!turnId) return false; const turnMs = codexUuidV7Ms(turnId); - return turnMs === null || state.forkStartMs === null || turnMs >= state.forkStartMs; + return turnMs !== null && turnMs >= state.forkStartMs; } function codexContext(state, turnId) { @@ -675,9 +676,12 @@ function parseCodex(record, ctx) { const firstHeader = !state.session; if (threadId && firstHeader) { state.session = threadId; - if (asString(payload.forked_from_id ?? payload.parent_thread_id, "")) { + const forkStartMs = codexUuidV7Ms(threadId); + // Without a time-ordered thread id there is no safe boundary; keep + // the previous behavior rather than risk dropping the child's usage. + if (forkStartMs !== null && asString(payload.forked_from_id ?? payload.parent_thread_id, "")) { state.forkReplay = true; - state.forkStartMs = codexUuidV7Ms(threadId); + state.forkStartMs = forkStartMs; } } if (firstHeader || !state.project) { diff --git a/herdr-insights/mods/insights/core/scan.mjs b/herdr-insights/mods/insights/core/scan.mjs index da10fdf..0f63083 100644 --- a/herdr-insights/mods/insights/core/scan.mjs +++ b/herdr-insights/mods/insights/core/scan.mjs @@ -32,7 +32,7 @@ import { const CHUNK_BYTES = 256 * 1024; const MAX_WARNINGS = 50; const MAX_TRAVERSAL_ENTRIES = 250_000; -const PARSER_VERSIONS = { pi: 3, codex: 6, claude: 2, antigravity: 2 }; +const PARSER_VERSIONS = { pi: 3, codex: 7, claude: 2, antigravity: 2 }; const PROGRESS_CHUNK_INTERVAL = 64; export class ScanStopError extends Error { diff --git a/tests/herdr-insights/collector-adapters.test.mjs b/tests/herdr-insights/collector-adapters.test.mjs index e4c73b0..9afab7d 100644 --- a/tests/herdr-insights/collector-adapters.test.mjs +++ b/tests/herdr-insights/collector-adapters.test.mjs @@ -674,7 +674,7 @@ async function codexRows(fixture) { return snapshot.rows; } -for (const fixture of ["codex-fork-replay-context.jsonl", "codex-fork-replay-bare.jsonl"]) { +for (const fixture of ["codex-fork-replay-context.jsonl", "codex-fork-replay-bare.jsonl", "codex-fork-replay-v4-turn.jsonl"]) { test(`Codex fork counts only the child's own usage, not the copied parent history (${fixture})`, async () => { const rows = await codexRows(fixture); assert.deepEqual(rows.map((row) => row.model), ["child-model"], "no unknown or parent-model rows"); diff --git a/tests/herdr-insights/fixtures/codex-fork-replay-v4-turn.jsonl b/tests/herdr-insights/fixtures/codex-fork-replay-v4-turn.jsonl new file mode 100644 index 0000000..13705dc --- /dev/null +++ b/tests/herdr-insights/fixtures/codex-fork-replay-v4-turn.jsonl @@ -0,0 +1,9 @@ +{"type":"session_meta","timestamp":"2026-03-08T08:00:00.000Z","payload":{"id":"019f0000-1000-7000-8000-000000000001","cwd":"/repo/fork-replay","model_provider":"openai","timestamp":"2026-03-08T08:00:00.000Z","forked_from_id":"019f0000-0000-7000-8000-000000000002","parent_thread_id":"019f0000-0000-7000-8000-000000000002","thread_source":"subagent","source":{"subagent":{"thread_spawn":{"parent_thread_id":"019f0000-0000-7000-8000-000000000002","depth":1}}}}} +{"type":"event_msg","timestamp":"2026-03-08T08:00:00.001Z","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":800,"cached_input_tokens":0,"output_tokens":200,"reasoning_output_tokens":0,"total_tokens":1000}}}} +{"type":"event_msg","timestamp":"2026-03-08T08:00:00.001Z","payload":{"type":"task_started","turn_id":"523e7e07-782e-41b1-886f-4e90def0241b"}} +{"type":"turn_context","timestamp":"2026-03-08T08:00:00.001Z","payload":{"turn_id":"523e7e07-782e-41b1-886f-4e90def0241b","model":"parent-model","cwd":"/repo/fork-replay"}} +{"type":"event_msg","timestamp":"2026-03-08T08:00:00.001Z","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":2000,"cached_input_tokens":0,"output_tokens":500,"reasoning_output_tokens":0,"total_tokens":2500}}}} +{"type":"event_msg","timestamp":"2026-03-08T08:00:01.000Z","payload":{"type":"task_started","turn_id":"019f0000-2000-7000-8000-000000000004"}} +{"type":"turn_context","timestamp":"2026-03-08T08:00:02.000Z","payload":{"turn_id":"019f0000-2000-7000-8000-000000000004","model":"child-model","cwd":"/repo/fork-replay"}} +{"type":"event_msg","timestamp":"2026-03-08T08:00:08.000Z","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":2080,"cached_input_tokens":0,"output_tokens":520,"reasoning_output_tokens":0,"total_tokens":2600}}}} +{"type":"event_msg","timestamp":"2026-03-08T08:00:12.000Z","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":2200,"cached_input_tokens":0,"output_tokens":550,"reasoning_output_tokens":0,"total_tokens":2750}}}} From 291814643ba3106619b43d29e8f8f48f1e134b7e Mon Sep 17 00:00:00 2001 From: GaganSD Date: Mon, 16 Nov 2026 16:42:52 +0000 Subject: [PATCH 3/4] fix(insights): reconcile fork usage and live worker state --- herdr-insights/README.md | 11 ++ herdr-insights/mods/insights/collector.mjs | 25 ++-- .../mods/insights/core/adapters.mjs | 53 ++++--- herdr-insights/mods/insights/core/scan.mjs | 13 +- .../collector-live-progress.test.mjs | 32 +++++ tests/herdr-insights/ship-readiness.test.mjs | 134 ++++++++++++++++++ 6 files changed, 238 insertions(+), 30 deletions(-) diff --git a/herdr-insights/README.md b/herdr-insights/README.md index 6353f10..20df61b 100644 --- a/herdr-insights/README.md +++ b/herdr-insights/README.md @@ -37,6 +37,17 @@ click-state rules. Set `HERDR_APP_HOME` to use a different app home. - Open **Settings** to choose sources, edit paths, toggle cached tokens, or change the palette and refresh interval. +### Codex fork accounting + +For fork headers with canonical UUIDv7 thread IDs, Insights excludes replayed +usage until a turn ID has a strictly newer UUIDv7 millisecond. Equal-millisecond, +missing, malformed, or non-v7 turn IDs are ambiguous: child usage can be excluded, +with a bounded scan warning that remains visible even after a later boundary. +Non-v7 initial thread IDs retain the previous accounting behavior. Existing +available Codex files are automatically rescanned once for this parser update. +Modern thread cumulative counters and legacy totals share the raw baseline; +turn-only counters are not treated as thread totals. + ## Privacy No network requests or uploads. The mod runs a local Node helper, reads session diff --git a/herdr-insights/mods/insights/collector.mjs b/herdr-insights/mods/insights/collector.mjs index 767e251..a77d3d9 100644 --- a/herdr-insights/mods/insights/collector.mjs +++ b/herdr-insights/mods/insights/collector.mjs @@ -197,27 +197,28 @@ async function overlayLiveProgress(storePath, snapshot) { const scan = snapshot?.scan; if (!scan || !["running", "partial", "starting"].includes(scan.status)) return snapshot; if (!Number.isInteger(scan.generation)) return snapshot; - const live = await readLiveProgress(storePath); - if (!live) return snapshot; - if (live.store !== path.resolve(storePath) || live.generation !== scan.generation) return snapshot; - const [claim, owner] = await Promise.all([ + const [live, claim, owner] = await Promise.all([ + readLiveProgress(storePath), readWorkerClaim(storePath), readStoreLockInfo(storePath), ]); - if (!claim || claim.host !== os.hostname() || !workerClaimActive(claim)) return snapshot; - if (claim.pid !== live.pid || claim.generation !== live.generation || claim.token !== live.token) { - return snapshot; - } + if (!claim || claim.host !== os.hostname() || !Number.isInteger(claim.pid) + || claim.pid <= 0 || !workerClaimActive(claim)) return snapshot; + if (claim.generation !== scan.generation) return snapshot; if (!owner || owner.role !== "worker" || owner.host !== os.hostname()) return snapshot; - if (owner.pid !== live.pid || Number(owner.generation) !== live.generation) return snapshot; - if (Number.isInteger(scan.workerPid) && scan.workerPid !== live.pid) return snapshot; + if (owner.pid !== claim.pid || Number(owner.generation) !== scan.generation) return snapshot; + if (Number.isInteger(scan.workerPid) && scan.workerPid !== claim.pid) return snapshot; + if (live && (live.store !== path.resolve(storePath) || live.generation !== scan.generation + || claim.pid !== live.pid || claim.token !== live.token)) return snapshot; + // The lock can precede the first running commit/progress publication. The + // validated worker is running even then; optional counters are not authority. return { ...snapshot, scan: { ...scan, status: "running", - workerPid: live.pid, - live: liveView(live), + workerPid: claim.pid, + ...(live ? { live: liveView(live) } : {}), }, }; } diff --git a/herdr-insights/mods/insights/core/adapters.mjs b/herdr-insights/mods/insights/core/adapters.mjs index bc9130c..08ce09a 100644 --- a/herdr-insights/mods/insights/core/adapters.mjs +++ b/herdr-insights/mods/insights/core/adapters.mjs @@ -459,22 +459,23 @@ function codexRecordSession(state, ctx, payload) { } function codexUuidV7Ms(id) { - const match = /^([0-9a-f]{8})-([0-9a-f]{4})-7/i.exec(id); + const match = /^([0-9a-f]{8})-([0-9a-f]{4})-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.exec(id); return match ? Number.parseInt(match[1] + match[2], 16) : null; } -// A forked thread's file starts with a copy of its parent's history. Its own -// work begins with the first UUIDv7 turn created after the fork. Copied -// history can hold turns with other id versions (goal continuations use -// UUIDv4), and their bits are not a time, so they never end the copy. +// Supported fork boundary: a canonical UUIDv7 turn strictly newer than the +// initial thread's UUIDv7 millisecond. Equal milliseconds cannot distinguish +// copied from child turns; missing/other-version IDs cannot order them at all. +// Conservatively exclude that prefix and retain a scanner diagnostic rather +// than guessing from record timestamps (which can be rewritten during a copy). function codexEndsForkReplay(state, record, payload) { if (record.type !== "turn_context" && !(record.type === "event_msg" && payload.type === "task_started")) { return false; } const turnId = asString(payload.turn_id ?? payload.root_turn_id, ""); - if (!turnId) return false; const turnMs = codexUuidV7Ms(turnId); - return turnMs !== null && turnMs >= state.forkStartMs; + if (turnMs === null || turnMs === state.forkStartMs) state.forkBoundaryAmbiguous = true; + return turnMs !== null && turnMs > state.forkStartMs; } function codexContext(state, turnId) { @@ -501,6 +502,19 @@ function codexNormalized(raw) { }); } +// Both modern thread_token_usage and legacy total_token_usage use raw Codex +// counters: input includes cache, output includes reasoning. Never store the +// normalized (cache-subtracted) vector as the legacy cumulative baseline. +function codexRawTotals(raw) { + return { + input_tokens: toNonNegInt(raw.input_tokens), + cached_input_tokens: toNonNegInt(raw.cached_input_tokens), + cache_write_input_tokens: toNonNegInt(raw.cache_write_input_tokens), + output_tokens: toNonNegInt(raw.output_tokens), + reasoning_output_tokens: toNonNegInt(raw.reasoning_output_tokens), + }; +} + const CODEX_TOKEN_FIELDS = ["input", "output", "cacheRead", "cacheWrite", "reasoning"]; const CODEX_TWIN_WINDOW_MS = 5 * 60 * 1000; @@ -720,8 +734,6 @@ function parseCodex(record, ctx) { } if (record.type === "token_usage_record") { - // Copied parent usage is already counted from the parent's own file. - if (state.forkReplay) return; state.modern = true; if (timestamp === null) { ctx.warn("Codex token usage record has no usable timestamp; skipped"); @@ -752,6 +764,18 @@ function parseCodex(record, ctx) { tokens.reasoning, ]; let dedupKey = makeDedupKey("codex", ...identity); + if (state.forkReplay) { + const observed = codexObserveModern(state, dedupKey, tokens); + // A copied THREAD cumulative counter is the raw baseline for a legacy + // suffix; a turn counter is not. Without it retain local observations. + if (isPlainObject(payload.thread_token_usage)) { + state.legacyTotals = codexRawTotals(payload.thread_token_usage); + state.modernPending = codexZeroTokens(); + } else { + codexAddPending(state, observed); + } + return; + } let mergedTwin = false; const twinKey = codexPendingTwinKey( state.pendingTwin, @@ -827,16 +851,11 @@ function parseCodex(record, ctx) { let reset = false; if (totals) { - const current = { - input_tokens: toNonNegInt(totals.input_tokens), - cached_input_tokens: toNonNegInt(totals.cached_input_tokens), - cache_write_input_tokens: toNonNegInt(totals.cache_write_input_tokens), - output_tokens: toNonNegInt(totals.output_tokens), - reasoning_output_tokens: toNonNegInt(totals.reasoning_output_tokens), - }; + const current = codexRawTotals(totals); if (state.forkReplay) { - // The child's counters continue from the parent's last total. + // This baseline also covers any previously copied modern observations. state.legacyTotals = current; + state.modernPending = codexZeroTokens(); return; } const previous = isPlainObject(state.legacyTotals) diff --git a/herdr-insights/mods/insights/core/scan.mjs b/herdr-insights/mods/insights/core/scan.mjs index 0f63083..adead1e 100644 --- a/herdr-insights/mods/insights/core/scan.mjs +++ b/herdr-insights/mods/insights/core/scan.mjs @@ -32,7 +32,7 @@ import { const CHUNK_BYTES = 256 * 1024; const MAX_WARNINGS = 50; const MAX_TRAVERSAL_ENTRIES = 250_000; -const PARSER_VERSIONS = { pi: 3, codex: 7, claude: 2, antigravity: 2 }; +const PARSER_VERSIONS = { pi: 3, codex: 8, claude: 2, antigravity: 2 }; const PROGRESS_CHUNK_INTERVAL = 64; export class ScanStopError extends Error { @@ -949,6 +949,13 @@ async function scanOpenCodeDb(store, source, filePath, sink, stats, options = {} return { dirty: true, aborted: false }; } +function noteCodexForkReplay(entry, sink) { + if (entry?.adapter !== "codex") return; + if (entry.state.forkReplay || entry.state.forkBoundaryAmbiguous) { + sink.note(`codex ${entry.path}: fork replay usage excluded until a strictly newer UUIDv7 turn; missing, non-v7 or equal-millisecond boundaries may exclude child usage.`); + } +} + async function scanOneFile(store, source, filePath, sink, stats, options = {}) { const sourceSink = { warn(message) { @@ -997,6 +1004,7 @@ async function scanOneFile(store, source, filePath, sink, stats, options = {}) { && existing.mtimeMs === fileStat.mtimeMs ) { stats.unchangedFiles += 1; + noteCodexForkReplay(existing, sourceSink); return { dirty: false, aborted: false }; } @@ -1065,6 +1073,7 @@ async function scanOneFile(store, source, filePath, sink, stats, options = {}) { ) { stats.unchangedFiles += 1; commitStoreTransaction(tx); + noteCodexForkReplay(entry, sourceSink); return { dirty, aborted: false }; } @@ -1135,6 +1144,7 @@ async function scanOneFile(store, source, filePath, sink, stats, options = {}) { if (parserVersion) entry.state[parserKey] = parserVersion; delete entry.state.zstdError; delete entry.state.scanPending; + noteCodexForkReplay(entry, sourceSink); return { dirty, aborted: false }; } @@ -1162,6 +1172,7 @@ async function scanOneFile(store, source, filePath, sink, stats, options = {}) { entry.mtimeMs = fileStat.mtimeMs; delete entry.state.scanPending; } + noteCodexForkReplay(entry, sourceSink); return { dirty, aborted: false, pending: plain.pending }; } catch (error) { rollbackStoreTransaction(tx); diff --git a/tests/herdr-insights/collector-live-progress.test.mjs b/tests/herdr-insights/collector-live-progress.test.mjs index 0a11660..4ac5e5e 100644 --- a/tests/herdr-insights/collector-live-progress.test.mjs +++ b/tests/herdr-insights/collector-live-progress.test.mjs @@ -159,6 +159,38 @@ test("prepare/status/read overlay only allowlisted live worker metadata", async } }); +test("a validated worker lock reports running before its first commit or progress sidecar", async () => { + const ctx = await fixture(); + try { + const loaded = emptyStore(normalizeConfig({ sources: [] })); + loaded.generation = 4; + await saveStore(ctx.storePath, loaded, { status: "partial" }); + const before = await readCommittedSnapshot(ctx.storePath); + assert.equal(before.scan.status, "partial"); + await withStoreLock(ctx.storePath, async () => { + for (const command of ["prepare", "status"]) { + const result = await runPresenter([command, "--store", ctx.storePath]); + assert.equal(result.exitCode, 0); + const snapshot = JSON.parse(result.stdout).snapshot; + assert.equal(snapshot.scan.status, "running"); + assert.equal(snapshot.scan.workerPid, process.pid); + assert.equal(snapshot.scan.live, undefined, "no invented progress counters"); + } + const snapshot = await readCommittedSnapshot(ctx.storePath); + assert.deepEqual(snapshot.rows, before.rows); + assert.deepEqual(snapshot.stats, before.stats); + for (const overrides of [{ generation: 3 }, { pid: 999_999_999 }, { host: "foreign-host" }]) { + await atomicWriteJson(workerClaimPath(ctx.storePath), { + version: 1, token: WORKER_TOKEN, pid: process.pid, host: hostname(), generation: 4, ...overrides, + }); + assert.equal((await readCommittedSnapshot(ctx.storePath)).scan.status, "partial"); + } + }, { meta: { role: "worker", generation: 4 } }); + } finally { + await rm(ctx.root, { recursive: true, force: true }); + } +}); + test("stale job pid, generation, token, store, or non-worker lock reject live progress", async () => { const ctx = await fixture(); try { diff --git a/tests/herdr-insights/ship-readiness.test.mjs b/tests/herdr-insights/ship-readiness.test.mjs index 3e86dfc..f3b6f3f 100644 --- a/tests/herdr-insights/ship-readiness.test.mjs +++ b/tests/herdr-insights/ship-readiness.test.mjs @@ -5,6 +5,7 @@ import { hostname, tmpdir } from "node:os"; import path from "node:path"; import { registerHooks } from "node:module"; import test from "node:test"; +import { collectSnapshot } from "../../herdr-insights/mods/insights/collector.mjs"; import { progressWriter } from "../../herdr-insights/mods/insights/core/progress.mjs"; import { compactDashboard, deriveDashboard } from "../../herdr-insights/mods/insights/analytics.js"; import { MAX_LINE_BYTES, withStoreLock } from "../../herdr-insights/mods/insights/core/util.mjs"; @@ -75,6 +76,139 @@ test("Codex counter resets reset the entire vector when fields move in different assert.equal(snapshotRows(store)[0].tokens.output, 30); }); +// Synthetic fork sequences use the raw counter schema already exercised by +// codex-modern.jsonl; they establish parser behavior, not production ordering. +const forkThread = "019f0000-1000-7000-8000-000000000001"; +const copiedTurn = "019f0000-0001-7000-8000-000000000002"; +const childTurn = "019f0000-2000-7000-8000-000000000003"; +const forkParent = { input_tokens: 2000, cached_input_tokens: 800, cache_write_input_tokens: 100, + output_tokens: 500, reasoning_output_tokens: 100 }; +const forkChild = { input_tokens: 80, cached_input_tokens: 30, cache_write_input_tokens: 10, + output_tokens: 20, reasoning_output_tokens: 5 }; +const forkTotal = Object.fromEntries(Object.keys(forkParent).map((key) => [key, forkParent[key] + forkChild[key]])); +const forkEncode = (records) => records.map((record) => JSON.stringify(record) + "\n").join(""); +const forkRecord = (type, payload) => ({ type, timestamp: "2026-03-08T08:00:00Z", payload }); +const forkMarker = (turn_id) => forkRecord("turn_context", { turn_id, model: "fork-model" }); +const forkCount = (raw, lastOnly = false) => forkRecord("event_msg", { type: "token_count", + info: { [lastOnly ? "last_token_usage" : "total_token_usage"]: raw } }); +const forkModern = (response_id, usage, thread_token_usage, turn_id) => forkRecord("token_usage_record", + { response_id, usage, thread_token_usage, turn_token_usage: usage, turn_id }); +function forkHeader(id = forkThread) { + return forkRecord("session_meta", { id, forked_from_id: copiedTurn, cwd: "/repo" }); +} +const forkExpected = { input: 40, output: 20, cacheRead: 30, cacheWrite: 10, reasoning: 5, total: 100 }; + +for (const carrier of ["modern-mixed", "modern-thread-prefix", "modern-only", "modern-no-thread", "mixed", "legacy-last-only"]) { + test(`Codex fork ${carrier}: full and persisted incremental scans agree in every token category`, async (t) => { + const root = await fixture(t); + const file = path.join(root, "fork.jsonl"); + const parent = carrier === "legacy-last-only" ? forkCount(forkParent, true) + : forkModern("copied", carrier === "modern-thread-prefix" ? forkChild : forkParent, + carrier === "modern-no-thread" ? undefined : forkParent, copiedTurn); + const child = carrier === "legacy-last-only" ? forkCount(forkChild, true) + : forkModern("own", forkChild, forkTotal, childTurn); + const copied = carrier === "modern-no-thread" ? [ + forkModern("copied-first", forkChild, undefined, copiedTurn), + forkModern("copied-first", forkChild, undefined, copiedTurn), // repeated response is not pending twice + forkModern("copied-rest", Object.fromEntries(Object.keys(forkParent).map((key) => + [key, forkParent[key] - forkChild[key]])), undefined, copiedTurn), + ] : [parent]; + const marker = carrier === "modern-thread-prefix" + ? forkRecord("event_msg", { type: "task_started", root_turn_id: childTurn }) : forkMarker(childTurn); + const records = [forkHeader(), forkMarker(copiedTurn), ...copied, + ...(carrier === "mixed" ? [forkCount(forkParent)] : []), marker, child, + ...(["modern-mixed", "modern-thread-prefix", "modern-no-thread", "mixed"].includes(carrier) ? [forkCount(forkTotal)] : [])]; + await writeFile(file, forkEncode(records)); + const config = sourceStore(file, "codex").config; + const oracle = await collectSnapshot("configure", { home: root, store: path.join(root, "oracle.json"), config }); + assert.equal(oracle.rows.length, 1); + assert.deepEqual(oracle.rows[0].tokens, forkExpected); + assert.equal(oracle.rows[0].messages, 1); + assert.deepEqual(oracle.warnings, []); + + await writeFile(file, ""); + const store = path.join(root, "incremental.json"); + await collectSnapshot("configure", { home: root, store, config }); + let incremental; + for (const record of records) { + await appendFile(file, forkEncode([record])); + incremental = await collectSnapshot("snapshot", { home: root, store }); + } + assert.deepEqual(incremental.rows, oracle.rows); + assert.deepEqual((await collectSnapshot("snapshot", { home: root, store })).rows, oracle.rows); + const saved = JSON.parse(await readFile(store, "utf8")); + const entry = Object.values(saved.files)[0]; + assert.equal(entry.state.codexParserVersion, 8); + entry.state.codexParserVersion = 7; + entry.rows[0].input += 2500; // A stale PR parser contribution must be rebuilt, not retained. + await writeFile(store, JSON.stringify(saved)); + const migrated = await collectSnapshot("snapshot", { home: root, store }); + assert.equal(migrated.stats.filesRead, 1); + assert.deepEqual(migrated.rows, oracle.rows); + assert.equal((await collectSnapshot("snapshot", { home: root, store })).stats.filesRead, 0); + }); +} + +test("Codex fork replay state survives a budget slice before and after the boundary", async (t) => { + const root = await fixture(t); + const file = path.join(root, "fork.jsonl"); + const padding = forkEncode([forkRecord("ignored", { padding: "x".repeat(300_000) })]); + const prefix = forkEncode([forkHeader(), forkMarker(copiedTurn), forkModern("copied", forkParent, forkParent, copiedTurn)]); + const boundary = forkEncode([forkMarker(childTurn)]); + const suffix = forkEncode([forkModern("own", forkChild, forkTotal, childTurn), forkCount(forkTotal)]); + await writeFile(file, prefix + padding + boundary + padding + suffix); + const oracle = sourceStore(file, "codex"); + await scanSources(oracle, { home: root }); + const sliced = sourceStore(file, "codex"); + let result = await scanSources(sliced, { home: root, deadline: Date.now() }); + assert.equal(result.complete, false); + assert.equal(Object.values(sliced.files)[0].state.forkReplay, true); + assert.deepEqual(snapshotRows(sliced), []); + let sawBoundary = false; + for (let i = 0; i < 10 && !result.complete; i += 1) { + result = await scanSources(sliced, { home: root, deadline: Date.now() }); + if (Object.values(sliced.files)[0].state.forkReplay === false && !result.complete) sawBoundary = true; + } + assert.equal(sawBoundary, true); + assert.equal(result.complete, true); + assert.deepEqual(snapshotRows(sliced), snapshotRows(oracle)); +}); + +for (const marker of [undefined, "12345678-1234-4000-8000-000000000001", childTurn.slice(0, 15) + "garbage", + "019f0000-1000-7001-8000-000000000004"]) { + test(`Codex ambiguous fork boundary ${marker ?? "missing"} stays excluded and diagnosed`, async (t) => { + const root = await fixture(t); + const file = path.join(root, "fork.jsonl"); + await writeFile(file, forkEncode([forkHeader(), forkMarker(marker), forkCount(forkParent), forkCount(forkTotal)])); + const store = sourceStore(file, "codex"); + for (let i = 0; i < 2; i += 1) { + const result = await scanSources(store, { home: root }); + assert.deepEqual(snapshotRows(store), []); + assert.equal(result.warnings.length, 1, "bounded diagnostic survives unchanged scans"); + assert.match(result.warnings[0], /may exclude child usage/); + } + // A newer marker cannot retroactively establish whether the excluded + // equal-ms/non-v7 activity was copied or the child's own activity. + await appendFile(file, forkEncode([forkMarker(childTurn), forkCount({ ...forkTotal, input_tokens: 2090 })])); + const resolved = await scanSources(store, { home: root }); + assert.equal(snapshotRows(store)[0].tokens.input, 10); + assert.equal(resolved.warnings.length, 1); + }); +} + +for (const initial of ["12345678-1234-4000-8000-000000000001", "019f0000-1000-7not-a-uuid", + "019f0000-1000-7000-0000-000000000001"]) { + test(`Codex non-v7/malformed initial ID keeps the old accounting path: ${initial}`, async (t) => { + const root = await fixture(t); + const file = path.join(root, "fork.jsonl"); + await writeFile(file, forkEncode([forkHeader(initial), forkMarker(copiedTurn), forkCount(forkParent)])); + const store = sourceStore(file, "codex"); + const result = await scanSources(store, { home: root }); + assert.equal(snapshotRows(store)[0].tokens.total, 2500); + assert.deepEqual(result.warnings, []); + }); +} + test("Claude fork copies deduplicate task duration even with rewritten session IDs", async (t) => { const root = await fixture(t); const original = await readFile(new URL("./fixtures/claude-session.jsonl", import.meta.url), "utf8"); From 174aa91485298c05b8aee8dcf79c7d5f0f0fdb27 Mon Sep 17 00:00:00 2001 From: GaganSD Date: Mon, 16 Nov 2026 18:40:34 +0000 Subject: [PATCH 4/4] fix(insights): harden replay baselines and worker overlays --- herdr-insights/README.md | 18 +- herdr-insights/mods/insights/collector.mjs | 7 +- .../mods/insights/core/adapters.mjs | 109 ++++++++--- herdr-insights/mods/insights/core/scan.mjs | 10 +- .../collector-live-progress.test.mjs | 51 ++++- tests/herdr-insights/ship-readiness.test.mjs | 175 ++++++++++++++++++ 6 files changed, 330 insertions(+), 40 deletions(-) diff --git a/herdr-insights/README.md b/herdr-insights/README.md index 20df61b..adf00a9 100644 --- a/herdr-insights/README.md +++ b/herdr-insights/README.md @@ -43,8 +43,22 @@ For fork headers with canonical UUIDv7 thread IDs, Insights excludes replayed usage until a turn ID has a strictly newer UUIDv7 millisecond. Equal-millisecond, missing, malformed, or non-v7 turn IDs are ambiguous: child usage can be excluded, with a bounded scan warning that remains visible even after a later boundary. -Non-v7 initial thread IDs retain the previous accounting behavior. Existing -available Codex files are automatically rescanned once for this parser update. +Non-v7 initial thread IDs retain the previous accounting behavior. A healthy +unfinished scan slice is not itself an ambiguous boundary; reaching EOF without +a boundary is diagnosed. Existing available Codex files are automatically +rescanned once for this parser update. + +Current upstream Codex (`090be4df0724cdef5758e6a0934ba557af8b6bbe`) requires +per-response, turn, and thread usage together, and pairs legacy total and last +usage. Missing-cumulative/last-only records remain compatibility inputs, not an +established current native rollout format. Copied last-only observations use +response IDs or exact legacy record identities, never just equal amounts within +a turn. Mixed carriers lacking a reliable shared identity can be indistinguishable +from distinct responses: Insights retains a bounded diagnostic that accounting +may be inaccurate, preserves trustworthy copied cumulative baselines, and does +not subtract an unidentified last-only echo from them twice. Exact repeated +records are recognized; rewritten echoes without identity cannot be guaranteed. +Production ordering of these historical/imported shapes remains unverified. Modern thread cumulative counters and legacy totals share the raw baseline; turn-only counters are not treated as thread totals. diff --git a/herdr-insights/mods/insights/collector.mjs b/herdr-insights/mods/insights/collector.mjs index a77d3d9..f9ec181 100644 --- a/herdr-insights/mods/insights/collector.mjs +++ b/herdr-insights/mods/insights/collector.mjs @@ -207,9 +207,8 @@ async function overlayLiveProgress(storePath, snapshot) { if (claim.generation !== scan.generation) return snapshot; if (!owner || owner.role !== "worker" || owner.host !== os.hostname()) return snapshot; if (owner.pid !== claim.pid || Number(owner.generation) !== scan.generation) return snapshot; - if (Number.isInteger(scan.workerPid) && scan.workerPid !== claim.pid) return snapshot; - if (live && (live.store !== path.resolve(storePath) || live.generation !== scan.generation - || claim.pid !== live.pid || claim.token !== live.token)) return snapshot; + const matchingLive = live && live.store === path.resolve(storePath) + && live.generation === scan.generation && claim.pid === live.pid && claim.token === live.token; // The lock can precede the first running commit/progress publication. The // validated worker is running even then; optional counters are not authority. return { @@ -218,7 +217,7 @@ async function overlayLiveProgress(storePath, snapshot) { ...scan, status: "running", workerPid: claim.pid, - ...(live ? { live: liveView(live) } : {}), + ...(matchingLive ? { live: liveView(live) } : {}), }, }; } diff --git a/herdr-insights/mods/insights/core/adapters.mjs b/herdr-insights/mods/insights/core/adapters.mjs index 08ce09a..efbdd83 100644 --- a/herdr-insights/mods/insights/core/adapters.mjs +++ b/herdr-insights/mods/insights/core/adapters.mjs @@ -593,6 +593,16 @@ function codexCumulativeTwin(entry, cumulative) { return codexSameTokens(known, cumulative); } +// Replay-only fallback matches are not proof of a shared response. Keep the +// existing twin policy, but expose uncertain attribution instead of claiming +// exact accounting for carrier shapes without a shared identity. +function codexNoteReplayTwin(state, candidate, timestamp, responseId) { + if (state?.forkReplay && !(responseId && responseId === candidate.responseId) + && !(Number.isFinite(timestamp) && timestamp === candidate.timestamp)) { + state.forkUsageAmbiguous = true; + } +} + function codexPendingTwinKey( pending, tokens, @@ -600,6 +610,7 @@ function codexPendingTwinKey( cumulative, responseId, turnId, + state, ) { if ( !isPlainObject(pending) || @@ -620,6 +631,7 @@ function codexPendingTwinKey( return null; } if (turnId && pending.turnId && turnId !== pending.turnId) return null; + codexNoteReplayTwin(state, pending, timestamp, responseId); return pending.key; } @@ -668,6 +680,7 @@ function codexTakeModernTwin( (!turnId || !candidate.turnId || turnId === candidate.turnId) ) { state.modernRecent.splice(index, 1); + codexNoteReplayTwin(state, candidate, timestamp, responseId); return candidate.key; } } @@ -680,7 +693,14 @@ function parseCodex(record, ctx) { const payload = isPlainObject(record.payload) ? record.payload : {}; const timestamp = parseTimestamp(record.timestamp) ?? state.timestamp ?? null; - if (state.forkReplay && codexEndsForkReplay(state, record, payload)) state.forkReplay = false; + // A header timestamp is an emission fallback, not an exact carrier identity. + const twinTimestamp = state.forkReplay ? parseTimestamp(record.timestamp) : timestamp; + if (state.forkReplay && codexEndsForkReplay(state, record, payload)) { + state.forkReplay = false; + // Copied twins must never absorb an equal-sized child response. + state.pendingTwin = null; + state.modernRecent = []; + } if (record.type === "session_meta") { const threadId = asString( @@ -735,7 +755,13 @@ function parseCodex(record, ctx) { if (record.type === "token_usage_record") { state.modern = true; + // A raw copied baseline does not require an emit-ready per-turn record. + if (state.forkReplay && isPlainObject(payload.thread_token_usage)) { + state.legacyTotals = codexRawTotals(payload.thread_token_usage); + state.modernPending = codexZeroTokens(); + } if (timestamp === null) { + if (state.forkReplay && !isPlainObject(payload.thread_token_usage)) state.forkUsageAmbiguous = true; ctx.warn("Codex token usage record has no usable timestamp; skipped"); return; } @@ -764,26 +790,15 @@ function parseCodex(record, ctx) { tokens.reasoning, ]; let dedupKey = makeDedupKey("codex", ...identity); - if (state.forkReplay) { - const observed = codexObserveModern(state, dedupKey, tokens); - // A copied THREAD cumulative counter is the raw baseline for a legacy - // suffix; a turn counter is not. Without it retain local observations. - if (isPlainObject(payload.thread_token_usage)) { - state.legacyTotals = codexRawTotals(payload.thread_token_usage); - state.modernPending = codexZeroTokens(); - } else { - codexAddPending(state, observed); - } - return; - } let mergedTwin = false; const twinKey = codexPendingTwinKey( state.pendingTwin, tokens, - timestamp, + twinTimestamp, cumulative, responseId, turnId, + state, ); if ( responseId && @@ -802,6 +817,14 @@ function parseCodex(record, ctx) { state.responseKeys[responseId] = dedupKey; } } + if (state.forkReplay) { + const observed = codexObserveModern(state, dedupKey, tokens); + // Thread baselines already cover the observation; turn counters do not. + if (!isPlainObject(payload.thread_token_usage) && !twinKey) codexAddPending(state, observed); + state.pendingTwin = null; + codexRememberModern(state, dedupKey, tokens, twinTimestamp, { thread, turn, responseId, turnId }); + return; + } const result = ctx.addUsage( metaFor( "codex", @@ -838,13 +861,33 @@ function parseCodex(record, ctx) { record.type === "event_msg" && payload.type === "token_count" ) { - if (timestamp === null) return; - state.pendingTwin = null; const info = isPlainObject(payload.info) ? payload.info : {}; const totals = isPlainObject(info.total_token_usage) ? info.total_token_usage : null; const session = codexRecordSession(state, ctx, payload); + if (state.forkReplay && totals) { + // Baselines need neither a usable timestamp nor per-response usage. + state.legacyTotals = codexRawTotals(totals); + state.modernPending = codexZeroTokens(); + state.pendingTwin = null; + if (isPlainObject(info.last_token_usage)) { + const last = codexNormalized(info.last_token_usage); + const key = makeDedupKey("codex", session, "legacy", record.timestamp ?? ctx.lineHash, + ...Object.values(state.legacyTotals)); + const identity = { thread: codexCumulativeVector(totals), turn: null, + responseId: asString(payload.response_id, ""), turnId: asString(payload.turn_id ?? payload.root_turn_id, "") }; + codexRememberModern(state, key, last, twinTimestamp, identity); + state.pendingTwin = { key, tokens: codexTokenVector(last), timestamp: twinTimestamp, + cumulative: identity.thread, responseId: identity.responseId, turnId: identity.turnId }; + } + return; + } + if (timestamp === null) { + if (state.forkReplay && isPlainObject(info.last_token_usage)) state.forkUsageAmbiguous = true; + return; + } + state.pendingTwin = null; const context = codexContext(state, asString(payload.turn_id, "")); let raw; let identityTail; @@ -852,12 +895,6 @@ function parseCodex(record, ctx) { if (totals) { const current = codexRawTotals(totals); - if (state.forkReplay) { - // This baseline also covers any previously copied modern observations. - state.legacyTotals = current; - state.modernPending = codexZeroTokens(); - return; - } const previous = isPlainObject(state.legacyTotals) ? state.legacyTotals : null; @@ -879,7 +916,6 @@ function parseCodex(record, ctx) { current.reasoning_output_tokens, ]; } else if (isPlainObject(info.last_token_usage)) { - if (state.forkReplay) return; raw = info.last_token_usage; identityTail = [ "legacy-last", @@ -904,13 +940,16 @@ function parseCodex(record, ctx) { } if (!tokens.nonzero) return; - let dedupKey = makeDedupKey("codex", session, ...identityTail); + let dedupKey = state.forkReplay && asString(payload.response_id, "") + ? makeDedupKey("codex", "response", payload.response_id) + : makeDedupKey("codex", session, ...identityTail); + const observationKey = dedupKey; let usedModernTwin = false; if (!reset) { const modernKey = codexTakeModernTwin( state, tokens, - timestamp, + twinTimestamp, cumulative, asString(payload.response_id, ""), asString(payload.turn_id ?? payload.root_turn_id, ""), @@ -920,6 +959,26 @@ function parseCodex(record, ctx) { usedModernTwin = true; } } + if (state.forkReplay) { + const observed = codexObserveModern(state, dedupKey, tokens); + // Remember the exact legacy identity too, so repeated echoes remain + // observed after the consumable cross-carrier twin has been taken. + if (usedModernTwin && observationKey !== dedupKey) codexObserveModern(state, observationKey, tokens); + if (!usedModernTwin && CODEX_TOKEN_FIELDS.some((field) => observed[field] > 0)) { + if (isPlainObject(state.legacyTotals)) { + // Without a matching last/response identity this may already be in + // the copied total. Preserve that baseline, do not subtract twice. + state.forkUsageAmbiguous = true; + } else { + codexAddPending(state, observed); + } + } + // Keep a consumed copied twin available for a later modern echo too. + state.pendingTwin = { key: dedupKey, tokens: codexTokenVector(tokens), timestamp: twinTimestamp, + cumulative, responseId: asString(payload.response_id, ""), + turnId: asString(payload.turn_id ?? payload.root_turn_id, "") }; + return; + } const result = ctx.addUsage( metaFor( "codex", diff --git a/herdr-insights/mods/insights/core/scan.mjs b/herdr-insights/mods/insights/core/scan.mjs index adead1e..4d334a7 100644 --- a/herdr-insights/mods/insights/core/scan.mjs +++ b/herdr-insights/mods/insights/core/scan.mjs @@ -74,6 +74,7 @@ function warningSink(warnings, stats, sourceState) { }, note(message) { if (warnings.length < MAX_WARNINGS) warnings.push(message); + else hidden += 1; }, finish() { if (hidden > 0 && !reportedHidden) { @@ -949,11 +950,14 @@ async function scanOpenCodeDb(store, source, filePath, sink, stats, options = {} return { dirty: true, aborted: false }; } -function noteCodexForkReplay(entry, sink) { +function noteCodexForkReplay(entry, sink, { pending = false } = {}) { if (entry?.adapter !== "codex") return; - if (entry.state.forkReplay || entry.state.forkBoundaryAmbiguous) { + if (entry.state.forkBoundaryAmbiguous || (entry.state.forkReplay && !pending)) { sink.note(`codex ${entry.path}: fork replay usage excluded until a strictly newer UUIDv7 turn; missing, non-v7 or equal-millisecond boundaries may exclude child usage.`); } + if (entry.state.forkUsageAmbiguous) { + sink.note(`codex ${entry.path}: copied usage carriers lack a reliable shared identity; fork accounting may be inaccurate.`); + } } async function scanOneFile(store, source, filePath, sink, stats, options = {}) { @@ -1172,7 +1176,7 @@ async function scanOneFile(store, source, filePath, sink, stats, options = {}) { entry.mtimeMs = fileStat.mtimeMs; delete entry.state.scanPending; } - noteCodexForkReplay(entry, sourceSink); + noteCodexForkReplay(entry, sourceSink, { pending: plain.pending }); return { dirty, aborted: false, pending: plain.pending }; } catch (error) { rollbackStoreTransaction(tx); diff --git a/tests/herdr-insights/collector-live-progress.test.mjs b/tests/herdr-insights/collector-live-progress.test.mjs index 4ac5e5e..a5c957b 100644 --- a/tests/herdr-insights/collector-live-progress.test.mjs +++ b/tests/herdr-insights/collector-live-progress.test.mjs @@ -191,6 +191,41 @@ test("a validated worker lock reports running before its first commit or progres } }); +for (const committedPid of [null, 999_999_999]) { + test(`a replacement worker overrides partial status and stale sidecar/committed pid ${committedPid}`, async () => { + const ctx = await fixture(); + try { + const loaded = emptyStore(normalizeConfig({ sources: [] })); + loaded.generation = 4; + loaded.scan = { ...loaded.scan, workerPid: committedPid }; + await saveStore(ctx.storePath, loaded, { status: "partial" }); + const before = await readCommittedSnapshot(ctx.storePath); + assert.equal(before.scan.status, "partial"); + assert.equal(before.scan.workerPid, committedPid); + await writeProgress(ctx.storePath, { pid: 999_999_999, workerPid: 999_999_999, token: "old-worker" }); + await withStoreLock(ctx.storePath, async () => { + for (const command of ["prepare", "status"]) { + const result = await runPresenter([command, "--store", ctx.storePath]); + assert.equal(result.exitCode, 0); + const snapshot = JSON.parse(result.stdout).snapshot; + assert.equal(snapshot.scan.status, "running"); + assert.equal(snapshot.scan.workerPid, process.pid); + assert.equal(snapshot.scan.live, undefined); + } + const snapshot = await readCommittedSnapshot(ctx.storePath); + assert.deepEqual(snapshot.rows, before.rows); + assert.deepEqual(snapshot.stats, before.stats); + await writeProgress(ctx.storePath); + const matching = await readCommittedSnapshot(ctx.storePath); + assert.equal(matching.scan.workerPid, process.pid); + assert.equal(matching.scan.live.decodedBytes, 123_456, "stale committed pid cannot veto current counters"); + }, { meta: { role: "worker", generation: 4 } }); + } finally { + await rm(ctx.root, { recursive: true, force: true }); + } + }); +} + test("stale job pid, generation, token, store, or non-worker lock reject live progress", async () => { const ctx = await fixture(); try { @@ -205,15 +240,19 @@ test("stale job pid, generation, token, store, or non-worker lock reject live pr await writeProgress(ctx.storePath, overrides); const snapshot = await readCommittedSnapshot(ctx.storePath); assert.equal(snapshot.scan.live, undefined, label); - assert.equal(snapshot.scan.workerPid, null, label); + assert.equal(snapshot.scan.status, "running", label); + assert.equal(snapshot.scan.workerPid, process.pid, label); } }, { timeoutMs: 2_000, meta: { role: "worker", generation: 4 } }); - await withStoreLock(ctx.storePath, async () => { - await writeProgress(ctx.storePath); - const snapshot = await readCommittedSnapshot(ctx.storePath); - assert.equal(snapshot.scan.live, undefined, "a non-worker lock owner rejects progress"); - }, { timeoutMs: 2_000, meta: { role: "collector", generation: 4 } }); + for (const meta of [{ role: "collector", generation: 4 }, { role: "worker", generation: 3 }]) { + await withStoreLock(ctx.storePath, async () => { + await writeProgress(ctx.storePath); + const snapshot = await readCommittedSnapshot(ctx.storePath); + assert.equal(snapshot.scan.live, undefined, "non-worker or wrong-generation lock rejects progress"); + assert.equal(snapshot.scan.workerPid, null, "invalid lock cannot establish worker identity"); + }, { timeoutMs: 2_000, meta }); + } } finally { await rm(ctx.root, { recursive: true, force: true }); } diff --git a/tests/herdr-insights/ship-readiness.test.mjs b/tests/herdr-insights/ship-readiness.test.mjs index f3b6f3f..3942cd9 100644 --- a/tests/herdr-insights/ship-readiness.test.mjs +++ b/tests/herdr-insights/ship-readiness.test.mjs @@ -149,6 +149,152 @@ for (const carrier of ["modern-mixed", "modern-thread-prefix", "modern-only", "m }); } +// Missing/last-only carriers are compatibility inputs, not the required paired +// cumulative fields in current upstream Codex TokenUsageRecord/TokenUsageInfo. +async function checkForkCompatibility(t, records, { ambiguous = false, timestampWarning = false } = {}) { + const root = await fixture(t); + const file = path.join(root, "compatibility.jsonl"); + await writeFile(file, forkEncode(records)); + const config = sourceStore(file, "codex").config; + const store = path.join(root, "full.json"); + const full = await collectSnapshot("configure", { home: root, store, config }); + if (!ambiguous) { + assert.equal(full.rows.length, 1); + assert.deepEqual(full.rows[0].tokens, forkExpected); + assert.equal(full.rows[0].messages, 1); + } + assert.equal(full.warnings.filter((message) => /lack a reliable shared identity/.test(message)).length, ambiguous ? 1 : 0); + assert.equal(full.warnings.length, Number(ambiguous) + Number(timestampWarning)); + await writeFile(file, ""); + const incrementalStore = path.join(root, "incremental.json"); + await collectSnapshot("configure", { home: root, store: incrementalStore, config }); + let incremental; + for (const record of records) { + await appendFile(file, forkEncode([record])); + incremental = await collectSnapshot("snapshot", { home: root, store: incrementalStore }); + } + assert.deepEqual(incremental.rows, full.rows); + const unchanged = await collectSnapshot("snapshot", { home: root, store: incrementalStore }); + assert.deepEqual(unchanged.rows, full.rows); + assert.equal(unchanged.warnings.filter((message) => /lack a reliable shared identity/.test(message)).length, ambiguous ? 1 : 0); +} + +for (const shape of ["zero-usage", "missing-usage", "modern-missing-time", "modern-invalid-time", "legacy-missing-time", "legacy-invalid-time"]) { + test(`Codex replay raw cumulative baseline survives ${shape}`, async (t) => { + const header = forkHeader(); + const modern = !shape.startsWith("legacy"); + const parent = modern ? forkModern("copied", forkParent, forkParent, copiedTurn) : forkCount(forkParent); + if (shape === "zero-usage") parent.payload.usage = {}; + if (shape === "missing-usage") delete parent.payload.usage; + if (shape.endsWith("time")) { + delete header.timestamp; + if (shape.includes("invalid")) parent.timestamp = "not-a-timestamp"; + else delete parent.timestamp; + } + await checkForkCompatibility(t, [header, forkMarker(copiedTurn), parent, forkMarker(childTurn), forkCount(forkTotal)], + { timestampWarning: modern && shape.endsWith("time") }); + }); +} + +for (const shape of ["zero-usage", "missing-usage", "missing-time", "invalid-time"]) { + test(`Codex own usage validation still skips ${shape} without consuming its baseline`, async (t) => { + const root = await fixture(t); + const file = path.join(root, "own.jsonl"); + const header = forkHeader("non-v7-thread"); + delete header.timestamp; + const record = forkModern("own-invalid", forkParent, forkParent, childTurn); + if (shape === "zero-usage") record.payload.usage = {}; + if (shape === "missing-usage") delete record.payload.usage; + if (shape === "missing-time") delete record.timestamp; + if (shape === "invalid-time") record.timestamp = "invalid"; + await writeFile(file, forkEncode([header, record])); + const store = sourceStore(file, "codex"); + await scanSources(store, { home: root }); + assert.deepEqual(snapshotRows(store), []); + assert.equal(Object.values(store.files)[0].state.legacyTotals, undefined); + await appendFile(file, forkEncode([forkCount(forkParent)])); + await scanSources(store, { home: root }); + assert.equal(snapshotRows(store)[0].tokens.total, 2500); + }); +} + +for (const shape of ["last-to-total", "repeated-exact-last", "repeated-response", "distinct-equal-last", "distinct-equal-response", + "modern-last", "last-modern", "modern-thread-last", "last-modern-thread", "response-modern-last", "response-last-modern", + "total-last", "last-total-last", "total-last-modern", "total-modern-last", "equal-child-modern"]) { + test(`Codex copied compatibility observations reconcile ${shape} in full and append scans`, async (t) => { + const last = forkCount(forkParent, true); + const modern = forkModern("copied", forkParent, undefined, copiedTurn); + const total = forkCount(forkParent); + total.payload.info.last_token_usage = forkParent; + let prefix = [last]; + if (shape === "repeated-exact-last") prefix = [last, last, last]; + if (shape === "repeated-response") prefix = [ + { ...last, payload: { ...last.payload, response_id: "copied" } }, + { ...last, timestamp: "2026-03-08T08:00:01Z", payload: { ...last.payload, response_id: "copied" } }, + ]; + if (shape === "distinct-equal-last" || shape === "distinct-equal-response") { + const half = Object.fromEntries(Object.entries(forkParent).map(([key, value]) => [key, value / 2])); + prefix = [forkCount(half, true), { ...forkCount(half, true), timestamp: "2026-03-08T08:00:01Z" }]; + if (shape === "distinct-equal-response") { + prefix[0].payload.response_id = "first"; + prefix[1].payload.response_id = "second"; + prefix[1].timestamp = prefix[0].timestamp; + } + } + if (shape === "modern-last") prefix = [modern, last, last]; + if (shape === "last-modern") prefix = [last, modern, modern]; + if (shape === "modern-thread-last" || shape === "last-modern-thread") { + modern.payload.thread_token_usage = forkParent; + prefix = shape === "modern-thread-last" ? [modern, last, last] : [last, modern, modern]; + } + if (shape === "response-modern-last" || shape === "response-last-modern") { + last.payload.response_id = "copied"; + last.timestamp = "2026-03-08T08:00:01Z"; + prefix = shape === "response-modern-last" ? [modern, last, last] : [last, modern, modern]; + } + if (shape === "total-last") prefix = [total, last, last]; + if (shape === "last-total-last") prefix = [last, total, last]; + if (shape === "total-last-modern") prefix = [total, last, modern]; + if (shape === "total-modern-last") prefix = [total, modern, last]; + const suffix = shape === "equal-child-modern" + ? [forkModern("own", forkChild, undefined, childTurn), forkCount(forkTotal)] : [forkCount(forkTotal)]; + if (shape === "equal-child-modern") { + // An equal-sized copied twin cannot consume a distinct child response. + const rest = Object.fromEntries(Object.keys(forkParent).map((key) => [key, forkParent[key] - forkChild[key]])); + prefix = [forkCount(rest, true), { ...forkCount(forkChild, true), timestamp: "2026-03-08T08:00:01Z" }]; + } + await checkForkCompatibility(t, [forkHeader(), forkMarker(copiedTurn), ...prefix, forkMarker(childTurn), ...suffix]); + }); +} + +for (const shape of ["unidentified-total-last", "weak-modern-last", "weak-last-modern", "inferred-time-modern-last"]) { + test(`Codex unsupported copied attribution ${shape} preserves a bounded persistent diagnostic`, async (t) => { + const last = { ...forkCount(forkParent, true), timestamp: "2026-03-08T08:00:01Z" }; + const modern = forkModern("copied", forkParent, undefined, copiedTurn); + const prefix = shape === "unidentified-total-last" ? [forkCount(forkParent), last, last] + : shape === "weak-last-modern" ? [last, modern] : [modern, last]; + if (shape === "inferred-time-modern-last") { + delete modern.timestamp; + delete last.timestamp; + } + await checkForkCompatibility(t, [forkHeader(), forkMarker(copiedTurn), ...prefix, + forkMarker(childTurn), forkCount(forkTotal)], { ambiguous: true }); + }); +} + +for (const carrier of ["last-only", "modern-no-thread"]) { + test(`Codex timestamp-free copied ${carrier} retains an accounting diagnostic`, async (t) => { + const header = forkHeader(); + delete header.timestamp; + const copied = carrier === "last-only" ? forkCount(forkParent, true) + : forkModern("copied", forkParent, undefined, copiedTurn); + delete copied.timestamp; + await checkForkCompatibility(t, [header, forkMarker(copiedTurn), copied, + forkMarker(childTurn), forkCount(forkTotal)], + { ambiguous: true, timestampWarning: carrier === "modern-no-thread" }); + }); +} + test("Codex fork replay state survives a budget slice before and after the boundary", async (t) => { const root = await fixture(t); const file = path.join(root, "fork.jsonl"); @@ -163,10 +309,12 @@ test("Codex fork replay state survives a budget slice before and after the bound let result = await scanSources(sliced, { home: root, deadline: Date.now() }); assert.equal(result.complete, false); assert.equal(Object.values(sliced.files)[0].state.forkReplay, true); + assert.deepEqual(result.warnings, [], "unfinished healthy prefix is not an ambiguous boundary"); assert.deepEqual(snapshotRows(sliced), []); let sawBoundary = false; for (let i = 0; i < 10 && !result.complete; i += 1) { result = await scanSources(sliced, { home: root, deadline: Date.now() }); + assert.deepEqual(result.warnings, []); if (Object.values(sliced.files)[0].state.forkReplay === false && !result.complete) sawBoundary = true; } assert.equal(sawBoundary, true); @@ -209,6 +357,33 @@ for (const initial of ["12345678-1234-4000-8000-000000000001", "019f0000-1000-7n }); } +test("Codex fork notes count bounded overflow without inflating errors", async (t) => { + const root = await fixture(t); + const records = forkEncode([forkHeader(), forkMarker(undefined), forkCount(forkParent)]); + await Promise.all(Array.from({ length: 53 }, (_, index) => writeFile(path.join(root, `${index}.jsonl`), records))); + const store = sourceStore(root, "codex"); + for (let i = 0; i < 2; i += 1) { + const result = await scanSources(store, { home: root }); + assert.equal(result.stats.errors, 0); + assert.equal(result.sourceStats.codex.errors, 0); + assert.equal(result.warnings.length, 51); + assert.equal(result.warnings[50], "3 additional scan warnings were not shown."); + } +}); + +test("Codex replay reaching EOF without a boundary remains diagnosed", async (t) => { + const root = await fixture(t); + const file = path.join(root, "fork.jsonl"); + await writeFile(file, forkEncode([forkHeader(), forkMarker(copiedTurn), forkCount(forkParent)])); + const store = sourceStore(file, "codex"); + for (let i = 0; i < 2; i += 1) { + const result = await scanSources(store, { home: root }); + assert.equal(result.complete, true); + assert.equal(result.warnings.length, 1); + assert.match(result.warnings[0], /may exclude child usage/); + } +}); + test("Claude fork copies deduplicate task duration even with rewritten session IDs", async (t) => { const root = await fixture(t); const original = await readFile(new URL("./fixtures/claude-session.jsonl", import.meta.url), "utf8");