Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions herdr-insights/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,31 @@ click-state rules. Set `HERDR_APP_HOME` to use a different app home.
- Open **Settings** to choose sources, edit paths, toggle cached tokens, or
change the palette and refresh interval.

### Codex fork accounting

For fork headers with canonical UUIDv7 thread IDs, Insights excludes replayed
usage until a turn ID has a strictly newer UUIDv7 millisecond. Equal-millisecond,
missing, malformed, or non-v7 turn IDs are ambiguous: child usage can be excluded,
with a bounded scan warning that remains visible even after a later boundary.
Non-v7 initial thread IDs retain the previous accounting behavior. A healthy
unfinished scan slice is not itself an ambiguous boundary; reaching EOF without
a boundary is diagnosed. Existing available Codex files are automatically
rescanned once for this parser update.

Current upstream Codex (`090be4df0724cdef5758e6a0934ba557af8b6bbe`) requires
per-response, turn, and thread usage together, and pairs legacy total and last
usage. Missing-cumulative/last-only records remain compatibility inputs, not an
established current native rollout format. Copied last-only observations use
response IDs or exact legacy record identities, never just equal amounts within
a turn. Mixed carriers lacking a reliable shared identity can be indistinguishable
from distinct responses: Insights retains a bounded diagnostic that accounting
may be inaccurate, preserves trustworthy copied cumulative baselines, and does
not subtract an unidentified last-only echo from them twice. Exact repeated
records are recognized; rewritten echoes without identity cannot be guaranteed.
Production ordering of these historical/imported shapes remains unverified.
Modern thread cumulative counters and legacy totals share the raw baseline;
turn-only counters are not treated as thread totals.

## Privacy

No network requests or uploads. The mod runs a local Node helper, reads session
Expand Down
24 changes: 12 additions & 12 deletions herdr-insights/mods/insights/collector.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -197,27 +197,27 @@ async function overlayLiveProgress(storePath, snapshot) {
const scan = snapshot?.scan;
if (!scan || !["running", "partial", "starting"].includes(scan.status)) return snapshot;
if (!Number.isInteger(scan.generation)) return snapshot;
const live = await readLiveProgress(storePath);
if (!live) return snapshot;
if (live.store !== path.resolve(storePath) || live.generation !== scan.generation) return snapshot;
const [claim, owner] = await Promise.all([
const [live, claim, owner] = await Promise.all([
readLiveProgress(storePath),
readWorkerClaim(storePath),
readStoreLockInfo(storePath),
]);
if (!claim || claim.host !== os.hostname() || !workerClaimActive(claim)) return snapshot;
if (claim.pid !== live.pid || claim.generation !== live.generation || claim.token !== live.token) {
return snapshot;
}
if (!claim || claim.host !== os.hostname() || !Number.isInteger(claim.pid)
|| claim.pid <= 0 || !workerClaimActive(claim)) return snapshot;
if (claim.generation !== scan.generation) return snapshot;
if (!owner || owner.role !== "worker" || owner.host !== os.hostname()) return snapshot;
if (owner.pid !== live.pid || Number(owner.generation) !== live.generation) return snapshot;
if (Number.isInteger(scan.workerPid) && scan.workerPid !== live.pid) return snapshot;
if (owner.pid !== claim.pid || Number(owner.generation) !== scan.generation) return snapshot;
const matchingLive = live && live.store === path.resolve(storePath)
&& live.generation === scan.generation && claim.pid === live.pid && claim.token === live.token;
// The lock can precede the first running commit/progress publication. The
// validated worker is running even then; optional counters are not authority.
return {
...snapshot,
scan: {
...scan,
status: "running",
workerPid: live.pid,
live: liveView(live),
workerPid: claim.pid,
...(matchingLive ? { live: liveView(live) } : {}),
},
};
}
Expand Down
141 changes: 128 additions & 13 deletions herdr-insights/mods/insights/core/adapters.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -458,6 +458,26 @@ function codexRecordSession(state, ctx, payload) {
return codexSession(state, ctx);
}

function codexUuidV7Ms(id) {
const match = /^([0-9a-f]{8})-([0-9a-f]{4})-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.exec(id);
return match ? Number.parseInt(match[1] + match[2], 16) : null;
}

// Supported fork boundary: a canonical UUIDv7 turn strictly newer than the
// initial thread's UUIDv7 millisecond. Equal milliseconds cannot distinguish
// copied from child turns; missing/other-version IDs cannot order them at all.
// Conservatively exclude that prefix and retain a scanner diagnostic rather
// than guessing from record timestamps (which can be rewritten during a copy).
function codexEndsForkReplay(state, record, payload) {
if (record.type !== "turn_context" && !(record.type === "event_msg" && payload.type === "task_started")) {
return false;
}
const turnId = asString(payload.turn_id ?? payload.root_turn_id, "");
const turnMs = codexUuidV7Ms(turnId);
if (turnMs === null || turnMs === state.forkStartMs) state.forkBoundaryAmbiguous = true;
return turnMs !== null && turnMs > state.forkStartMs;
}

function codexContext(state, turnId) {
if (!isPlainObject(state.turns)) state.turns = {};
if (turnId && isPlainObject(state.turns[turnId])) return state.turns[turnId];
Expand All @@ -482,6 +502,19 @@ function codexNormalized(raw) {
});
}

// Both modern thread_token_usage and legacy total_token_usage use raw Codex
// counters: input includes cache, output includes reasoning. Never store the
// normalized (cache-subtracted) vector as the legacy cumulative baseline.
function codexRawTotals(raw) {
return {
input_tokens: toNonNegInt(raw.input_tokens),
cached_input_tokens: toNonNegInt(raw.cached_input_tokens),
cache_write_input_tokens: toNonNegInt(raw.cache_write_input_tokens),
output_tokens: toNonNegInt(raw.output_tokens),
reasoning_output_tokens: toNonNegInt(raw.reasoning_output_tokens),
};
}

const CODEX_TOKEN_FIELDS = ["input", "output", "cacheRead", "cacheWrite", "reasoning"];
const CODEX_TWIN_WINDOW_MS = 5 * 60 * 1000;

Expand Down Expand Up @@ -560,13 +593,24 @@ function codexCumulativeTwin(entry, cumulative) {
return codexSameTokens(known, cumulative);
}

// Replay-only fallback matches are not proof of a shared response. Keep the
// existing twin policy, but expose uncertain attribution instead of claiming
// exact accounting for carrier shapes without a shared identity.
function codexNoteReplayTwin(state, candidate, timestamp, responseId) {
if (state?.forkReplay && !(responseId && responseId === candidate.responseId)
&& !(Number.isFinite(timestamp) && timestamp === candidate.timestamp)) {
state.forkUsageAmbiguous = true;
}
}

function codexPendingTwinKey(
pending,
tokens,
timestamp,
cumulative,
responseId,
turnId,
state,
) {
if (
!isPlainObject(pending) ||
Expand All @@ -587,6 +631,7 @@ function codexPendingTwinKey(
return null;
}
if (turnId && pending.turnId && turnId !== pending.turnId) return null;
codexNoteReplayTwin(state, pending, timestamp, responseId);
return pending.key;
}

Expand Down Expand Up @@ -635,6 +680,7 @@ function codexTakeModernTwin(
(!turnId || !candidate.turnId || turnId === candidate.turnId)
) {
state.modernRecent.splice(index, 1);
codexNoteReplayTwin(state, candidate, timestamp, responseId);
return candidate.key;
}
}
Expand All @@ -647,14 +693,31 @@ function parseCodex(record, ctx) {
const payload = isPlainObject(record.payload) ? record.payload : {};
const timestamp =
parseTimestamp(record.timestamp) ?? state.timestamp ?? null;
// A header timestamp is an emission fallback, not an exact carrier identity.
const twinTimestamp = state.forkReplay ? parseTimestamp(record.timestamp) : timestamp;
if (state.forkReplay && codexEndsForkReplay(state, record, payload)) {
state.forkReplay = false;
// Copied twins must never absorb an equal-sized child response.
state.pendingTwin = null;
state.modernRecent = [];
}

if (record.type === "session_meta") {
const threadId = asString(
payload.id ?? payload.session_id ?? payload.thread_id,
"",
);
const firstHeader = !state.session;
if (threadId && firstHeader) state.session = threadId;
if (threadId && firstHeader) {
state.session = threadId;
const forkStartMs = codexUuidV7Ms(threadId);
// Without a time-ordered thread id there is no safe boundary; keep
// the previous behavior rather than risk dropping the child's usage.
if (forkStartMs !== null && asString(payload.forked_from_id ?? payload.parent_thread_id, "")) {
state.forkReplay = true;
state.forkStartMs = forkStartMs;
}
}
if (firstHeader || !state.project) {
state.project = asString(payload.cwd, state.project ?? "unknown");
}
Expand Down Expand Up @@ -692,7 +755,13 @@ function parseCodex(record, ctx) {

if (record.type === "token_usage_record") {
state.modern = true;
// A raw copied baseline does not require an emit-ready per-turn record.
if (state.forkReplay && isPlainObject(payload.thread_token_usage)) {
state.legacyTotals = codexRawTotals(payload.thread_token_usage);
state.modernPending = codexZeroTokens();
}
if (timestamp === null) {
if (state.forkReplay && !isPlainObject(payload.thread_token_usage)) state.forkUsageAmbiguous = true;
ctx.warn("Codex token usage record has no usable timestamp; skipped");
return;
}
Expand Down Expand Up @@ -725,10 +794,11 @@ function parseCodex(record, ctx) {
const twinKey = codexPendingTwinKey(
state.pendingTwin,
tokens,
timestamp,
twinTimestamp,
cumulative,
responseId,
turnId,
state,
);
if (
responseId &&
Expand All @@ -747,6 +817,14 @@ function parseCodex(record, ctx) {
state.responseKeys[responseId] = dedupKey;
}
}
if (state.forkReplay) {
const observed = codexObserveModern(state, dedupKey, tokens);
// Thread baselines already cover the observation; turn counters do not.
if (!isPlainObject(payload.thread_token_usage) && !twinKey) codexAddPending(state, observed);
state.pendingTwin = null;
codexRememberModern(state, dedupKey, tokens, twinTimestamp, { thread, turn, responseId, turnId });
return;
}
const result = ctx.addUsage(
metaFor(
"codex",
Expand Down Expand Up @@ -783,26 +861,40 @@ function parseCodex(record, ctx) {
record.type === "event_msg" &&
payload.type === "token_count"
) {
if (timestamp === null) return;
state.pendingTwin = null;
const info = isPlainObject(payload.info) ? payload.info : {};
const totals = isPlainObject(info.total_token_usage)
? info.total_token_usage
: null;
const session = codexRecordSession(state, ctx, payload);
if (state.forkReplay && totals) {
// Baselines need neither a usable timestamp nor per-response usage.
state.legacyTotals = codexRawTotals(totals);
state.modernPending = codexZeroTokens();
state.pendingTwin = null;
if (isPlainObject(info.last_token_usage)) {
const last = codexNormalized(info.last_token_usage);
const key = makeDedupKey("codex", session, "legacy", record.timestamp ?? ctx.lineHash,
...Object.values(state.legacyTotals));
const identity = { thread: codexCumulativeVector(totals), turn: null,
responseId: asString(payload.response_id, ""), turnId: asString(payload.turn_id ?? payload.root_turn_id, "") };
codexRememberModern(state, key, last, twinTimestamp, identity);
state.pendingTwin = { key, tokens: codexTokenVector(last), timestamp: twinTimestamp,
cumulative: identity.thread, responseId: identity.responseId, turnId: identity.turnId };
}
return;
}
if (timestamp === null) {
if (state.forkReplay && isPlainObject(info.last_token_usage)) state.forkUsageAmbiguous = true;
return;
}
state.pendingTwin = null;
const context = codexContext(state, asString(payload.turn_id, ""));
let raw;
let identityTail;
let reset = false;

if (totals) {
const current = {
input_tokens: toNonNegInt(totals.input_tokens),
cached_input_tokens: toNonNegInt(totals.cached_input_tokens),
cache_write_input_tokens: toNonNegInt(totals.cache_write_input_tokens),
output_tokens: toNonNegInt(totals.output_tokens),
reasoning_output_tokens: toNonNegInt(totals.reasoning_output_tokens),
};
const current = codexRawTotals(totals);
const previous = isPlainObject(state.legacyTotals)
? state.legacyTotals
: null;
Expand Down Expand Up @@ -848,13 +940,16 @@ function parseCodex(record, ctx) {
}
if (!tokens.nonzero) return;

let dedupKey = makeDedupKey("codex", session, ...identityTail);
let dedupKey = state.forkReplay && asString(payload.response_id, "")
? makeDedupKey("codex", "response", payload.response_id)
: makeDedupKey("codex", session, ...identityTail);
const observationKey = dedupKey;
let usedModernTwin = false;
if (!reset) {
const modernKey = codexTakeModernTwin(
state,
tokens,
timestamp,
twinTimestamp,
cumulative,
asString(payload.response_id, ""),
asString(payload.turn_id ?? payload.root_turn_id, ""),
Expand All @@ -864,6 +959,26 @@ function parseCodex(record, ctx) {
usedModernTwin = true;
}
}
if (state.forkReplay) {
const observed = codexObserveModern(state, dedupKey, tokens);
// Remember the exact legacy identity too, so repeated echoes remain
// observed after the consumable cross-carrier twin has been taken.
if (usedModernTwin && observationKey !== dedupKey) codexObserveModern(state, observationKey, tokens);
if (!usedModernTwin && CODEX_TOKEN_FIELDS.some((field) => observed[field] > 0)) {
if (isPlainObject(state.legacyTotals)) {
// Without a matching last/response identity this may already be in
// the copied total. Preserve that baseline, do not subtract twice.
state.forkUsageAmbiguous = true;
} else {
codexAddPending(state, observed);
}
}
// Keep a consumed copied twin available for a later modern echo too.
state.pendingTwin = { key: dedupKey, tokens: codexTokenVector(tokens), timestamp: twinTimestamp,
cumulative, responseId: asString(payload.response_id, ""),
turnId: asString(payload.turn_id ?? payload.root_turn_id, "") };
return;
}
const result = ctx.addUsage(
metaFor(
"codex",
Expand Down
17 changes: 16 additions & 1 deletion herdr-insights/mods/insights/core/scan.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ import {
const CHUNK_BYTES = 256 * 1024;
const MAX_WARNINGS = 50;
const MAX_TRAVERSAL_ENTRIES = 250_000;
const PARSER_VERSIONS = { pi: 3, codex: 5, claude: 2, antigravity: 2 };
const PARSER_VERSIONS = { pi: 3, codex: 8, claude: 2, antigravity: 2 };
const PROGRESS_CHUNK_INTERVAL = 64;

export class ScanStopError extends Error {
Expand Down Expand Up @@ -74,6 +74,7 @@ function warningSink(warnings, stats, sourceState) {
},
note(message) {
if (warnings.length < MAX_WARNINGS) warnings.push(message);
else hidden += 1;
},
finish() {
if (hidden > 0 && !reportedHidden) {
Expand Down Expand Up @@ -949,6 +950,16 @@ async function scanOpenCodeDb(store, source, filePath, sink, stats, options = {}
return { dirty: true, aborted: false };
}

function noteCodexForkReplay(entry, sink, { pending = false } = {}) {
if (entry?.adapter !== "codex") return;
if (entry.state.forkBoundaryAmbiguous || (entry.state.forkReplay && !pending)) {
sink.note(`codex ${entry.path}: fork replay usage excluded until a strictly newer UUIDv7 turn; missing, non-v7 or equal-millisecond boundaries may exclude child usage.`);
}
if (entry.state.forkUsageAmbiguous) {
sink.note(`codex ${entry.path}: copied usage carriers lack a reliable shared identity; fork accounting may be inaccurate.`);
}
}

async function scanOneFile(store, source, filePath, sink, stats, options = {}) {
const sourceSink = {
warn(message) {
Expand Down Expand Up @@ -997,6 +1008,7 @@ async function scanOneFile(store, source, filePath, sink, stats, options = {}) {
&& existing.mtimeMs === fileStat.mtimeMs
) {
stats.unchangedFiles += 1;
noteCodexForkReplay(existing, sourceSink);
return { dirty: false, aborted: false };
}

Expand Down Expand Up @@ -1065,6 +1077,7 @@ async function scanOneFile(store, source, filePath, sink, stats, options = {}) {
) {
stats.unchangedFiles += 1;
commitStoreTransaction(tx);
noteCodexForkReplay(entry, sourceSink);
return { dirty, aborted: false };
}

Expand Down Expand Up @@ -1135,6 +1148,7 @@ async function scanOneFile(store, source, filePath, sink, stats, options = {}) {
if (parserVersion) entry.state[parserKey] = parserVersion;
delete entry.state.zstdError;
delete entry.state.scanPending;
noteCodexForkReplay(entry, sourceSink);
return { dirty, aborted: false };
}

Expand Down Expand Up @@ -1162,6 +1176,7 @@ async function scanOneFile(store, source, filePath, sink, stats, options = {}) {
entry.mtimeMs = fileStat.mtimeMs;
delete entry.state.scanPending;
}
noteCodexForkReplay(entry, sourceSink, { pending: plain.pending });
return { dirty, aborted: false, pending: plain.pending };
} catch (error) {
rollbackStoreTransaction(tx);
Expand Down
Loading
Loading