Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
167 changes: 167 additions & 0 deletions benchmark/runtime/session_observation.py
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,173 @@ def observe_codex_session_tail(
}


def collect_context_policy_telemetry(
runtime_root: Path | None = None,
*,
goal_id: str | None = None,
context_dir: Path | None = None,
usage: dict | None = None,
) -> dict:
"""Collect {KeepRaw, FoldToReceipt, EpochReset}, prefetch, quiet-turn, and cold/warm KV telemetry."""
operator_counts = {"KeepRaw": 0, "FoldToReceipt": 0, "EpochReset": 0}
quiet_turn_skips = 0
prefetch_attempts = 0
prefetch_latency_ms = 0
prefetch_bytes = 0
prefetch_bytes_saved = 0
round_trips_saved = 0
seen_receipts: set[str] = set()
seen_quiet_turns: set[str] = set()

def _consume_prefetch(entry: object) -> None:
nonlocal prefetch_attempts, prefetch_latency_ms, prefetch_bytes, prefetch_bytes_saved, round_trips_saved
if not isinstance(entry, dict):
return
receipt_id = entry.get("receipt_sha256")
if isinstance(receipt_id, str) and receipt_id:
if receipt_id in seen_receipts:
return
seen_receipts.add(receipt_id)
prefetch_attempts += 1
op = entry.get("operator")
if op in operator_counts:
operator_counts[op] += 1
if isinstance(entry.get("latency_ms"), int) and entry["latency_ms"] >= 0:
prefetch_latency_ms += entry["latency_ms"]
if isinstance(entry.get("retained_bytes"), int) and entry["retained_bytes"] >= 0:
prefetch_bytes += entry["retained_bytes"]
if isinstance(entry.get("bytes_saved"), int) and entry["bytes_saved"] >= 0:
prefetch_bytes_saved += entry["bytes_saved"]
if isinstance(entry.get("round_trips_saved"), int) and entry["round_trips_saved"] >= 0:
round_trips_saved += entry["round_trips_saved"]

def _consume_quiet(entry: object, dedup_key: str | None = None) -> None:
nonlocal quiet_turn_skips
if not isinstance(entry, dict) or entry.get("skipped") is not True:
return
if dedup_key:
if dedup_key in seen_quiet_turns:
return
seen_quiet_turns.add(dedup_key)
quiet_turn_skips += 1

if runtime_root is not None and goal_id:
goal_root = Path(runtime_root) / "goals" / goal_id
turns_dir = goal_root / "turns"
if turns_dir.is_dir():
for path in sorted(turns_dir.glob("*.json")):
try:
journal = json.loads(path.read_text(encoding="utf-8"))
except (OSError, ValueError):
continue
if isinstance(journal, dict):
_consume_prefetch(journal.get("epoch_prefetch"))
turn_id = str(journal.get("turn_key") or path.stem)
_consume_quiet(journal.get("quiet_turn"), turn_id)
quiet_skips_dir = goal_root / "quiet_turn" / "skips"
if quiet_skips_dir.is_dir():
for path in sorted(quiet_skips_dir.glob("*.json")):
try:
receipt = json.loads(path.read_text(encoding="utf-8"))
except (OSError, ValueError):
continue
if isinstance(receipt, dict):
turn_id = str(receipt.get("turn_key") or path.stem)
_consume_quiet(receipt.get("quiet_turn") or receipt, turn_id)
sqlite_path = goal_root / "turn_prefetch_state.sqlite"
if sqlite_path.is_file():
import sqlite3
from contextlib import closing

try:
with closing(sqlite3.connect(sqlite_path, timeout=2.0)) as conn:
rows = conn.execute(
"SELECT turn_key, receipt_json FROM quiet_turn_skips ORDER BY recorded_at"
).fetchall()
for row_key, row_json in rows:
try:
receipt = json.loads(row_json)
except (TypeError, ValueError):
continue
if isinstance(receipt, dict):
_consume_quiet(receipt.get("quiet_turn") or receipt, str(row_key))
except sqlite3.Error:
pass

if context_dir is not None:
session_file = Path(context_dir) / "session.json"
if session_file.is_file():
try:
session_data = json.loads(session_file.read_text(encoding="utf-8"))
except (OSError, ValueError):
session_data = {}
if isinstance(session_data, dict):
_consume_prefetch(session_data.get("epoch_prefetch"))
_consume_quiet(session_data.get("quiet_turn"), "__codex_session__")

cold = {"requests": 0, "uncached_prefix_kv_tokens": 0, "cached_prefix_kv_tokens": 0}
warm = {"requests": 0, "uncached_prefix_kv_tokens": 0, "cached_prefix_kv_tokens": 0}
if isinstance(usage, dict):
embedded = usage.get("context_policy_telemetry")
embedded_kv = embedded.get("prefix_kv") if isinstance(embedded, dict) else None
if (
isinstance(embedded_kv, dict)
and isinstance(embedded_kv.get("cold"), dict)
and isinstance(embedded_kv.get("warm"), dict)
):
for dst, src in ((cold, embedded_kv["cold"]), (warm, embedded_kv["warm"])):
for key in ("requests", "uncached_prefix_kv_tokens", "cached_prefix_kv_tokens"):
val = src.get(key)
if isinstance(val, int) and not isinstance(val, bool) and val >= 0:
dst[key] = val
else:
coverage = usage.get("request_coverage")
requests = coverage.get("requests") if isinstance(coverage, dict) else None
if isinstance(requests, list):
for req in requests:
if not isinstance(req, dict):
continue
inp = req.get("input_tokens")
cached = req.get("cached_input_tokens")
if not isinstance(inp, int) or not isinstance(cached, int) or inp < cached:
continue
uncached = inp - cached
is_cold = (
cached == 0
or req.get("cache_regime") == "cold"
or req.get("compacted") is True
)
bucket = cold if is_cold else warm
bucket["requests"] += 1
bucket["uncached_prefix_kv_tokens"] += uncached
bucket["cached_prefix_kv_tokens"] += cached
elif isinstance(usage.get("input_tokens"), int) and isinstance(usage.get("cached_input_tokens"), int):
cached = usage["cached_input_tokens"]
uncached = max(0, usage["input_tokens"] - cached)
bucket = cold if cached == 0 else warm
bucket["requests"] = int(usage.get("completed_turns") or 1)
bucket["uncached_prefix_kv_tokens"] = uncached
bucket["cached_prefix_kv_tokens"] = cached

return {
"schema_version": "loopx_context_policy_telemetry_v0",
"operator_counts": operator_counts,
"quiet_turn_skips": quiet_turn_skips,
"prefetch_latency_ms": prefetch_latency_ms,
"prefetch_bytes": prefetch_bytes,
"prefetch_bytes_saved": prefetch_bytes_saved,
"round_trips_saved": round_trips_saved,
"prefetch": {
"attempts": prefetch_attempts,
"latency_ms": prefetch_latency_ms,
"output_bytes": prefetch_bytes,
"bytes_saved": prefetch_bytes_saved,
"round_trips_saved": round_trips_saved,
},
"prefix_kv": {"cold": cold, "warm": warm},
}


def main(argv: list[str] | None = None) -> int:
"""Print one bounded observation for an explicitly selected rollout."""
parser = argparse.ArgumentParser(
Expand Down
31 changes: 21 additions & 10 deletions loopx/capabilities/benchmark_toolkit/experiment_board.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@
build_benchmark_factorial_contrasts,
build_benchmark_metric_delta,
)
from .token_capacity import (
CONTEXT_POLICY_EFFORT_TELEMETRY_FIELDS,
normalize_operator_counts,
)

Payload = dict[str, object]
PayloadView = Mapping[str, object]
Expand Down Expand Up @@ -204,25 +208,26 @@ def _countability(value: object) -> dict[str, bool]:
return result


def _effort(value: object) -> dict[str, float | int]:
def _effort(value: object) -> Payload:
if value in (None, {}):
return {}
if not isinstance(value, Mapping):
raise TypeError("effort must be an object")
view = _as_map(value)
int_fields = (
"duration_ms",
"agent_steps",
"goal_turns",
"token_count",
*CONTEXT_POLICY_EFFORT_TELEMETRY_FIELDS,
)
_reject_unknown_fields(
view,
allowed={
"duration_ms",
"agent_steps",
"goal_turns",
"token_count",
"estimated_cost_usd",
},
allowed={*int_fields, "estimated_cost_usd", "operator_counts"},
field="effort",
)
result: dict[str, float | int] = {}
for field in ("duration_ms", "agent_steps", "goal_turns", "token_count"):
result: Payload = {}
for field in int_fields:
if field in view:
result[field] = _non_negative_int(view[field], field=f"effort.{field}")
if "estimated_cost_usd" in view:
Expand All @@ -231,6 +236,12 @@ def _effort(value: object) -> dict[str, float | int]:
field="effort.estimated_cost_usd",
non_negative=True,
)
if "operator_counts" in view:
result["operator_counts"] = normalize_operator_counts(
view["operator_counts"],
field="effort.operator_counts",
validate_int=lambda v, f: _non_negative_int(v, field=f),
)
return result


Expand Down
2 changes: 2 additions & 0 deletions loopx/capabilities/benchmark_toolkit/study_projection.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
BENCHMARK_RUNTIME_OBSERVATION_SCHEMA_VERSION,
build_benchmark_runtime_observation,
)
from .token_capacity import aggregate_context_policy_effort

Payload = dict[str, object]
PayloadView = Mapping[str, object]
Expand Down Expand Up @@ -1007,6 +1008,7 @@ def _effort_aggregate(rows: Sequence[PayloadView]) -> Payload:
"mean": statistics.fmean(values) if values else None,
"median": statistics.median(values) if values else None,
}
output.update(aggregate_context_policy_effort(rows))
return output


Expand Down
Loading