Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions benchmark/edgebench/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,14 @@ def main(argv=None):
help="Explicit complete v1 Turn contract with shared exact-Turn settlement; heartbeat workers only")
parser.add_argument("--context-delivery", choices=("off", "on"),
help="Native Codex context delivery: on by default for heartbeat workers; pass off to disable")
parser.add_argument("--compact-direct-read", action="store_true",
help="Opt-in Phase-2 lock-free compact hook direct-read (--compact-direct-read in hooks.json)")
parser.add_argument("--quiet-turn", action="store_true",
help="Opt-in Quiet-Turn skip metadata evaluation for supported heartbeat/turn workers")
parser.add_argument("--epoch-prefetch-json", type=json.loads, default=None,
help="Opt-in Stage-then-Swap EpochReset JSON configuration for supported heartbeat/turn workers")
parser.add_argument("--execution-yardstick-enabled", action=argparse.BooleanOptionalAction, default=None,
help="Opt-in Yardstick measurement review toggle for configure-goal (including WarmStart)")
parser.add_argument("--execution-observation", choices=("local_probe",),
help="Optional local execution probe for heartbeat workers")
parser.add_argument("--native-result-comparison", action="store_true",
Expand Down Expand Up @@ -216,6 +224,9 @@ def main(argv=None):
context_delivery=args.context_delivery, turn_envelope=args.turn_envelope,
complete_turn=args.complete_turn, validation_command=args.validation_command_json,
native_child_limit=args.native_child_limit,
compact_direct_read=args.compact_direct_read,
quiet_turn=args.quiet_turn,
epoch_prefetch_json=args.epoch_prefetch_json,
)
except ValueError as error:
parser.error(str(error))
Expand Down Expand Up @@ -275,6 +286,8 @@ def main(argv=None):
parser.error("--stride-strategy requires a heartbeat worker or turn-resume")
if args.planning_horizon_goal is not None and not execution.uses_loopx:
parser.error("--planning-horizon-goal requires a heartbeat worker or turn-resume")
if args.execution_yardstick_enabled is not None and not execution.uses_loopx:
parser.error("--execution-yardstick-enabled requires a heartbeat worker or turn-resume")
checkpoint_request = None
if args.checkpoint_request is not None:
if not args.worker.startswith("heartbeat-"):
Expand Down Expand Up @@ -464,6 +477,12 @@ def main(argv=None):
validation_command=execution.validation_command,
turn_envelope=args.turn_envelope,
**({"complete_turn": True} if args.complete_turn else {}),
**({"compact_direct_read": True} if args.compact_direct_read else {}),
**({"quiet_turn": True} if args.quiet_turn else {}),
**({"epoch_prefetch_json": execution.epoch_prefetch_json}
if execution.epoch_prefetch_json is not None else {}),
**({"execution_yardstick_enabled": args.execution_yardstick_enabled}
if args.execution_yardstick_enabled is not None else {}),
context_delivery=args.context_delivery,
**({"execution_observation": args.execution_observation}
if args.execution_observation is not None else {}),
Expand Down
77 changes: 72 additions & 5 deletions benchmark/runtime/codex.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,10 @@
import json
import math
import os
from collections.abc import Mapping
from dataclasses import dataclass
from pathlib import Path
from typing import Any


# New benchmark runs and the product both default to six settled work Turns.
Expand All @@ -20,6 +22,34 @@
SANDBOXES = ("read-only", "workspace-write", "danger-full-access")


def normalize_epoch_prefetch_json(value: Any) -> str | None:
"""Normalize optional Stage-then-Swap EpochReset config to canonical compact JSON."""
if value is None or value == "":
return None
if isinstance(value, Mapping):
parsed = dict(value)
elif isinstance(value, str):
try:
decoded = json.loads(value)
except json.JSONDecodeError as exc:
raise ValueError("epoch_prefetch_json must be valid JSON") from exc
if not isinstance(decoded, dict):
raise ValueError("epoch_prefetch_json must decode to a JSON object")
parsed = decoded
else:
raise ValueError("epoch_prefetch_json must be a JSON string or mapping")
return json.dumps(parsed, ensure_ascii=False, sort_keys=True, separators=(",", ":"))


def epoch_prefetch_enabled(value: Any) -> bool:
"""Return True iff normalized epoch_prefetch JSON explicitly enables the treatment."""
normalized = normalize_epoch_prefetch_json(value)
if normalized is None:
return False
decoded = json.loads(normalized)
return isinstance(decoded, dict) and decoded.get("enabled") is True


@dataclass(frozen=True)
class Execution:
mode: str = "heartbeat"
Expand All @@ -32,6 +62,9 @@ class Execution:
context_delivery: str | None = None
native_child_limit: int | None = None
complete_turn: bool = False
compact_direct_read: bool = False
quiet_turn: bool = False
epoch_prefetch_json: str | None = None

def __post_init__(self) -> None:
if self.native_child_limit is not None and (
Expand All @@ -48,6 +81,22 @@ def __post_init__(self) -> None:
raise ValueError("turn_envelope requires heartbeat execution and a boolean opt-in")
if type(self.complete_turn) is not bool or (self.complete_turn and (self.mode != "heartbeat" or self.turn_envelope)):
raise ValueError("complete_turn requires heartbeat execution without legacy turn_envelope")
if type(self.compact_direct_read) is not bool or (
self.compact_direct_read and (self.mode != "heartbeat" or self.context_delivery != "on")
):
raise ValueError("compact_direct_read requires heartbeat execution with context_delivery=on")
if type(self.quiet_turn) is not bool or (
self.quiet_turn and self.mode not in {"heartbeat", "turn"}
):
raise ValueError("quiet_turn requires heartbeat or turn execution and a boolean opt-in")
if self.quiet_turn and self.mode == "heartbeat" and self.context_delivery != "on":
raise ValueError("quiet_turn in heartbeat mode requires context_delivery=on")
normalized_prefetch = normalize_epoch_prefetch_json(self.epoch_prefetch_json)
object.__setattr__(self, "epoch_prefetch_json", normalized_prefetch)
if normalized_prefetch is not None and self.mode not in {"heartbeat", "turn"}:
raise ValueError("epoch_prefetch_json requires heartbeat or turn execution")
if normalized_prefetch is not None and self.mode == "heartbeat" and self.context_delivery != "on":
raise ValueError("epoch_prefetch_json in heartbeat mode requires context_delivery=on")
if self.mode not in MODES or self.context not in CONTEXTS:
raise ValueError("unsupported execution mode or iteration context")
if self.task_entry is None:
Expand Down Expand Up @@ -89,22 +138,40 @@ def native_goal(self) -> bool:
return self.mode in {"native-goal", "loopx-goal"}


def turn_envelope_receipt(*, explicit: bool, complete: bool = False) -> dict:
def turn_envelope_receipt(*, explicit: bool, complete: bool = False) -> dict[str, Any]:
"""Record only the explicit display opt-in; checkpoint capture is independent."""
return {**({"turn_envelope": True, "turn_envelope_source": "explicit"} if explicit else {}),
**({"complete_turn": True, "complete_turn_source": "explicit"} if complete else {})}


def explicit_turn_display(env: dict[str, str]) -> Execution:
def explicit_turn_display(env: Mapping[str, str]) -> Execution:
"""Decode the shared explicit display request through its Execution owner."""
selected = {}
selected: dict[str, bool] = {}
for variable, field in (("LOOPX_TURN_ENVELOPE", "turn_envelope"),
("LOOPX_COMPLETE_TURN", "complete_turn")):
value = env.get(variable, "0")
if value not in {"0", "1"}:
raise ValueError(f"{variable} must be 0 or 1")
selected[field] = value == "1"
return Execution(**selected)
return Execution(
turn_envelope=selected["turn_envelope"],
complete_turn=selected["complete_turn"],
)


def explicit_context_opt_ins(env: Mapping[str, str]) -> dict[str, Any]:
"""Decode strictly default-off hook/Turn optimization opt-ins from worker env."""
parsed: dict[str, Any] = {}
for variable, field in (("LOOPX_COMPACT_DIRECT_READ", "compact_direct_read"),
("LOOPX_QUIET_TURN", "quiet_turn")):
value = env.get(variable, "0")
if value not in {"0", "1"}:
raise ValueError(f"{variable} must be 0 or 1")
parsed[field] = value == "1"
parsed["epoch_prefetch_json"] = normalize_epoch_prefetch_json(
env.get("LOOPX_EPOCH_PREFETCH_JSON")
)
return parsed


def native_child_instructions(limit: int | None) -> str:
Expand All @@ -126,7 +193,7 @@ def native_child_instructions(limit: int | None) -> str:
"and close child work before the parent finishes.\n"
)

def validate_model_catalog(model_catalog: Path, model: str) -> dict:
def validate_model_catalog(model_catalog: Path, model: str) -> dict[str, Any]:
"""Admit only catalog entries that keep Codex's function-tool surface.

Real catalog entries are not capability-only. ``tool_mode`` selects how
Expand Down
35 changes: 34 additions & 1 deletion benchmark/runtime/context_delivery.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,19 @@
"""
from __future__ import annotations

import inspect
import json
import os
import stat
import shlex
from pathlib import Path
from typing import Any

from .codex import explicit_context_opt_ins


class ContextOptInConfigurationError(ValueError):
"""Explicit opt-in hook parameter failed validation or product compatibility."""


def preflight_context_instruction(env: dict[str, str], path: Path, digest: str, turn_id: str, *,
Expand Down Expand Up @@ -52,8 +60,33 @@ def prepare_context_delivery(env: dict[str, str], home: Path) -> None:
if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid() or info.st_mode & 0o077:
raise ValueError("context delivery requires a private, physical trial directory")
env["LOOPX_CODEX_CONTEXT_DIR"] = str(root)
try:
opt_ins = explicit_context_opt_ins(env)
except ValueError as exc:
raise ContextOptInConfigurationError(str(exc)) from exc
params = inspect.signature(codex_context_configuration).parameters
extra_config: dict[str, Any] = {}
if opt_ins["compact_direct_read"]:
if "compact_direct_read" not in params:
raise ContextOptInConfigurationError(
"compact_direct_read requires a LoopX product release supporting --compact-direct-read"
)
extra_config["compact_direct_read"] = True
if opt_ins["quiet_turn"]:
if "quiet_turn" not in params:
raise ContextOptInConfigurationError(
"quiet_turn requires a LoopX product release supporting --quiet-turn on context-config/hook"
)
extra_config["quiet_turn"] = True
if opt_ins["epoch_prefetch_json"] is not None:
if "epoch_prefetch" not in params:
raise ContextOptInConfigurationError(
"epoch_prefetch_json requires a LoopX product release supporting --epoch-prefetch-json on context-config/hook"
)
extra_config["epoch_prefetch"] = json.loads(opt_ins["epoch_prefetch_json"])
configuration = codex_context_configuration(str(root), goal_id=env["LOOPX_GOAL_ID"],
agent_id=env["LOOPX_AGENT_ID"], scenario="codex_benchmark_heartbeat", cli_argv=[env["LOOPX_CLI"], "--cli-bin", env["LOOPX_CLI"]])
agent_id=env["LOOPX_AGENT_ID"], scenario="codex_benchmark_heartbeat", cli_argv=[env["LOOPX_CLI"], "--cli-bin", env["LOOPX_CLI"]],
**extra_config)
content = json.dumps({"hooks": configuration["hooks"]})
path = home / "hooks.json"
if path.is_symlink() or (path.exists() and path.read_text() != content):
Expand Down
Loading
Loading