Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions benchmark/runtime/RUNTIME.md
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,30 @@ agents:
replan_after_turns: 6
```

### Official-style Codex Stop hook

Set `execution_mode: plain` and `official_stop_hook: true` in
`BenchmarkCodex` kwargs for a native Codex baseline. One
`codex exec` owns the conversation. On attempted completion, the native Stop
hook asks Codex to continue working, using the same instruction as EdgeBench's
SForge official profile. The worker uses the remaining phase clock through
native execution. The task controller may stop it earlier when the task's
original final submission is accepted.

The runner stages only its own hook in the trial's Codex home and trusts the
exact handler identity discovered by that Codex binary. Both discovery requests
share the worker deadline when one is declared. With no worker or phase deadline,
discovery retains its bounded RPC timeouts while model work has no independent
cutoff; the task controller still owns submission and cancellation. If the phase expires during discovery, the wake
records budget exhaustion and no model launch. Earlier discovery failures
still fail the trial before inference. Install and trajectory receipts record
the opt-in. The wake receipt records successful hook discovery.
Use the same pinned task, model, reasoning effort, time and evaluator in
comparison arms. The benchmark adapter still owns feedback, final submission,
and scoring. EdgeBench's SForge auto-resume remains a provider-specific part of
its official profile. Use a fresh trial with the option omitted for a
single-pass Codex baseline.

Historical Short trials request `turn_envelope: true` in heartbeat `kwargs`.
Before planning or execution starts a model, the worker previews the selected
product's quota response through the same CLI. Its wake receipt records the
Expand Down
4 changes: 4 additions & 0 deletions benchmark/runtime/codex.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ class Execution:
context_delivery: str | None = None
native_child_limit: int | None = None
complete_turn: bool = False
official_stop_hook: bool = False

def __post_init__(self) -> None:
if self.native_child_limit is not None and (
Expand All @@ -48,6 +49,8 @@ def __post_init__(self) -> None:
raise ValueError("turn_envelope requires heartbeat execution and a boolean opt-in")
if type(self.complete_turn) is not bool or (self.complete_turn and (self.mode != "heartbeat" or self.turn_envelope)):
raise ValueError("complete_turn requires heartbeat execution without legacy turn_envelope")
if type(self.official_stop_hook) is not bool or (self.official_stop_hook and self.mode != "plain"):
raise ValueError("official_stop_hook requires plain execution and a boolean opt-in")
if self.mode not in MODES or self.context not in CONTEXTS:
raise ValueError("unsupported execution mode or iteration context")
if self.task_entry is None:
Expand Down Expand Up @@ -208,6 +211,7 @@ def codex_settings(
"[features]",
f"goals = {str(execution.native_goal).lower()}",
"unified_exec = true",
*(["hooks = true"] if execution.official_stop_hook else []),
*(["[agents]", f"max_concurrent_threads_per_session = {execution.native_child_limit}"]
if execution.native_child_limit is not None else []),
"[memories]",
Expand Down
56 changes: 4 additions & 52 deletions benchmark/runtime/context_delivery.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@
import shlex
from pathlib import Path

from .native_hooks import prepare_native_hooks


def preflight_context_instruction(env: dict[str, str], path: Path, digest: str, turn_id: str, *,
guide_in_host: bool = False) -> str:
Expand Down Expand Up @@ -44,7 +46,6 @@ def preflight_context_instruction(env: dict[str, str], path: Path, digest: str,

def prepare_context_delivery(env: dict[str, str], home: Path) -> None:
from loopx.control_plane.turn_driver.codex_context import codex_context_configuration
from loopx.control_plane.turn_driver.codex_stdio import StdioNativeGoalTransport

root = home / "loopx-context"
root.mkdir(mode=0o700, exist_ok=True)
Expand All @@ -54,59 +55,10 @@ def prepare_context_delivery(env: dict[str, str], home: Path) -> None:
env["LOOPX_CODEX_CONTEXT_DIR"] = str(root)
configuration = codex_context_configuration(str(root), goal_id=env["LOOPX_GOAL_ID"],
agent_id=env["LOOPX_AGENT_ID"], scenario="codex_benchmark_heartbeat", cli_argv=[env["LOOPX_CLI"], "--cli-bin", env["LOOPX_CLI"]])
content = json.dumps({"hooks": configuration["hooks"]})
path = home / "hooks.json"
if path.is_symlink() or (path.exists() and path.read_text() != content):
raise ValueError("native context hooks changed within a trial; use a new trial")
if not path.exists():
path.write_text(content)
path.chmod(0o600)
# The native host is the hash/key owner. Do not duplicate its hashing or
# bypass trust for unrelated workspace hooks. Discovery launches no model.
with (home / "context-hooks-stderr.log").open("a") as stderr:
with StdioNativeGoalTransport.spawn(
# This process only reads native hook identities. Catalog/plugin
# startup can clone repositories into an otherwise private home;
# it is unrelated to discovery and must not affect solver config.
[env["CODEX_BIN"], "-c", "features.plugins=false",
"-c", "features.remote_plugin=false", "-c", "features.plugin_sharing=false",
"app-server", "--listen", "stdio://"],
cwd=env["LOOPX_PROJECT"], env=env, stderr=stderr, response_timeout_sec=10,
) as native:
native.request("initialize", {"clientInfo": {"name": "loopx_context_transport", "version": "0.1.0"},
"capabilities": {"experimentalApi": True}})
native.notify("initialized", {})
listed = native.request("hooks/list", {"cwds": [env["LOOPX_PROJECT"]]})
hooks = [hook for entry in listed["data"] for hook in entry["hooks"]
if hook.get("sourcePath") == str(path.resolve())]
# Match the generated handler multiset, not just its size. Native discovery
# can reorder it; duplicates, disabled handlers and substitutions cannot
# stand in for an expected event/command/settings combination.
expected = [{"eventName": event[0].lower() + event[1:],
"handlerType": handler["type"], "command": handler["command"],
"matcher": group.get("matcher"), "timeoutSec": handler.get("timeout"),
"async": handler.get("async", False)}
for event, groups in configuration["hooks"].items()
for group in groups for handler in group["hooks"]]
if not expected or len(hooks) != len(expected) or path.read_text() != content:
raise ValueError("native host did not discover the exact trial context hooks")
entries, keys = [], set()
for hook in hooks:
actual = {field: hook.get(field) for field in expected[0]}
if hook.get("enabled") is not True or type(hook.get("async")) is not bool or actual not in expected:
raise ValueError("native host did not discover the exact trial context hooks")
expected.remove(actual)
key, digest = hook.get("key"), hook.get("currentHash")
if not isinstance(key, str) or not key or key in keys or not isinstance(digest, str) or not digest:
raise ValueError("native host returned no unique context-hook trust identity")
keys.add(key)
entries.append(f"{json.dumps(key)} = {{ trusted_hash = {json.dumps(digest)} }}")
# CLI override keys are dot-separated; native keys contain paths/dots.
# Carry the entire table as TOML so native keys remain literal strings.
native_args = prepare_native_hooks(env, home, configuration["hooks"], purpose="context")
# This isolated trial's native developer instructions survive compaction;
# a failed SessionStart handler cannot erase the read-only recovery route.
native_args = ["-c", "hooks.state={" + ",".join(entries) + "}",
"-c", "developer_instructions=" + json.dumps(configuration["developer_instructions"])]
native_args += ["-c", "developer_instructions=" + json.dumps(configuration["developer_instructions"])]
native_args += [argument for key, value in configuration["config_overrides"].items()
for argument in ("-c", key + "=" + json.dumps(value))]
env["LOOPX_CODEX_CONTEXT_TRUST_ARGS"] = json.dumps(native_args)
12 changes: 9 additions & 3 deletions benchmark/runtime/harbor.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ def __init__(
agent_id=_AGENT_ID,
defer_codex_home=False,
complete_turn=False,
official_stop_hook=False,
**kwargs,
):
if "research_loop" in kwargs:
Expand All @@ -105,6 +106,7 @@ def __init__(
context_delivery,
native_child_limit,
complete_turn=complete_turn,
official_stop_hook=official_stop_hook,
)
if type(provider_request_admission) is not bool:
raise ValueError("provider_request_admission must be a boolean opt-in")
Expand Down Expand Up @@ -414,6 +416,7 @@ async def install(self, environment: BaseEnvironment) -> None:
if self.execution_observation is not None else {}),
**self._turn_envelope_receipt(),
**self._planning_receipt(),
**({"official_stop_hook": True} if self.execution.official_stop_hook else {}),
**({"provider_request_admission": "on"} if self.provider_request_admission else {}),
"home_scope": "trial",
"loopx_usage_ping_enabled": False,
Expand Down Expand Up @@ -695,6 +698,7 @@ def _worker_env(self, *, cwd: str) -> dict[str, str]:
**({"LOOPX_TURN_ENVELOPE": "1"} if self.execution.turn_envelope else {}),
"LOOPX_COMPLETE_TURN": "1" if self.execution.complete_turn else "0",
"LOOPX_CONTEXT_DELIVERY": self.execution.context_delivery,
"LOOPX_OFFICIAL_STOP_HOOK": "1" if self.execution.official_stop_hook else "0",
# A trial's explicit None must not inherit a container opt-in.
"LOOPX_NATIVE_CHILD_LIMIT": (
str(self.execution.native_child_limit)
Expand Down Expand Up @@ -831,6 +835,7 @@ def token_delta(key: str) -> int | None:
"task_entry": self.execution.task_entry,
**self._turn_envelope_receipt(),
**self._planning_receipt(),
**({"official_stop_hook": True} if self.execution.official_stop_hook else {}),
**({"provider_request_admission": "on"} if self.provider_request_admission else {}),
"home_scope": "trial",
**self._replan_receipt(),
Expand Down Expand Up @@ -897,7 +902,7 @@ async def run(
if entry["status"] == "blocked":
return
remaining = int(deadline - time.monotonic())
reserve = phase_work_reserve(self.execution.mode)
reserve = phase_work_reserve(self.execution.mode, official_stop_hook=self.execution.official_stop_hook)
if remaining <= reserve:
raise TimeoutError("Task budget exhausted before execution handoff")
# Planning, admission and work share one absolute phase clock.
Expand All @@ -913,11 +918,12 @@ async def run(
resume_budget_stop=self._phase_number > 1,
)
phase_log = f"/logs/agent/worker-phase-{self._phase_number:03d}.log"
supervisor_timeout = remaining + (30 if self.execution.official_stop_hook else 0)
shell = (
"set +e; "
# Use the task environment's clock, including remote backends.
f"export LOOPX_PHASE_DEADLINE_EPOCH=$(( $(date +%s) + {remaining} )); "
f"timeout --signal=TERM --kill-after=30 {remaining}s "
f"timeout --signal=TERM --kill-after=30 {supervisor_timeout}s "
f"{shlex.join(command)} >> {shlex.quote(phase_log)} 2>&1; "
"rc=$?; "
# Budget exhaustion retains partial artifacts for native scoring.
Expand All @@ -928,7 +934,7 @@ async def run(
command=shell,
env=env,
cwd=cwd,
timeout_sec=remaining + 60,
timeout_sec=supervisor_timeout + 60,
)
finally:
# Remote Harbor backends download logs after run(). Read them now
Expand Down
106 changes: 106 additions & 0 deletions benchmark/runtime/native_hooks.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
"""Stage and trust only the benchmark's generated native Codex hooks."""

from __future__ import annotations

import inspect
import json
import shlex
import subprocess
import time
from collections.abc import Mapping
from functools import partial
from pathlib import Path
from typing import Any


OFFICIAL_STOP_HOOK_REASON = "Do not stop. Continue working on the implementation."


def prepare_official_stop_hook(env: dict[str, str], home: Path, *, deadline: float | None = None) -> list[str]:
# Put the entire handler in the declaration so native trust covers the
# command body as well as the path to the hook declaration.
response = shlex.quote(json.dumps({"decision": "block", "reason": OFFICIAL_STOP_HOOK_REASON}))
command = rf"cat >/dev/null && printf '%s\n' {response}"
return prepare_native_hooks(env, home, {
"Stop": [{"hooks": [{"type": "command", "command": command, "timeout": 30}]}],
}, purpose="official-stop", deadline=deadline)


def prepare_native_hooks(env: dict[str, str], home: Path, configuration: dict, *,
purpose: str, deadline: float | None = None) -> list[str]:
from loopx.control_plane.turn_driver.codex_stdio import NativeGoalProtocolError, StdioNativeGoalTransport

content = json.dumps({"hooks": configuration})
path = home / "hooks.json"
if path.is_symlink() or (path.exists() and path.read_text() != content):
raise ValueError(f"native {purpose} hooks changed within a trial; use a new trial")
if not path.exists():
path.write_text(content)
path.chmod(0o600)
command = [
env["CODEX_BIN"], "-c", "features.plugins=false",
"-c", "features.remote_plugin=false", "-c", "features.plugin_sharing=false",
"app-server", "--listen", "stdio://",
]
if deadline is not None and deadline <= time.monotonic():
raise subprocess.TimeoutExpired(command, 0)
# The native host owns keys and hashes. Discovery launches no model, and
# discovery-only plugin overrides leave the solver configuration intact.
with (home / f"{purpose}-hooks-stderr.log").open("a") as stderr:
with StdioNativeGoalTransport.spawn(
command, cwd=env["LOOPX_PROJECT"], env=env, stderr=stderr, response_timeout_sec=10,
) as native:
supports_deadline = "deadline" in inspect.signature(native.request).parameters
if deadline is None:
request = native.request
elif supports_deadline:
request = partial(native.request, deadline=deadline)
else:
def request(method: str, params: Mapping[str, Any]) -> Mapping[str, Any]:
started = time.monotonic()
remaining = deadline - started
if remaining <= 0:
raise subprocess.TimeoutExpired(command, 0)
rpc_timeout = 10.0
caller_limited = remaining <= rpc_timeout
# Split-pinned product releases expose the 2-arg transport;
# cap per-RPC waiting and translate caller-budget timeouts uniformly.
native.response_timeout_sec = remaining if caller_limited else rpc_timeout
try:
return native.request(method, params)
except NativeGoalProtocolError as exc:
if caller_limited and str(exc) == f"app_server_response_timeout:{method}":
raise subprocess.TimeoutExpired(command, remaining) from exc
raise

request("initialize", {"clientInfo": {"name": "loopx_context_transport", "version": "0.1.0"},
"capabilities": {"experimentalApi": True}})
native.notify("initialized", {})
listed = request("hooks/list", {"cwds": [env["LOOPX_PROJECT"]]})
hooks = [hook for entry in listed["data"] for hook in entry["hooks"]
if hook.get("sourcePath") == str(path.resolve())]
# Discovery can reorder handlers. Counts alone cannot rule out
# duplicates, disabled handlers or substitutions.
expected = [{"eventName": event[0].lower() + event[1:],
"handlerType": handler["type"], "command": handler["command"],
"matcher": group.get("matcher"), "timeoutSec": handler.get("timeout"),
"async": handler.get("async", False)}
for event, groups in configuration.items()
for group in groups for handler in group["hooks"]]
mismatch = f"native host did not discover the exact trial {purpose} hooks"
if not expected or len(hooks) != len(expected) or path.read_text() != content:
raise ValueError(mismatch)
entries, keys = [], set()
for hook in hooks:
actual = {field: hook.get(field) for field in expected[0]}
if hook.get("enabled") is not True or type(hook.get("async")) is not bool or actual not in expected:
raise ValueError(mismatch)
expected.remove(actual)
key, digest = hook.get("key"), hook.get("currentHash")
if not isinstance(key, str) or not key or key in keys or not isinstance(digest, str) or not digest:
raise ValueError(f"native host returned no unique {purpose}-hook trust identity")
keys.add(key)
entries.append(f"{json.dumps(key)} = {{ trusted_hash = {json.dumps(digest)} }}")
# Native keys contain dots and paths. Pass the whole table with literal
# keys instead of nesting keys via CLI dot notation.
return ["-c", "hooks.state={" + ",".join(entries) + "}"]
6 changes: 3 additions & 3 deletions benchmark/runtime/scheduler.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@
PHASE_WORK_RESERVE_SECONDS = 160


def phase_work_reserve(mode: str) -> int:
"""Heartbeat cleanup must not consume the remaining solver allowance."""
return 0 if mode == "heartbeat" else PHASE_WORK_RESERVE_SECONDS
def phase_work_reserve(mode: str, *, official_stop_hook: bool = False) -> int:
"""Give deadline-driven workers the full phase; bound cleanup separately."""
return 0 if mode == "heartbeat" or official_stop_hook else PHASE_WORK_RESERVE_SECONDS


def read_phase_deadline(raw: bytes) -> int | float:
Expand Down
Loading
Loading