From a75a89c88d945df6e50286c05e5c6be3d793a970 Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Sat, 21 Nov 2026 03:20:26 +0800 Subject: [PATCH] fix(benchmark): restore frozen v1 archives and align checkpoint/runtime tests with C1 defaults Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- benchmark/deepswe-gptxhigh-v1/goal_codex.py | 1 + .../local_runner.py | 1 + benchmark/runtime/RUNTIME.md | 8 ++-- benchmark/runtime/checkpoint_wake.py | 4 +- benchmark/tests/test_native_checkpoint.py | 7 +-- benchmark/tests/test_sforge_runtime.py | 2 +- benchmark/tests/test_shared_codex_runtime.py | 44 +++++++++++++++++-- benchmark/tests/test_task_entry.py | 3 +- 8 files changed, 57 insertions(+), 13 deletions(-) diff --git a/benchmark/deepswe-gptxhigh-v1/goal_codex.py b/benchmark/deepswe-gptxhigh-v1/goal_codex.py index b7ef14a187..2eaf0d99f8 100644 --- a/benchmark/deepswe-gptxhigh-v1/goal_codex.py +++ b/benchmark/deepswe-gptxhigh-v1/goal_codex.py @@ -228,6 +228,7 @@ async def exec_as_agent(self, environment, command: str = "", env=None, **kwargs # option. Packed once per run rather than once per task would be nicer # still, but the tar is ~13 MB and building it is far cheaper than the # model turn that follows. + runner_source = getattr(self, "_goal_runner_source", None) with tempfile.TemporaryDirectory() as tmp: tarball = Path(tmp) / "loopx.tar.gz" subprocess.run( diff --git a/benchmark/frontiercs2-single-host-v1/local_runner.py b/benchmark/frontiercs2-single-host-v1/local_runner.py index 6de153ec42..bf6ae9f831 100644 --- a/benchmark/frontiercs2-single-host-v1/local_runner.py +++ b/benchmark/frontiercs2-single-host-v1/local_runner.py @@ -13,6 +13,7 @@ from pathlib import Path import signal import subprocess +import sys import tarfile import time diff --git a/benchmark/runtime/RUNTIME.md b/benchmark/runtime/RUNTIME.md index a5bbd696dd..e5385db2dd 100644 --- a/benchmark/runtime/RUNTIME.md +++ b/benchmark/runtime/RUNTIME.md @@ -176,9 +176,11 @@ release remains the original typed transport failure, without publishing an optimized result or rerunning the guard. An interrupted guard continues with the original full packet; no delta is required for checkpoint recovery. Native cursor verification binds the context-directory/root option and the worker's -original `--complete-turn` and `--turn-envelope` choices. A checkpoint without -a complete-mode binding cannot verify a pending command that uses -`--complete-turn`. Historical ordinary and envelope guards remain readable. +original `--complete-turn` and `--turn-envelope` choices. Current generated +heartbeat guards use the default complete v1 contract without emitting either +alias. A checkpoint without a complete-mode binding cannot verify a pending +command that uses `--complete-turn`. Historical ordinary and envelope guards +remain readable. The existing Darwin native-checkpoint refusal remains in force. Restoring a pending replan for a fresh native conversation and checking whether diff --git a/benchmark/runtime/checkpoint_wake.py b/benchmark/runtime/checkpoint_wake.py index 127d3151fc..b28cdb3bba 100644 --- a/benchmark/runtime/checkpoint_wake.py +++ b/benchmark/runtime/checkpoint_wake.py @@ -135,8 +135,10 @@ def _capture(self, held, deadline): "registry": self.env["LOOPX_REGISTRY"], "runtime_root": self.env["LOOPX_RUNTIME_ROOT"], "goal_id": self.env["LOOPX_GOAL_ID"], "agent_id": self.env["LOOPX_AGENT_ID"], "turn_instance_id": held["quota"]["heartbeat_receipt"]["turn_instance_id"], + # Generated guards use the default complete v1 contract without emitting + # --complete-turn; bind the CLI flag explicitly so retired aliases are rejected. "turn_envelope": self.receipt.get("turn_envelope", False), - "complete_turn": self.receipt.get("complete_turn", False), + "complete_turn": False, "capture_root": str(self.capture_root), "socket_path": self.socket_path, "decision_sha256": held["sha256"]} if self.env.get("LOOPX_CODEX_CONTEXT_DIR"): diff --git a/benchmark/tests/test_native_checkpoint.py b/benchmark/tests/test_native_checkpoint.py index 0dc9528cfe..d3a1c371a9 100644 --- a/benchmark/tests/test_native_checkpoint.py +++ b/benchmark/tests/test_native_checkpoint.py @@ -1077,8 +1077,8 @@ def test_held_complete_guard_checkpoint_binds_original_settlement_mode(tmp_path, block = body.split("```sh\n", 1)[1].split("\n```", 1)[0] command = shlex.join(["/bin/bash", "-lc", block]) original = _guard_cli_arguments(command, {"cli": str(cli), "turn_instance_id": TURN_ID}) - assert "--complete-turn" in original and "--turn-envelope" not in original - observed = original if mode_matches else [word for word in original if word != "--complete-turn"] + assert "--complete-turn" not in original and "--turn-envelope" not in original + observed = original if mode_matches else [*original, "--complete-turn"] item = {"type": "commandExecution", "id": "pending-guard", "cwd": str(project), "command": shlex.join(["/bin/bash", "-lc", shlex.join([str(cli), *observed])]), "status": "failed", "exitCode": 137, "aggregatedOutput": None} @@ -1092,6 +1092,7 @@ def test_held_complete_guard_checkpoint_binds_original_settlement_mode(tmp_path, agent_id=AGENT_ID, turn_instance_id=TURN_ID, deadline=time.monotonic() + 30) as held: assert held["quota"]["ok"] is True and held["intent"]["capture"] is True + assert held["quota"]["schema_version"] == "loopx_turn_envelope_v1" quota = held["quota"]["heartbeat_receipt"] assert quota["turn_instance_id"] == TURN_ID digest = hashlib.sha256(held["path"].read_bytes()).hexdigest() @@ -1105,7 +1106,7 @@ def test_held_complete_guard_checkpoint_binds_original_settlement_mode(tmp_path, assert result["sha256"] == captured["sha256"] manifest = json.loads((point / "checkpoint.json").read_bytes())["point_metadata"] stored = manifest["native_session"]["interrupted_guard"] - assert stored["complete_turn"] is True and stored["turn_envelope"] is False + assert stored["complete_turn"] is False and stored["turn_envelope"] is False assert stored["turn_instance_id"] == TURN_ID and stored["native_call_id"] == item["id"] assert stored["decision_sha256"] == digest assert (point / "components" / manifest["held_decision_component_id"]).read_bytes() == held["path"].read_bytes() diff --git a/benchmark/tests/test_sforge_runtime.py b/benchmark/tests/test_sforge_runtime.py index 020aea7509..8c1668f02e 100644 --- a/benchmark/tests/test_sforge_runtime.py +++ b/benchmark/tests/test_sforge_runtime.py @@ -797,7 +797,7 @@ async def installed(self, environment): CODEX_THREAD_ID=event["session_id"], LOOPX_PHASE_DEADLINE_EPOCH="9876543210.125") body = heartbeat_body(env, turn, native_goal=False) block = body.split("```sh\n", 1)[1].split("\n```", 1)[0] - assert block.count("--complete-turn") == 1 and "--turn-envelope" not in block + assert "--complete-turn" not in block and "--turn-envelope" not in block if checkpoint_preflight: from benchmark.runtime.checkpoint import capture_before_replan diff --git a/benchmark/tests/test_shared_codex_runtime.py b/benchmark/tests/test_shared_codex_runtime.py index c229f26f00..cdca15e337 100644 --- a/benchmark/tests/test_shared_codex_runtime.py +++ b/benchmark/tests/test_shared_codex_runtime.py @@ -285,14 +285,51 @@ def test_process_failure_is_not_reported_as_success(tmp_path): assert receipt["return_code"] == 7 +def _sqlite_heartbeat_env(tmp_path, monkeypatch): + from loopx.global_registry import sync_project_registry_to_global + from tests.control_plane import test_quota_settlement_cli as fixture + from tests.control_plane.canonical_authority_fixture import isolate_sqlite_runtime + from tests.control_plane.host_process_fixture import write_python_host_fixture + + isolate_sqlite_runtime(tmp_path, monkeypatch) + monkeypatch.setenv("LOOPX_USAGE_PING", "0") + monkeypatch.setenv("PYTHONPATH", str(Path(__file__).resolve().parents[2])) + project, runtime, source = fixture._write_fixture(tmp_path) + fixture._initialize_fixture_authority(project, runtime, provider="sqlite") + assert sync_project_registry_to_global( + registry_path=source, runtime_root_override=str(runtime), goal_id=fixture.GOAL_ID + )["ok"] + env = worker_env(project) + skills = tmp_path / "shared-skills" + skills.mkdir() + env.update( + LOOPX_EXECUTION_MODE="heartbeat", + LOOPX_ITERATION_CONTEXT="resume", + LOOPX_TASK_ENTRY="seeded-todo", + LOOPX_TASK_STAGE="execute", + LOOPX_RUNTIME_ROOT=str(runtime), + LOOPX_REGISTRY=str(runtime / "registry.global.json"), + LOOPX_GOAL_ID=fixture.GOAL_ID, + LOOPX_AGENT_ID=fixture.AGENT_ID, + LOOPX_SHARED_SKILLS=str(skills), + LOOPX_CODEX_TURN_TIMEOUT_SEC="90", + ) + env["LOOPX_CLI"] = str( + write_python_host_fixture( + tmp_path / "loopx-cli", + f"#!{sys.executable}\nfrom loopx.cli import main\nraise SystemExit(main())\n", + ) + ) + return env + + @pytest.mark.parametrize("code,expected", [("insufficient_quota", "quota_exhausted"), ("model_at_capacity", "provider_capacity"), ("unrecognized", "unknown")]) def test_heartbeat_real_native_process_preserves_content_free_host_failure(tmp_path, monkeypatch, code, expected): import benchmark.runtime.worker as worker - from tests.control_plane.test_benchmark_current_admission import _environment - env, _project, _runtime, _source, _state = _environment(tmp_path, monkeypatch, "sqlite") + env = _sqlite_heartbeat_env(tmp_path, monkeypatch) monkeypatch.setattr(worker, "heartbeat_body", lambda *a, **kw: "synthetic work") event = {"type": "turn.failed", "error": {"code": code, "message": "PRIVATE-SENTINEL"}} Path(env["CODEX_BIN"]).write_text( @@ -330,10 +367,9 @@ def observe(self, path): @pytest.mark.parametrize("context", ["fresh", "resume"]) def test_native_heartbeat_retry_uses_real_session_owner(tmp_path, monkeypatch, context): - from tests.control_plane.test_benchmark_current_admission import _environment import benchmark.runtime.worker as worker - env, _project, _runtime, _source, _state = _environment(tmp_path, monkeypatch, "sqlite") + env = _sqlite_heartbeat_env(tmp_path, monkeypatch) env["LOOPX_ITERATION_CONTEXT"] = context # This fake executable models session retries, not native hook discovery. env["LOOPX_CONTEXT_DELIVERY"] = "off" diff --git a/benchmark/tests/test_task_entry.py b/benchmark/tests/test_task_entry.py index 0be9d0f086..7db2a9ba09 100644 --- a/benchmark/tests/test_task_entry.py +++ b/benchmark/tests/test_task_entry.py @@ -1239,7 +1239,8 @@ def test_execute_wake_keeps_host_clocks_out_of_fresh_and_resumed_input( assert receipt["session"]["action"] == ("resume" if context == "resume" else "start_new") assert "turn_envelope" not in receipt assert receipt.get("complete_turn") is (True if complete else None) - assert ("--complete-turn" in body) is complete + assert "--complete-turn" not in body and "--turn-envelope" not in body + assert ("--decision-output-root" in body) is (complete or checkpoint_preflight) if checkpoint_preflight: original = Path(env["LOOPX_WAKE_LOG_DIR"]) / receipt["turn_id"] / "checkpoint-start-guard.json" assert json.loads(original.read_bytes())["schema_version"] == "loopx_turn_envelope_v1"