From 636de77c1379283c1a0446ec15eea433d9f6284e Mon Sep 17 00:00:00 2001 From: rica-v3 Date: Mon, 27 Jul 2026 21:41:16 +0900 Subject: [PATCH 1/7] execution_policy.py: bound measurable benchmark calls --- runtime/base_runtime.py | 29 +- runtime/codex_runner.py | 346 ++++++++++++++++++- runtime/execution_policy.py | 432 ++++++++++++++++++++++++ runtime/internal/goal_sourcing.py | 3 + runtime/internal/intent_parser.py | 3 + runtime/model_telemetry.py | 171 +++++++++- runtime/research_runtime.py | 59 ++++ tests/test_execution_policy.py | 163 +++++++++ tests/test_model_telemetry.py | 317 ++++++++++++++++- workflows/orchestration/team_service.py | 30 +- workflows/sprints/lifecycle.py | 32 ++ 11 files changed, 1550 insertions(+), 35 deletions(-) create mode 100644 runtime/execution_policy.py create mode 100644 tests/test_execution_policy.py diff --git a/runtime/base_runtime.py b/runtime/base_runtime.py index bdb0670..020151e 100644 --- a/runtime/base_runtime.py +++ b/runtime/base_runtime.py @@ -8,6 +8,7 @@ from teams_runtime.shared.paths import RuntimePaths from teams_runtime.runtime.codex_runner import CodexRunner, extract_json_object +from teams_runtime.runtime.execution_policy import ModelExecutionPolicy from teams_runtime.runtime.identities import service_runtime_identity from teams_runtime.runtime.model_telemetry import ( InvocationSequence, @@ -39,6 +40,7 @@ TelemetryRuntimeConfig, ) from teams_runtime.shared.prompt_context import ( + PROMPT_EVENT_SELECTION_POLICY, PromptRequestProjection, project_request_record_for_prompt, render_prompt_event_history_notice, @@ -273,6 +275,7 @@ def __init__( session_identity: str | None = None, telemetry_config: TelemetryRuntimeConfig | None = None, prompt_context_config: PromptContextRuntimeConfig | None = None, + execution_policy: ModelExecutionPolicy | None = None, ): self.paths = paths self.role = role @@ -295,6 +298,7 @@ def __init__( runtime_config, role=role, telemetry_recorder=self.telemetry_recorder, + execution_policy=execution_policy, ) self.runtime_config = runtime_config self._run_lock = threading.Lock() @@ -362,7 +366,11 @@ def _request_requires_default_bypass( envelope: MessageEnvelope, request_record: RequestRecord, ) -> bool: - return True + return not self._benchmark_execution_enabled() + + def _benchmark_execution_enabled(self) -> bool: + execution_policy = getattr(self.codex_runner, "execution_policy", None) + return bool(getattr(execution_policy, "benchmark_mode", False)) def run_task( self, @@ -383,10 +391,20 @@ def run_task( ) session_manager = self._session_manager_for_sprint(current_sprint_id) state = session_manager.ensure_session() + request_projection = self._project_request_for_prompt( + request_record, + purpose=telemetry_purpose, + ) + invocation_sequence.set_prompt_context_projection( + request_projection, + enabled=self.prompt_context_config.enabled, + selection_policy=PROMPT_EVENT_SELECTION_POLICY, + ) prompt = self._build_prompt( envelope, request_record, current_sprint_id=current_sprint_id, + request_projection=request_projection, ) force_fresh_role_session = ( bool((envelope.params or {}).get("_repair_invalid_role_payload_on_resume")) @@ -437,7 +455,11 @@ def run_task( ) active_session_id = resolved_session_id or active_session_id payload = self._parse_role_output(output, request_record) - if not default_bypass and self._should_retry_with_bypass(payload): + if ( + not self._benchmark_execution_enabled() + and not default_bypass + and self._should_retry_with_bypass(payload) + ): retry_session_id = None if active_session_id else active_session_id LOGGER.warning( "[%s] sandbox_denial_detected retrying_with_bypass request_id=%s sprint_id=%s todo_id=%s backlog_id=%s workspace=%s session_id=%s retry_session_mode=%s", @@ -772,8 +794,9 @@ def _build_prompt( request_record: RequestRecord, *, current_sprint_id: str | None = None, + request_projection: PromptRequestProjection | None = None, ) -> str: - request_projection = self._project_request_for_prompt( + request_projection = request_projection or self._project_request_for_prompt( request_record, purpose="role_task", ) diff --git a/runtime/codex_runner.py b/runtime/codex_runner.py index c411f67..0edec83 100644 --- a/runtime/codex_runner.py +++ b/runtime/codex_runner.py @@ -4,12 +4,20 @@ import logging import os import re +import signal import subprocess import time from datetime import datetime from pathlib import Path from typing import Any +from teams_runtime.runtime.execution_policy import ( + DEFAULT_MODEL_EXECUTION_POLICY, + InvocationReservation, + ModelExecutionPolicy, + ModelExecutionPolicyViolation, + ModelInvocationTimeout, +) from teams_runtime.runtime.model_telemetry import ( ModelInvocationContext, ModelTelemetryRecorder, @@ -22,6 +30,51 @@ SESSION_ID_PATTERN = re.compile(r"session id:\s*([0-9a-fA-F-]+)", re.IGNORECASE) LOGGER = logging.getLogger(__name__) +CODEX_TOOL_ITEM_TYPES = frozenset( + { + "command_execution", + "file_change", + "mcp_tool_call", + "web_search", + } +) +BENCHMARK_DISABLED_CODEX_FEATURES = ( + "apps", + "browser_use", + "browser_use_external", + "computer_use", + "enable_fanout", + "enable_mcp_apps", + "hooks", + "image_generation", + "in_app_browser", + "multi_agent", + "multi_agent_v2", + "plugin_sharing", + "plugins", + "standalone_web_search", + "web_search_request", + "workspace_dependencies", +) +BENCHMARK_PROVIDER_ENVIRONMENT_KEYS = ( + "CODEX_API_KEY", + "CODEX_HOME", + "CURL_CA_BUNDLE", + "LANG", + "LC_ALL", + "LC_CTYPE", + "OPENAI_API_KEY", + "OPENAI_BASE_URL", + "OPENAI_ORG_ID", + "OPENAI_PROJECT_ID", + "PATH", + "REQUESTS_CA_BUNDLE", + "SSL_CERT_DIR", + "SSL_CERT_FILE", + "TEMP", + "TMP", + "TMPDIR", +) def _nested_mapping(payload: Any, *keys: str) -> dict[str, Any]: @@ -68,6 +121,8 @@ def parse_codex_jsonl(stdout: str) -> tuple[str | None, ModelUsage, str]: session_id: str | None = None usage = ModelUsage() final_message = "" + completed_tool_calls = 0 + completed_tool_item_ids: set[tuple[str, str]] = set() for raw_line in str(stdout or "").splitlines(): try: event = json.loads(raw_line) @@ -98,6 +153,20 @@ def parse_codex_jsonl(stdout: str) -> tuple[str | None, ModelUsage, str]: item = event.get("item") if isinstance(event.get("item"), dict) else params.get("item") item = item if isinstance(item, dict) else event item_type = str(item.get("type") or item.get("kind") or "").strip() + normalized_item_type = item_type.lower().replace("-", "_").replace(".", "_") + if ( + event_type in {"item.completed", "item/completed"} + and ( + normalized_item_type in CODEX_TOOL_ITEM_TYPES + or normalized_item_type.endswith("_tool_call") + ) + ): + item_id = str(item.get("id") or item.get("item_id") or "").strip() + item_identity = (normalized_item_type, item_id) + if not item_id or item_identity not in completed_tool_item_ids: + completed_tool_calls += 1 + if item_id: + completed_tool_item_ids.add(item_identity) if item_type in {"agent_message", "message"} or event_type == "agent_message": candidate_text = item.get("text") or item.get("message") or item.get("content") if isinstance(candidate_text, list): @@ -107,6 +176,15 @@ def parse_codex_jsonl(stdout: str) -> tuple[str | None, ModelUsage, str]: ) if candidate_text: final_message = str(candidate_text).strip() + if usage.source == "native" or completed_tool_calls: + usage = ModelUsage.from_values( + input_tokens=usage.input_tokens, + cached_input_tokens=usage.cached_input_tokens, + output_tokens=usage.output_tokens, + reasoning_output_tokens=usage.reasoning_output_tokens, + total_tokens=usage.total_tokens, + tool_calls=max(usage.tool_calls or 0, completed_tool_calls), + ) return session_id, usage, final_message @@ -211,10 +289,12 @@ def __init__( *, role: str = "", telemetry_recorder: ModelTelemetryRecorder | None = None, + execution_policy: ModelExecutionPolicy | None = None, ): self.runtime_config = runtime_config self.role = str(role or "").strip() self.telemetry_recorder = telemetry_recorder + self.execution_policy = execution_policy or DEFAULT_MODEL_EXECUTION_POLICY def _cli_version(self, cli_name: str) -> str: if self.telemetry_recorder is None or not self.telemetry_recorder.enabled: @@ -223,11 +303,19 @@ def _cli_version(self, cli_name: str) -> str: if cached is not None: return cached try: + run_options: dict[str, Any] = {} + if self.execution_policy.benchmark_mode: + run_options["env"] = self._provider_environment() + run_options["timeout"] = min( + float(self.execution_policy.call_timeout_seconds or 10.0), + 10.0, + ) process = subprocess.run( [cli_name, "--version"], capture_output=True, text=True, check=False, + **run_options, ) version = str(process.stdout or process.stderr or "").strip().splitlines()[0] except Exception: @@ -248,10 +336,172 @@ def _discover_extra_writable_dirs(self, workspace: Path) -> list[str]: continue resolved_text = str(resolved) if resolved != workspace and resolved_text not in seen: + self.execution_policy.assert_workspace_allowed(resolved) seen.add(resolved_text) extra_dirs.append(resolved_text) return extra_dirs + def _provider_environment(self) -> dict[str, str]: + if not self.execution_policy.benchmark_mode: + return {**os.environ, "HOME": str(Path.home())} + environment = { + key: value + for key in BENCHMARK_PROVIDER_ENVIRONMENT_KEYS + if (value := os.environ.get(key)) is not None + } + environment["HOME"] = str(Path.home()) + environment.setdefault("PATH", os.defpath) + environment["NO_COLOR"] = "1" + return environment + + def _append_benchmark_codex_controls( + self, + command: list[str], + *, + supports_sandbox_option: bool, + ) -> None: + if supports_sandbox_option: + command.extend(["--sandbox", "workspace-write"]) + command.extend( + [ + "--ignore-user-config", + "--ignore-rules", + "-c", + 'approval_policy="never"', + "-c", + 'sandbox_mode="workspace-write"', + "-c", + "mcp_servers={}", + "-c", + 'shell_environment_policy.inherit="none"', + ] + ) + for name, value in self.execution_policy.shell_environment.items(): + command.extend( + [ + "-c", + f"shell_environment_policy.set.{name}={json.dumps(value, ensure_ascii=True)}", + ] + ) + for feature_name in BENCHMARK_DISABLED_CODEX_FEATURES: + command.extend(["--disable", feature_name]) + + @staticmethod + def _terminate_process_group( + process: subprocess.Popen[str], + *, + process_group_id: int | None, + grace_seconds: float, + ) -> tuple[str, str]: + def send_group_signal(group_signal: signal.Signals) -> None: + if process.poll() is not None: + return + try: + if process_group_id is not None and hasattr(os, "killpg"): + os.killpg(process_group_id, group_signal) + elif group_signal == signal.SIGTERM: + process.terminate() + else: + process.kill() + except ProcessLookupError: + pass + + send_group_signal(signal.SIGTERM) + try: + stdout, stderr = process.communicate(timeout=grace_seconds) + except subprocess.TimeoutExpired: + send_group_signal(signal.SIGKILL) + stdout, stderr = process.communicate() + return str(stdout or ""), str(stderr or "") + + def _run_benchmark_process( + self, + command: list[str], + *, + cwd: Path, + stdin_input: str | None, + env: dict[str, str], + reservation: InvocationReservation, + ) -> subprocess.CompletedProcess[str]: + process = subprocess.Popen( + command, + cwd=str(cwd), + stdin=subprocess.PIPE if stdin_input is not None else None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + start_new_session=True, + ) + process_group_id: int | None + try: + process_group_id = os.getpgid(process.pid) if hasattr(os, "getpgid") else None + except ProcessLookupError: + process_group_id = process.pid + try: + reservation.mark_started( + pid=process.pid, + process_group_id=process_group_id, + ) + except BaseException: + self._terminate_process_group( + process, + process_group_id=process_group_id, + grace_seconds=float(self.execution_policy.kill_grace_seconds), + ) + raise + + timeout_seconds = float(self.execution_policy.call_timeout_seconds or 0) + try: + stdout, stderr = process.communicate( + input=stdin_input, + timeout=timeout_seconds, + ) + except subprocess.TimeoutExpired: + stdout, stderr = self._terminate_process_group( + process, + process_group_id=process_group_id, + grace_seconds=float(self.execution_policy.kill_grace_seconds), + ) + completed_process = subprocess.CompletedProcess( + command, + process.returncode, + stdout, + stderr, + ) + raise ModelInvocationTimeout( + timeout_seconds, + completed_process=completed_process, + ) + return subprocess.CompletedProcess( + command, + process.returncode, + str(stdout or ""), + str(stderr or ""), + ) + + @staticmethod + def _reservation_result( + *, + completed: bool, + process: Any, + captured_error: BaseException | None, + ) -> tuple[str, str]: + if isinstance(captured_error, ModelInvocationTimeout): + return "timeout", "timeout" + if process is None: + return "launch_failed", ( + normalized_error_category(captured_error) or "launch_failed" + ) + if process.returncode not in (None, 0): + return "failed", "nonzero_exit" + if captured_error is not None or not completed: + return "failed", ( + normalized_error_category(captured_error, exit_code=process.returncode) + or "runner_error" + ) + return "completed", "completed" + def _build_command( self, *, @@ -264,6 +514,11 @@ def _build_command( is_gemini = "gemini" in self.runtime_config.model.lower() if is_gemini: + if self.execution_policy.benchmark_mode: + raise ModelExecutionPolicyViolation( + "Benchmark execution currently supports only the Codex CLI because " + "Gemini cannot provide the same non-interactive workspace-write policy." + ) command = ["gemini"] if session_id: command.extend(["--resume", session_id]) @@ -291,7 +546,12 @@ def _build_command( "--skip-git-repo-check", ] ) - if bypass_sandbox: + if self.execution_policy.benchmark_mode: + self._append_benchmark_codex_controls( + command, + supports_sandbox_option=False, + ) + elif bypass_sandbox: command.append("--dangerously-bypass-approvals-and-sandbox") else: command.append("--full-auto") @@ -314,7 +574,12 @@ def _build_command( ) for extra_dir in self._discover_extra_writable_dirs(workspace): command.extend(["--add-dir", extra_dir]) - if bypass_sandbox: + if self.execution_policy.benchmark_mode: + self._append_benchmark_codex_controls( + command, + supports_sandbox_option=True, + ) + elif bypass_sandbox: command.append("--dangerously-bypass-approvals-and-sandbox") else: command.append("--full-auto") @@ -333,6 +598,19 @@ def run( invocation_context: ModelInvocationContext | None = None, ) -> tuple[str, str | None]: abs_workspace = workspace.expanduser().resolve() + self.execution_policy.assert_workspace_allowed(abs_workspace) + if self.execution_policy.benchmark_mode and bypass_sandbox: + raise ModelExecutionPolicyViolation( + "Benchmark execution forbids sandbox bypass requests." + ) + if self.execution_policy.benchmark_mode: + for directory_key in ("HOME", "TMPDIR", "TMP", "TEMP"): + directory_value = self.execution_policy.shell_environment.get(directory_key) + if not directory_value: + continue + directory_path = Path(directory_value).expanduser().resolve() + self.execution_policy.assert_workspace_allowed(directory_path) + directory_path.mkdir(mode=0o700, parents=True, exist_ok=True) output_file = abs_workspace / ".teams_runtime_codex_output.txt" try: output_file.unlink() @@ -347,7 +625,7 @@ def run( ) is_gemini = "gemini" in self.runtime_config.model.lower() - env = {**os.environ, "HOME": str(Path.home())} + env = self._provider_environment() if is_gemini: env["GEMINI_SYSTEM_MD"] = str(abs_workspace / "GEMINI.md") gemini_dir = abs_workspace / ".gemini" @@ -368,16 +646,40 @@ def run( usage = ModelUsage() captured_error: BaseException | None = None completed = False + reservation: InvocationReservation | None = None try: - process = subprocess.run( - command, - cwd=str(abs_workspace), - capture_output=True, - input=stdin_input, - text=True, - env=env, - check=False, - ) + if self.execution_policy.benchmark_mode: + budget = self.execution_policy.invocation_budget + if budget is None: + raise ModelExecutionPolicyViolation( + "Benchmark execution has no invocation budget." + ) + reservation = budget.reserve( + invocation_context, + provider="gemini_cli" if is_gemini else "codex_cli", + role=self.role, + ) + try: + process = self._run_benchmark_process( + command, + cwd=abs_workspace, + stdin_input=stdin_input, + env=env, + reservation=reservation, + ) + except ModelInvocationTimeout as exc: + process = exc.completed_process + raise + else: + process = subprocess.run( + command, + cwd=str(abs_workspace), + capture_output=True, + input=stdin_input, + text=True, + env=env, + check=False, + ) if is_gemini: try: res_json = json.loads(process.stdout) @@ -420,7 +722,25 @@ def run( captured_error = exc raise finally: - if self.telemetry_recorder is not None and invocation_context is not None: + if reservation is not None: + reservation_state, stop_reason = self._reservation_result( + completed=completed, + process=process, + captured_error=captured_error, + ) + reservation.complete( + state=reservation_state, + exit_code=process.returncode if process is not None else None, + stop_reason=stop_reason, + ) + should_record_telemetry = ( + not self.execution_policy.benchmark_mode or reservation is not None + ) + if ( + should_record_telemetry + and self.telemetry_recorder is not None + and invocation_context is not None + ): cli_name = "gemini" if is_gemini else "codex" exit_code = process.returncode if process is not None else None ended_at = runtime_now() diff --git a/runtime/execution_policy.py b/runtime/execution_policy.py new file mode 100644 index 0000000..82c9479 --- /dev/null +++ b/runtime/execution_policy.py @@ -0,0 +1,432 @@ +from __future__ import annotations + +import json +import math +import os +import re +import tempfile +import threading +import uuid +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from types import MappingProxyType +from typing import Any, Mapping + + +_ENVIRONMENT_NAME_PATTERN = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_SENSITIVE_ENVIRONMENT_NAME_PATTERN = re.compile( + r"(?:^|_)(?:API_?KEY|AUTH|CREDENTIALS?|PASSWORD|SECRET|TOKEN)(?:$|_)", + re.IGNORECASE, +) +_TERMINAL_STATES = {"completed", "failed", "timeout", "launch_failed"} + + +def _utc_timestamp() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _positive_finite_number(value: Any, *, name: str) -> float: + if isinstance(value, bool): + raise ValueError(f"{name} must be a positive finite number.") + try: + normalized = float(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{name} must be a positive finite number.") from exc + if not math.isfinite(normalized) or normalized <= 0: + raise ValueError(f"{name} must be a positive finite number.") + return normalized + + +def _non_negative_finite_number(value: Any, *, name: str) -> float: + if isinstance(value, bool): + raise ValueError(f"{name} must be a non-negative finite number.") + try: + normalized = float(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{name} must be a non-negative finite number.") from exc + if not math.isfinite(normalized) or normalized < 0: + raise ValueError(f"{name} must be a non-negative finite number.") + return normalized + + +class InvocationBudgetExceeded(RuntimeError): + def __init__(self, max_invocations: int, reserved_count: int): + self.max_invocations = max_invocations + self.reserved_count = reserved_count + super().__init__( + f"Model invocation budget exhausted: {reserved_count}/{max_invocations} calls are already reserved." + ) + + +class ModelExecutionPolicyViolation(RuntimeError): + """Raised before launch when a benchmark request violates its safety policy.""" + + +class ModelInvocationTimeout(RuntimeError, TimeoutError): + def __init__( + self, + timeout_seconds: float, + *, + completed_process: Any = None, + ): + self.timeout_seconds = timeout_seconds + self.completed_process = completed_process + super().__init__(f"Model invocation exceeded the {timeout_seconds:g}-second timeout.") + + +class InvocationReservation: + __slots__ = ("_budget", "reservation_id") + + def __init__(self, budget: "InvocationBudget", reservation_id: str): + self._budget = budget + self.reservation_id = reservation_id + + def mark_started(self, *, pid: int, process_group_id: int | None) -> None: + self._budget._mark_started( # noqa: SLF001 - reservation is the budget's public mutation handle + self.reservation_id, + pid=pid, + process_group_id=process_group_id, + ) + + def complete( + self, + *, + state: str, + exit_code: int | None, + stop_reason: str, + ) -> None: + self._budget._complete( # noqa: SLF001 - reservation is the budget's public mutation handle + self.reservation_id, + state=state, + exit_code=exit_code, + stop_reason=stop_reason, + ) + + +class InvocationBudget: + """Thread-safe physical-call budget with an atomic, privacy-safe journal.""" + + def __init__( + self, + max_invocations: int, + *, + journal_path: str | os.PathLike[str] | None = None, + ): + if isinstance(max_invocations, bool) or not isinstance(max_invocations, int) or max_invocations <= 0: + raise ValueError("max_invocations must be a positive integer.") + self.max_invocations = max_invocations + self.journal_path = ( + Path(journal_path).expanduser().resolve() + if journal_path is not None + else None + ) + self._lock = threading.RLock() + self._entries: list[dict[str, Any]] = [] + self._entries_by_id: dict[str, dict[str, Any]] = {} + self._rejected_count = 0 + + @property + def reserved_count(self) -> int: + with self._lock: + return len(self._entries) + + @property + def remaining(self) -> int: + with self._lock: + return max(self.max_invocations - len(self._entries), 0) + + @property + def rejected_count(self) -> int: + with self._lock: + return self._rejected_count + + def reserve( + self, + invocation_context: Any = None, + *, + provider: str, + role: str = "", + ) -> InvocationReservation: + with self._lock: + if len(self._entries) >= self.max_invocations: + self._rejected_count += 1 + self._persist_locked() + raise InvocationBudgetExceeded(self.max_invocations, len(self._entries)) + + reservation_id = uuid.uuid4().hex + entry = { + "reservation_id": reservation_id, + "provider": str(provider or "").strip(), + "invocation_id": str(getattr(invocation_context, "invocation_id", "") or "").strip(), + "operation_id": str(getattr(invocation_context, "operation_id", "") or "").strip(), + "logical_call_id": str(getattr(invocation_context, "logical_call_id", "") or "").strip(), + "attempt_index": getattr(invocation_context, "attempt_index", None), + "attempt_kind": str(getattr(invocation_context, "attempt_kind", "") or "").strip(), + "runtime_identity": str( + getattr(invocation_context, "runtime_identity", "") or "" + ).strip(), + "role": str(getattr(invocation_context, "role", "") or role or "").strip(), + "purpose": str(getattr(invocation_context, "purpose", "") or "").strip(), + "workflow_step": str( + getattr(invocation_context, "workflow_step", "") or "" + ).strip(), + "request_id": str(getattr(invocation_context, "request_id", "") or "").strip(), + "sprint_id": str(getattr(invocation_context, "sprint_id", "") or "").strip(), + "todo_id": str(getattr(invocation_context, "todo_id", "") or "").strip(), + "backlog_id": str(getattr(invocation_context, "backlog_id", "") or "").strip(), + "goal_id": str(getattr(invocation_context, "goal_id", "") or "").strip(), + "state": "reserved", + "reserved_at": _utc_timestamp(), + "started_at": "", + "completed_at": "", + "pid": None, + "process_group_id": None, + "exit_code": None, + "stop_reason": "", + } + self._entries.append(entry) + self._entries_by_id[reservation_id] = entry + self._persist_locked() + return InvocationReservation(self, reservation_id) + + def snapshot(self) -> dict[str, Any]: + with self._lock: + return self._snapshot_locked() + + def _snapshot_locked(self) -> dict[str, Any]: + return { + "schema_version": 1, + "max_invocations": self.max_invocations, + "reserved_count": len(self._entries), + "remaining": max(self.max_invocations - len(self._entries), 0), + "rejected_count": self._rejected_count, + "entries": [dict(entry) for entry in self._entries], + } + + def _mark_started( + self, + reservation_id: str, + *, + pid: int, + process_group_id: int | None, + ) -> None: + with self._lock: + entry = self._entries_by_id[reservation_id] + if entry["state"] != "reserved": + raise RuntimeError(f"Invocation reservation {reservation_id} is already started.") + entry.update( + { + "state": "running", + "started_at": _utc_timestamp(), + "pid": int(pid), + "process_group_id": ( + int(process_group_id) + if process_group_id is not None + else None + ), + } + ) + self._persist_locked() + + def _complete( + self, + reservation_id: str, + *, + state: str, + exit_code: int | None, + stop_reason: str, + ) -> None: + normalized_state = str(state or "").strip() + if normalized_state not in _TERMINAL_STATES: + raise ValueError(f"Unsupported invocation terminal state: {state}") + with self._lock: + entry = self._entries_by_id[reservation_id] + if entry["state"] in _TERMINAL_STATES: + return + entry.update( + { + "state": normalized_state, + "completed_at": _utc_timestamp(), + "exit_code": int(exit_code) if exit_code is not None else None, + "stop_reason": str(stop_reason or "").strip(), + } + ) + self._persist_locked() + + def _persist_locked(self) -> None: + if self.journal_path is None: + return + journal_path = self.journal_path + journal_path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + try: + journal_path.parent.chmod(0o700) + except OSError: + pass + descriptor, temporary_name = tempfile.mkstemp( + dir=str(journal_path.parent), + prefix=f".{journal_path.name}.", + suffix=".tmp", + ) + temporary_path = Path(temporary_name) + try: + try: + os.fchmod(descriptor, 0o600) + except OSError: + pass + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + json.dump( + self._snapshot_locked(), + handle, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ) + handle.write("\n") + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary_path, journal_path) + try: + journal_path.chmod(0o600) + except OSError: + pass + except BaseException: + try: + os.close(descriptor) + except OSError: + pass + try: + temporary_path.unlink() + except FileNotFoundError: + pass + raise + + +def _normalize_shell_environment(values: Mapping[str, str]) -> Mapping[str, str]: + normalized: dict[str, str] = {} + for raw_name, raw_value in values.items(): + name = str(raw_name or "").strip() + if not _ENVIRONMENT_NAME_PATTERN.fullmatch(name): + raise ValueError(f"Invalid shell environment variable name: {raw_name!r}") + if _SENSITIVE_ENVIRONMENT_NAME_PATTERN.search(name): + raise ValueError( + f"Benchmark shell environment must not expose secret-bearing variable {name}." + ) + if not isinstance(raw_value, str): + raise ValueError(f"Benchmark shell environment value for {name} must be a string.") + if "\x00" in raw_value or "\n" in raw_value or "\r" in raw_value: + raise ValueError(f"Benchmark shell environment value for {name} contains control characters.") + normalized[name] = raw_value + return MappingProxyType(dict(sorted(normalized.items()))) + + +@dataclass(slots=True, frozen=True) +class ModelExecutionPolicy: + """Immutable opt-in controls for benchmark model execution.""" + + benchmark_mode: bool = False + call_timeout_seconds: float | None = None + kill_grace_seconds: float = 5.0 + invocation_budget: InvocationBudget | None = field(default=None, compare=False) + allowed_workspace_root: Path | None = None + shell_environment: Mapping[str, str] = field( + default_factory=lambda: MappingProxyType({}), + hash=False, + ) + + def __post_init__(self) -> None: + normalized_environment = _normalize_shell_environment(self.shell_environment) + object.__setattr__(self, "shell_environment", normalized_environment) + object.__setattr__( + self, + "kill_grace_seconds", + _non_negative_finite_number( + self.kill_grace_seconds, + name="kill_grace_seconds", + ), + ) + if not self.benchmark_mode: + if ( + self.call_timeout_seconds is not None + or self.invocation_budget is not None + or self.allowed_workspace_root is not None + or normalized_environment + ): + raise ValueError( + "Bounded execution controls require benchmark_mode=True; use " + "ModelExecutionPolicy.for_benchmark()." + ) + return + + object.__setattr__( + self, + "call_timeout_seconds", + _positive_finite_number( + self.call_timeout_seconds, + name="call_timeout_seconds", + ), + ) + if self.invocation_budget is None: + raise ValueError("Benchmark execution requires an invocation budget.") + if self.allowed_workspace_root is None: + raise ValueError("Benchmark execution requires an allowed workspace root.") + allowed_root = Path(self.allowed_workspace_root).expanduser().resolve() + object.__setattr__(self, "allowed_workspace_root", allowed_root) + + @classmethod + def for_benchmark( + cls, + *, + allowed_workspace_root: str | os.PathLike[str], + invocation_budget: InvocationBudget, + call_timeout_seconds: float, + kill_grace_seconds: float = 5.0, + shell_environment: Mapping[str, str] | None = None, + ) -> "ModelExecutionPolicy": + allowed_root = Path(allowed_workspace_root).expanduser().resolve() + environment = { + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_TERMINAL_PROMPT": "0", + "HOME": str(allowed_root), + "PATH": os.environ.get("PATH") or os.defpath, + "TMPDIR": str(allowed_root / ".tmp"), + } + environment.update(shell_environment or {}) + return cls( + benchmark_mode=True, + call_timeout_seconds=call_timeout_seconds, + kill_grace_seconds=kill_grace_seconds, + invocation_budget=invocation_budget, + allowed_workspace_root=allowed_root, + shell_environment=environment, + ) + + def assert_workspace_allowed(self, workspace: Path) -> None: + if not self.benchmark_mode: + return + allowed_root = self.allowed_workspace_root + try: + resolved_workspace = Path(workspace).expanduser().resolve() + except (OSError, RuntimeError) as exc: + raise ModelExecutionPolicyViolation( + f"Benchmark workspace {workspace} could not be resolved safely." + ) from exc + if allowed_root is None or not resolved_workspace.is_relative_to(allowed_root): + raise ModelExecutionPolicyViolation( + f"Benchmark workspace {resolved_workspace} is outside the allowed root {allowed_root}." + ) + + +DEFAULT_MODEL_EXECUTION_POLICY = ModelExecutionPolicy() + + +__all__ = [ + "DEFAULT_MODEL_EXECUTION_POLICY", + "InvocationBudget", + "InvocationBudgetExceeded", + "InvocationReservation", + "ModelExecutionPolicy", + "ModelExecutionPolicyViolation", + "ModelInvocationTimeout", +] diff --git a/runtime/internal/goal_sourcing.py b/runtime/internal/goal_sourcing.py index ed3e566..a36ee29 100644 --- a/runtime/internal/goal_sourcing.py +++ b/runtime/internal/goal_sourcing.py @@ -8,6 +8,7 @@ from typing import Any from teams_runtime.runtime.codex_runner import CodexRunner, extract_json_object +from teams_runtime.runtime.execution_policy import ModelExecutionPolicy from teams_runtime.runtime.identities import service_runtime_identity from teams_runtime.runtime.model_telemetry import ( InvocationSequence, @@ -227,6 +228,7 @@ def __init__( runtime_config: RoleRuntimeConfig, session_identity: str | None = None, telemetry_config: TelemetryRuntimeConfig | None = None, + execution_policy: ModelExecutionPolicy | None = None, ): self.paths = paths self.role = "sourcer" @@ -244,6 +246,7 @@ def __init__( runtime_config, role=self.role, telemetry_recorder=self.telemetry_recorder, + execution_policy=execution_policy, ) self._run_lock = threading.Lock() diff --git a/runtime/internal/intent_parser.py b/runtime/internal/intent_parser.py index aebe11a..5a65b71 100644 --- a/runtime/internal/intent_parser.py +++ b/runtime/internal/intent_parser.py @@ -9,6 +9,7 @@ from teams_runtime.workflows.orchestration.ingress import is_manual_sprint_finalize_text, is_manual_sprint_start_text from teams_runtime.shared.paths import RuntimePaths from teams_runtime.runtime.codex_runner import CodexRunner, extract_json_object +from teams_runtime.runtime.execution_policy import ModelExecutionPolicy from teams_runtime.runtime.identities import service_runtime_identity from teams_runtime.runtime.model_telemetry import ( InvocationSequence, @@ -202,6 +203,7 @@ def __init__( runtime_config: RoleRuntimeConfig, session_identity: str | None = None, telemetry_config: TelemetryRuntimeConfig | None = None, + execution_policy: ModelExecutionPolicy | None = None, ): self.paths = paths self.role = "parser" @@ -219,6 +221,7 @@ def __init__( runtime_config, role=self.role, telemetry_recorder=self.telemetry_recorder, + execution_policy=execution_policy, ) self._run_lock = threading.Lock() diff --git a/runtime/model_telemetry.py b/runtime/model_telemetry.py index e1dade1..1c12048 100644 --- a/runtime/model_telemetry.py +++ b/runtime/model_telemetry.py @@ -115,6 +115,13 @@ class ModelInvocationContext: todo_id: str = "" backlog_id: str = "" goal_id: str = "" + prompt_context_enabled: bool | None = None + prompt_context_total_events: int | None = None + prompt_context_included_events: int | None = None + prompt_context_omitted_events: int | None = None + prompt_context_recent_events: int | None = None + prompt_context_max_events: int | None = None + prompt_context_selection_policy: str = "" class InvocationSequence: @@ -144,6 +151,13 @@ def __init__( self.operation_id = _text(operation_id) or uuid.uuid4().hex self.logical_call_id = uuid.uuid4().hex self._attempt_index = 0 + self.prompt_context_enabled: bool | None = None + self.prompt_context_total_events: int | None = None + self.prompt_context_included_events: int | None = None + self.prompt_context_omitted_events: int | None = None + self.prompt_context_recent_events: int | None = None + self.prompt_context_max_events: int | None = None + self.prompt_context_selection_policy = "" @classmethod def from_request( @@ -177,6 +191,41 @@ def start_logical_call(self, *, purpose: str | None = None) -> None: if purpose is not None: self.purpose = _text(purpose) or self.purpose + def set_prompt_context_projection( + self, + projection: Any, + *, + enabled: bool, + selection_policy: str = "", + ) -> None: + """Attach content-free prompt projection evidence to subsequent attempts.""" + self.prompt_context_enabled = bool(enabled) + self.prompt_context_total_events = _optional_non_negative_int( + getattr(projection, "total_events", None) + ) + self.prompt_context_included_events = _optional_non_negative_int( + getattr(projection, "included_events", None) + ) + self.prompt_context_omitted_events = _optional_non_negative_int( + getattr(projection, "omitted_events", None) + ) + self.prompt_context_recent_events = _optional_non_negative_int( + getattr(projection, "recent_events", None) + ) + self.prompt_context_max_events = _optional_non_negative_int( + getattr(projection, "max_events", None) + ) + self.prompt_context_selection_policy = _text(selection_policy) + + def clear_prompt_context_projection(self) -> None: + self.prompt_context_enabled = None + self.prompt_context_total_events = None + self.prompt_context_included_events = None + self.prompt_context_omitted_events = None + self.prompt_context_recent_events = None + self.prompt_context_max_events = None + self.prompt_context_selection_policy = "" + def next(self, attempt_kind: str = "primary") -> ModelInvocationContext: normalized_kind = _text(attempt_kind) if normalized_kind not in VALID_ATTEMPT_KINDS: @@ -197,6 +246,13 @@ def next(self, attempt_kind: str = "primary") -> ModelInvocationContext: todo_id=self.todo_id, backlog_id=self.backlog_id, goal_id=self.goal_id, + prompt_context_enabled=self.prompt_context_enabled, + prompt_context_total_events=self.prompt_context_total_events, + prompt_context_included_events=self.prompt_context_included_events, + prompt_context_omitted_events=self.prompt_context_omitted_events, + prompt_context_recent_events=self.prompt_context_recent_events, + prompt_context_max_events=self.prompt_context_max_events, + prompt_context_selection_policy=self.prompt_context_selection_policy, ) @@ -298,6 +354,13 @@ def record( "todo_id": context.todo_id, "backlog_id": context.backlog_id, "goal_id": context.goal_id, + "prompt_context_enabled": context.prompt_context_enabled, + "prompt_context_total_events": context.prompt_context_total_events, + "prompt_context_included_events": context.prompt_context_included_events, + "prompt_context_omitted_events": context.prompt_context_omitted_events, + "prompt_context_recent_events": context.prompt_context_recent_events, + "prompt_context_max_events": context.prompt_context_max_events, + "prompt_context_selection_policy": context.prompt_context_selection_policy, "provider": _text(provider), "model": _text(model), "reasoning": _text(reasoning), @@ -455,13 +518,18 @@ def _empty_group(role: str, purpose: str, provider: str, model: str) -> dict[str "provider": provider, "model": model, "invocation_count": 0, + "primary_count": 0, "failed_count": 0, "contract_repair_count": 0, "sandbox_retry_count": 0, + "tool_call_count": 0, "input_tokens": 0, "cached_input_tokens": 0, + "uncached_input_tokens": 0, "output_tokens": 0, "total_tokens": 0, + "prompt_context_observed_count": 0, + "prompt_context_compacted_count": 0, "duration_ms": 0, "estimated_cost_usd": None, } @@ -526,6 +594,7 @@ def aggregate_model_invocations( durations: list[int] = [] logical_calls: set[str] = set() groups: dict[tuple[str, str, str, str], dict[str, Any]] = {} + group_priced_counts: dict[tuple[str, str, str, str], int] = {} token_fields = ( "input_tokens", "cached_input_tokens", @@ -534,9 +603,18 @@ def aggregate_model_invocations( "total_tokens", ) tokens = {name: 0 for name in token_fields} - completed_count = failed_count = repair_count = sandbox_count = 0 + completed_count = failed_count = primary_count = repair_count = sandbox_count = 0 prompt_chars = output_chars = 0 - native_usage_count = priced_count = 0 + native_usage_count = priced_count = tool_call_coverage_count = 0 + tool_call_count = 0 + prompt_context_observed_count = 0 + prompt_context_enabled_count = 0 + prompt_context_eligible_count = 0 + prompt_context_compacted_count = 0 + prompt_context_total_events = 0 + prompt_context_included_events = 0 + prompt_context_omitted_events = 0 + prompt_context_selection_policies: set[str] = set() total_cost = 0.0 for record in invocations: logical_id = _text(record.get("logical_call_id")) @@ -550,14 +628,57 @@ def aggregate_model_invocations( completed_count += 1 else: failed_count += 1 - if _text(record.get("attempt_kind")) == "contract_repair": + attempt_kind = _text(record.get("attempt_kind")) + if attempt_kind == "primary": + primary_count += 1 + if attempt_kind == "contract_repair": repair_count += 1 - if _text(record.get("attempt_kind")) == "sandbox_retry": + if attempt_kind == "sandbox_retry": sandbox_count += 1 if _text(record.get("usage_source")) == "native": native_usage_count += 1 for name in token_fields: tokens[name] += _optional_non_negative_int(record.get(name)) or 0 + input_tokens = _optional_non_negative_int(record.get("input_tokens")) + cached_input_tokens = _optional_non_negative_int(record.get("cached_input_tokens")) or 0 + uncached_input_tokens = ( + max(input_tokens - min(cached_input_tokens, input_tokens), 0) + if input_tokens is not None + else 0 + ) + tool_calls = _optional_non_negative_int(record.get("tool_calls")) + if tool_calls is not None: + tool_call_coverage_count += 1 + tool_call_count += tool_calls + + prompt_context_enabled = record.get("prompt_context_enabled") + context_counts = tuple( + _optional_non_negative_int(record.get(name)) + for name in ( + "prompt_context_total_events", + "prompt_context_included_events", + "prompt_context_omitted_events", + ) + ) + prompt_context_observed = isinstance(prompt_context_enabled, bool) and all( + value is not None for value in context_counts + ) + prompt_context_compacted = False + if prompt_context_observed: + total_events, included_events, omitted_events = context_counts + prompt_context_observed_count += 1 + prompt_context_enabled_count += int(prompt_context_enabled) + prompt_context_total_events += int(total_events or 0) + prompt_context_included_events += int(included_events or 0) + prompt_context_omitted_events += int(omitted_events or 0) + max_events = _optional_non_negative_int(record.get("prompt_context_max_events")) + if max_events is not None and int(total_events or 0) > max_events: + prompt_context_eligible_count += 1 + prompt_context_compacted = bool(prompt_context_enabled) and int(omitted_events or 0) > 0 + prompt_context_compacted_count += int(prompt_context_compacted) + selection_policy = _text(record.get("prompt_context_selection_policy")) + if selection_policy: + prompt_context_selection_policies.add(selection_policy) cost = record.get("estimated_cost_usd") if isinstance(cost, (int, float)) and math.isfinite(float(cost)): priced_count += 1 @@ -567,12 +688,20 @@ def aggregate_model_invocations( group = groups.setdefault(key, _empty_group(*key)) group["invocation_count"] += 1 group["duration_ms"] += duration + if attempt_kind == "primary": + group["primary_count"] += 1 if _text(record.get("status")) != "completed": group["failed_count"] += 1 - if _text(record.get("attempt_kind")) == "contract_repair": + if attempt_kind == "contract_repair": group["contract_repair_count"] += 1 - if _text(record.get("attempt_kind")) == "sandbox_retry": + if attempt_kind == "sandbox_retry": group["sandbox_retry_count"] += 1 + group["tool_call_count"] += tool_calls or 0 + group["uncached_input_tokens"] += uncached_input_tokens + if prompt_context_observed: + group["prompt_context_observed_count"] += 1 + if prompt_context_compacted: + group["prompt_context_compacted_count"] += 1 for source, target in ( ("input_tokens", "input_tokens"), ("cached_input_tokens", "cached_input_tokens"), @@ -582,33 +711,61 @@ def aggregate_model_invocations( group[target] += _optional_non_negative_int(record.get(source)) or 0 if isinstance(cost, (int, float)) and math.isfinite(float(cost)): group["estimated_cost_usd"] = round((group["estimated_cost_usd"] or 0.0) + float(cost), 12) + group_priced_counts[key] = group_priced_counts.get(key, 0) + 1 count = len(invocations) + for key, group in groups.items(): + if group_priced_counts.get(key, 0) != group["invocation_count"]: + group["estimated_cost_usd"] = None return { "schema_version": TELEMETRY_SCHEMA_VERSION, "generated_at": runtime_now_iso(), "filters": filters, "totals": { "invocation_count": count, + "physical_attempt_count": count, "logical_call_count": len(logical_calls), + "primary_count": primary_count, "completed_count": completed_count, "failed_count": failed_count, "contract_repair_count": repair_count, "sandbox_retry_count": sandbox_count, + "tool_call_count": tool_call_count, "prompt_chars": prompt_chars, "output_chars": output_chars, - "estimated_cost_usd": round(total_cost, 12) if priced_count else None, + "estimated_cost_usd": ( + round(total_cost, 12) if count and priced_count == count else None + ), "token_coverage_percent": round(native_usage_count * 100 / count, 2) if count else 0.0, + "tool_call_coverage_percent": ( + round(tool_call_coverage_count * 100 / count, 2) if count else 0.0 + ), "pricing_coverage_percent": round(priced_count * 100 / count, 2) if count else 0.0, "invalid_record_count": invalid_count, }, "tokens": { "input": tokens["input_tokens"], "cached_input": tokens["cached_input_tokens"], + "uncached_input": sum( + int(group["uncached_input_tokens"]) for group in groups.values() + ), "output": tokens["output_tokens"], "reasoning_output": tokens["reasoning_output_tokens"], "total": tokens["total_tokens"], }, + "prompt_context": { + "observed_invocation_count": prompt_context_observed_count, + "enabled_invocation_count": prompt_context_enabled_count, + "eligible_invocation_count": prompt_context_eligible_count, + "compacted_invocation_count": prompt_context_compacted_count, + "total_events": prompt_context_total_events, + "included_events": prompt_context_included_events, + "omitted_events": prompt_context_omitted_events, + "coverage_percent": ( + round(prompt_context_observed_count * 100 / count, 2) if count else 0.0 + ), + "selection_policies": sorted(prompt_context_selection_policies), + }, "latency_ms": { "total": sum(durations), "p50": _nearest_rank(durations, 0.50), diff --git a/runtime/research_runtime.py b/runtime/research_runtime.py index 20ce7ce..2842d28 100644 --- a/runtime/research_runtime.py +++ b/runtime/research_runtime.py @@ -24,6 +24,7 @@ normalize_role_payload, ) from teams_runtime.runtime.codex_runner import extract_json_object +from teams_runtime.runtime.execution_policy import ModelExecutionPolicy from teams_runtime.runtime.model_telemetry import ( InvocationSequence, normalized_error_category, @@ -31,6 +32,10 @@ run_with_optional_telemetry, ) from teams_runtime.shared.persistence import runtime_now +from teams_runtime.shared.prompt_context import ( + PROMPT_EVENT_SELECTION_POLICY, + project_request_record_for_prompt, +) from teams_runtime.workflows.roles.research import ( RESEARCH_REPORT_LIST_FIELDS, RESEARCH_REASON_CODE_BLOCKED_DECISION_FAILED, @@ -95,6 +100,8 @@ def __init__( session_identity: str | None = None, telemetry_config: TelemetryRuntimeConfig | None = None, prompt_context_config: PromptContextRuntimeConfig | None = None, + allow_external_research: bool = True, + execution_policy: ModelExecutionPolicy | None = None, ): super().__init__( paths=paths, @@ -105,8 +112,10 @@ def __init__( session_identity=session_identity, telemetry_config=telemetry_config, prompt_context_config=prompt_context_config, + execution_policy=execution_policy, ) self.research_defaults = research_defaults + self.allow_external_research = bool(allow_external_research) def run_task( self, @@ -211,6 +220,46 @@ def run_task( "session_id": active_session_id or "", "session_workspace": state.workspace_path, } + if signal["needed"] and not self.allow_external_research: + disabled_details = { + "failure_stage": "external_research_policy", + "reason": "external_research_disabled", + } + requirement_traceability_matrix = mark_requirement_traceability_research_status( + requirement_traceability_matrix, + research_status="failed", + failure_details=disabled_details, + ) + sprint_prepass = _is_sprint_research_prepass_request(envelope, request_record) + payload["status"] = "completed" if sprint_prepass else "blocked" + payload["summary"] = ( + "외부 research 실행은 비활성화되어 unresolved research risk를 planner에 전달합니다." + if sprint_prepass + else "외부 research가 필요하지만 현재 실행 정책에서 비활성화되어 있습니다." + ) + payload["error"] = "" if sprint_prepass else "external_research_disabled" + payload["proposals"]["requirement_traceability_matrix"] = requirement_traceability_matrix + payload["proposals"]["research_report"] = { + "report_artifact": "", + "research_url": "", + "headline": "External research disabled by execution policy", + "planner_guidance": ( + f"{planner_guidance} 외부 research는 실행되지 않았으므로 관련 가정을 unresolved risk로 유지하세요." + ).strip(), + "research_subject_definition": subject_definition, + "requirement_traceability_matrix": requirement_traceability_matrix, + "research_execution_status": "disabled_by_policy", + "backing_sources": [], + **{field: [] for field in RESEARCH_REPORT_LIST_FIELDS}, + "open_questions": [ + str(signal.get("research_query") or signal.get("subject") or "외부 근거 확인 필요").strip() + ], + "effective_config": asdict(effective_config), + } + state = session_manager.finalize_session_id(state, active_session_id) + payload["session_id"] = state.session_id + payload["session_workspace"] = state.workspace_path + return normalize_role_payload(payload) if signal["needed"]: prompt = build_research_prompt( envelope, @@ -238,6 +287,7 @@ def run_task( else: reasoning_level = raw_reasoning invocation_sequence.start_logical_call(purpose="deep_research") + invocation_sequence.clear_prompt_context_projection() external_context = invocation_sequence.next("primary") external_started_at = runtime_now() external_started_monotonic = time.monotonic() @@ -535,6 +585,15 @@ def _run_research_decision( local_sources_checked: list[str], invocation_sequence: InvocationSequence, ) -> tuple[dict[str, Any], str | None]: + request_projection = project_request_record_for_prompt( + request_record, + self.prompt_context_config, + ) + invocation_sequence.set_prompt_context_projection( + request_projection, + enabled=self.prompt_context_config.enabled, + selection_policy=PROMPT_EVENT_SELECTION_POLICY, + ) prompt = build_research_decision_prompt( envelope, request_record, diff --git a/tests/test_execution_policy.py b/tests/test_execution_policy.py new file mode 100644 index 0000000..d728c06 --- /dev/null +++ b/tests/test_execution_policy.py @@ -0,0 +1,163 @@ +from __future__ import annotations + +import json +import os +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +from teams_runtime.runtime.codex_runner import CodexRunner +from teams_runtime.runtime.execution_policy import ( + InvocationBudget, + ModelExecutionPolicy, + ModelExecutionPolicyViolation, + ModelInvocationTimeout, +) +from teams_runtime.shared.models import RoleRuntimeConfig + + +class BenchmarkExecutionPolicyTests(unittest.TestCase): + def _policy( + self, + root: Path, + *, + max_invocations: int = 2, + timeout_seconds: float = 1.0, + ) -> tuple[ModelExecutionPolicy, InvocationBudget]: + budget = InvocationBudget( + max_invocations, + journal_path=root / "call_journal.json", + ) + policy = ModelExecutionPolicy.for_benchmark( + allowed_workspace_root=root, + invocation_budget=budget, + call_timeout_seconds=timeout_seconds, + kill_grace_seconds=0.1, + shell_environment={"PYTHONPATH": str(root)}, + ) + return policy, budget + + def test_benchmark_codex_command_is_sandboxed_without_bypass(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory).resolve() + workspace = root / "role" + workspace.mkdir() + policy, _budget = self._policy(root) + runner = CodexRunner( + RoleRuntimeConfig(model="gpt-benchmark", reasoning="high"), + role="developer", + execution_policy=policy, + ) + + command, stdin_input = runner._build_command( + workspace=workspace, + prompt="content-safe test prompt", + session_id=None, + output_file=workspace / "output.txt", + bypass_sandbox=False, + ) + + self.assertEqual(stdin_input, "content-safe test prompt") + self.assertIn("--sandbox", command) + self.assertIn("workspace-write", command) + self.assertIn("--ignore-user-config", command) + self.assertIn("--ignore-rules", command) + self.assertIn("mcp_servers={}", command) + self.assertNotIn( + "--dangerously-bypass-approvals-and-sandbox", + command, + ) + self.assertNotIn("--full-auto", command) + + def test_provider_environment_excludes_github_and_discord_secrets(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory).resolve() + policy, _budget = self._policy(root) + runner = CodexRunner( + RoleRuntimeConfig(model="gpt-benchmark"), + execution_policy=policy, + ) + source_environment = { + "OPENAI_API_KEY": "provider-secret", + "GH_TOKEN": "github-secret", + "DISCORD_TOKEN": "discord-secret", + "PATH": os.defpath, + } + + with mock.patch.dict(os.environ, source_environment, clear=True): + environment = runner._provider_environment() + + self.assertEqual(environment["OPENAI_API_KEY"], "provider-secret") + self.assertNotIn("GH_TOKEN", environment) + self.assertNotIn("DISCORD_TOKEN", environment) + + def test_benchmark_rejects_bypass_and_gemini_before_launch(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory).resolve() + workspace = root / "role" + workspace.mkdir() + policy, budget = self._policy(root) + codex_runner = CodexRunner( + RoleRuntimeConfig(model="gpt-benchmark"), + execution_policy=policy, + ) + with self.assertRaises(ModelExecutionPolicyViolation): + codex_runner.run( + workspace, + "prompt", + None, + bypass_sandbox=True, + ) + self.assertEqual(budget.reserved_count, 0) + + gemini_runner = CodexRunner( + RoleRuntimeConfig(model="gemini-benchmark"), + execution_policy=policy, + ) + with self.assertRaises(ModelExecutionPolicyViolation): + gemini_runner.run(workspace, "prompt", None) + self.assertEqual(budget.reserved_count, 0) + + def test_real_timeout_marks_journal_and_terminates_process_group(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory).resolve() + workspace = root / "role" + workspace.mkdir() + policy, budget = self._policy( + root, + max_invocations=1, + timeout_seconds=0.1, + ) + runner = CodexRunner( + RoleRuntimeConfig(model="gpt-benchmark"), + role="developer", + execution_policy=policy, + ) + sleeping_command = [ + sys.executable, + "-c", + "import time; time.sleep(30)", + ] + + with mock.patch.object( + runner, + "_build_command", + return_value=(sleeping_command, None), + ): + with self.assertRaises(ModelInvocationTimeout): + runner.run(workspace, "prompt", None) + + snapshot = budget.snapshot() + self.assertEqual(snapshot["reserved_count"], 1) + self.assertEqual(snapshot["entries"][0]["state"], "timeout") + self.assertEqual(snapshot["entries"][0]["stop_reason"], "timeout") + persisted = json.loads( + (root / "call_journal.json").read_text(encoding="utf-8") + ) + self.assertEqual(persisted["entries"][0]["state"], "timeout") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_model_telemetry.py b/tests/test_model_telemetry.py index 346dda8..a64a0d4 100644 --- a/tests/test_model_telemetry.py +++ b/tests/test_model_telemetry.py @@ -50,6 +50,36 @@ def test_codex_jsonl_parser_recovers_session_usage_and_final_message(self): "item": {"type": "agent_message", "text": '{"status":"completed"}'}, } ), + json.dumps( + { + "type": "item.completed", + "item": {"id": "command-1", "type": "command_execution"}, + } + ), + json.dumps( + { + "type": "item/completed", + "item": {"id": "mcp-1", "type": "mcp_tool_call"}, + } + ), + json.dumps( + { + "type": "item.completed", + "item": {"id": "mcp-1", "type": "mcp_tool_call"}, + } + ), + json.dumps( + { + "type": "item.completed", + "item": {"type": "file_change"}, + } + ), + json.dumps( + { + "type": "item.completed", + "item": {"type": "web_search"}, + } + ), json.dumps( { "type": "turn.completed", @@ -73,9 +103,41 @@ def test_codex_jsonl_parser_recovers_session_usage_and_final_message(self): self.assertEqual(usage.output_tokens, 25) self.assertEqual(usage.reasoning_output_tokens, 5) self.assertEqual(usage.total_tokens, 125) + self.assertEqual(usage.tool_calls, 4) self.assertEqual(usage.source, "native") self.assertEqual(final_message, '{"status":"completed"}') + def test_codex_jsonl_parser_does_not_double_count_terminal_tool_usage(self): + stdout = "\n".join( + ( + json.dumps( + { + "type": "item.completed", + "item": {"id": "command-1", "type": "command_execution"}, + } + ), + json.dumps( + { + "type": "turn.completed", + "usage": { + "input_tokens": 10, + "output_tokens": 2, + "total_tokens": 12, + "tool_calls": 3, + }, + } + ), + ) + ) + + _session_id, usage, _final_message = parse_codex_jsonl(stdout) + + self.assertEqual(usage.input_tokens, 10) + self.assertEqual(usage.output_tokens, 2) + self.assertEqual(usage.total_tokens, 12) + self.assertEqual(usage.tool_calls, 3) + self.assertEqual(usage.source, "native") + def test_gemini_usage_parser_sums_models_and_tool_calls(self): usage = parse_gemini_usage( { @@ -130,6 +192,39 @@ def test_cost_calculation_separates_cached_input(self): 1.25, ) + def test_invocation_sequence_carries_prompt_context_projection_across_attempts(self): + sequence = InvocationSequence( + runtime_identity="service-planner", + role="planner", + purpose="role_task", + ) + sequence.set_prompt_context_projection( + SimpleNamespace( + total_events=100, + included_events=16, + omitted_events=84, + recent_events=8, + max_events=16, + ), + enabled=True, + selection_policy="recent_tail_plus_latest_role_evidence", + ) + + primary = sequence.next("primary") + repair = sequence.next("contract_repair") + + for context in (primary, repair): + self.assertTrue(context.prompt_context_enabled) + self.assertEqual(context.prompt_context_total_events, 100) + self.assertEqual(context.prompt_context_included_events, 16) + self.assertEqual(context.prompt_context_omitted_events, 84) + self.assertEqual(context.prompt_context_recent_events, 8) + self.assertEqual(context.prompt_context_max_events, 16) + self.assertEqual( + context.prompt_context_selection_policy, + "recent_tail_plus_latest_role_evidence", + ) + def test_recorder_writes_privacy_safe_daily_shard_and_rate_snapshot(self): with tempfile.TemporaryDirectory() as tmpdir: paths = RuntimePaths.from_root(tmpdir) @@ -152,6 +247,17 @@ def test_recorder_writes_privacy_safe_daily_shard_and_rate_snapshot(self): sprint_id="sprint-a", goal_id="goal-1", ) + sequence.set_prompt_context_projection( + SimpleNamespace( + total_events=100, + included_events=16, + omitted_events=84, + recent_events=8, + max_events=16, + ), + enabled=True, + selection_policy="recent_tail_plus_latest_role_evidence", + ) now = runtime_now() recorder.record( sequence.next(), @@ -183,6 +289,16 @@ def test_recorder_writes_privacy_safe_daily_shard_and_rate_snapshot(self): self.assertEqual(record["session_id_hash"], hash_session_id("secret-session-id")) self.assertEqual(record["session_mode"], "resume") self.assertEqual(record["goal_id"], "goal-1") + self.assertTrue(record["prompt_context_enabled"]) + self.assertEqual(record["prompt_context_total_events"], 100) + self.assertEqual(record["prompt_context_included_events"], 16) + self.assertEqual(record["prompt_context_omitted_events"], 84) + self.assertEqual(record["prompt_context_recent_events"], 8) + self.assertEqual(record["prompt_context_max_events"], 16) + self.assertEqual( + record["prompt_context_selection_policy"], + "recent_tail_plus_latest_role_evidence", + ) self.assertIsNotNone(record["estimated_cost_usd"]) self.assertEqual(record["rate_card"]["input_per_million_usd"], 2.0) self.assertEqual(shards[0].parent.name, now.date().isoformat()) @@ -273,6 +389,17 @@ def test_aggregation_filters_counts_percentiles_cost_and_invalid_lines(self): request_id="request-1", sprint_id="sprint-a", ) + sequence.set_prompt_context_projection( + SimpleNamespace( + total_events=100, + included_events=16, + omitted_events=84, + recent_events=8, + max_events=16, + ), + enabled=True, + selection_policy="recent_tail_plus_latest_role_evidence", + ) for index, duration in enumerate((100, 200, 300), start=1): recorder.record( sequence.next("primary" if index == 1 else "contract_repair"), @@ -290,7 +417,12 @@ def test_aggregation_filters_counts_percentiles_cost_and_invalid_lines(self): error_category="" if index < 3 else "nonzero_exit", prompt_chars=10, output_chars=5, - usage=ModelUsage.from_values(input_tokens=100, cached_input_tokens=25, output_tokens=20), + usage=ModelUsage.from_values( + input_tokens=100, + cached_input_tokens=25, + output_tokens=20, + tool_calls=index, + ), ) shard = next(paths.model_invocations_dir.rglob("*.jsonl")) with shard.open("a", encoding="utf-8") as handle: @@ -306,18 +438,200 @@ def test_aggregation_filters_counts_percentiles_cost_and_invalid_lines(self): ) self.assertEqual(summary["totals"]["invocation_count"], 3) + self.assertEqual(summary["totals"]["physical_attempt_count"], 3) self.assertEqual(summary["totals"]["logical_call_count"], 1) + self.assertEqual(summary["totals"]["primary_count"], 1) self.assertEqual(summary["totals"]["contract_repair_count"], 2) self.assertEqual(summary["totals"]["failed_count"], 1) + self.assertEqual(summary["totals"]["tool_call_count"], 6) + self.assertEqual(summary["totals"]["tool_call_coverage_percent"], 100.0) self.assertEqual(summary["totals"]["invalid_record_count"], 1) self.assertEqual(summary["tokens"]["input"], 300) + self.assertEqual(summary["tokens"]["uncached_input"], 225) + self.assertEqual( + summary["prompt_context"], + { + "observed_invocation_count": 3, + "enabled_invocation_count": 3, + "eligible_invocation_count": 3, + "compacted_invocation_count": 3, + "total_events": 300, + "included_events": 48, + "omitted_events": 252, + "coverage_percent": 100.0, + "selection_policies": ["recent_tail_plus_latest_role_evidence"], + }, + ) self.assertEqual(summary["latency_ms"]["p50"], 200) self.assertEqual(summary["latency_ms"]["p95"], 300) self.assertEqual(summary["totals"]["token_coverage_percent"], 100.0) self.assertEqual(summary["totals"]["pricing_coverage_percent"], 100.0) self.assertEqual(len(summary["groups"]), 1) + self.assertEqual(summary["groups"][0]["primary_count"], 1) + self.assertEqual(summary["groups"][0]["tool_call_count"], 6) + self.assertEqual(summary["groups"][0]["uncached_input_tokens"], 225) + self.assertEqual(summary["groups"][0]["prompt_context_observed_count"], 3) + self.assertEqual(summary["groups"][0]["prompt_context_compacted_count"], 3) self.assertIn("role\tpurpose", render_model_metrics_summary(summary)) + def test_aggregation_accepts_records_without_optional_projection_or_tool_metadata(self): + with tempfile.TemporaryDirectory() as tmpdir: + paths = RuntimePaths.from_root(tmpdir) + recorder = ModelTelemetryRecorder(paths, "service-planner") + now = runtime_now() + sequence = InvocationSequence( + runtime_identity="service-planner", + role="planner", + purpose="role_task", + ) + recorder.record( + sequence.next(), + provider="codex_cli", + model="gpt-5.5", + reasoning="xhigh", + cli_version="test", + started_at=now, + ended_at=now, + duration_ms=10, + session_id_before=None, + session_id_after="session-1", + status="completed", + exit_code=0, + error_category="", + prompt_chars=10, + output_chars=5, + usage=ModelUsage.from_values( + input_tokens=100, + cached_input_tokens=150, + output_tokens=20, + ), + ) + shard = next(paths.model_invocations_dir.rglob("*.jsonl")) + legacy_record = json.loads(shard.read_text(encoding="utf-8")) + legacy_record.pop("tool_calls") + for key in tuple(legacy_record): + if key.startswith("prompt_context_"): + legacy_record.pop(key) + shard.write_text(json.dumps(legacy_record) + "\n", encoding="utf-8") + + summary = aggregate_model_invocations(paths, hours=1, now=now) + + self.assertEqual(summary["tokens"]["uncached_input"], 0) + self.assertEqual(summary["totals"]["tool_call_count"], 0) + self.assertEqual(summary["totals"]["tool_call_coverage_percent"], 0.0) + self.assertEqual(summary["prompt_context"]["observed_invocation_count"], 0) + self.assertEqual(summary["prompt_context"]["coverage_percent"], 0.0) + + def test_aggregation_hides_partial_cost_totals_and_group_subtotals(self): + with tempfile.TemporaryDirectory() as tmpdir: + paths = RuntimePaths.from_root(tmpdir) + recorder = ModelTelemetryRecorder( + paths, + "service-planner", + TelemetryRuntimeConfig( + rate_cards={ + "codex_cli/gpt-5.5": ModelRateCard( + input_per_million_usd=1.0, + output_per_million_usd=1.0, + ) + } + ), + ) + now = runtime_now() + sequence = InvocationSequence( + runtime_identity="service-planner", + role="planner", + purpose="role_task", + ) + usages = ( + ModelUsage.from_values(input_tokens=100, output_tokens=20), + ModelUsage(), + ) + for index, usage in enumerate(usages): + recorder.record( + sequence.next("primary" if index == 0 else "contract_repair"), + provider="codex_cli", + model="gpt-5.5", + reasoning="xhigh", + cli_version="test", + started_at=now, + ended_at=now, + duration_ms=10, + session_id_before=None, + session_id_after=f"session-{index}", + status="completed", + exit_code=0, + error_category="", + prompt_chars=10, + output_chars=5, + usage=usage, + ) + + summary = aggregate_model_invocations(paths, hours=1, now=now) + + self.assertEqual(summary["totals"]["pricing_coverage_percent"], 50.0) + self.assertIsNone(summary["totals"]["estimated_cost_usd"]) + self.assertEqual(len(summary["groups"]), 1) + self.assertIsNone(summary["groups"][0]["estimated_cost_usd"]) + + def test_aggregation_distinguishes_disabled_eligible_history_from_compaction(self): + with tempfile.TemporaryDirectory() as tmpdir: + paths = RuntimePaths.from_root(tmpdir) + recorder = ModelTelemetryRecorder(paths, "service-planner") + now = runtime_now() + variants = ( + (False, 100, 0), + (True, 16, 84), + ) + for index, (enabled, included_events, omitted_events) in enumerate(variants): + sequence = InvocationSequence( + runtime_identity="service-planner", + role="planner", + purpose=f"variant-{index}", + ) + sequence.set_prompt_context_projection( + SimpleNamespace( + total_events=100, + included_events=included_events, + omitted_events=omitted_events, + recent_events=8, + max_events=16, + ), + enabled=enabled, + selection_policy="recent_tail_plus_latest_role_evidence", + ) + recorder.record( + sequence.next(), + provider="codex_cli", + model="gpt-5.5", + reasoning="xhigh", + cli_version="test", + started_at=now, + ended_at=now, + duration_ms=10, + session_id_before=None, + session_id_after=f"session-{index}", + status="completed", + exit_code=0, + error_category="", + prompt_chars=10, + output_chars=5, + ) + + prompt_context = aggregate_model_invocations( + paths, + hours=1, + now=now, + )["prompt_context"] + + self.assertEqual(prompt_context["observed_invocation_count"], 2) + self.assertEqual(prompt_context["enabled_invocation_count"], 1) + self.assertEqual(prompt_context["eligible_invocation_count"], 2) + self.assertEqual(prompt_context["compacted_invocation_count"], 1) + self.assertEqual(prompt_context["total_events"], 200) + self.assertEqual(prompt_context["included_events"], 116) + self.assertEqual(prompt_context["omitted_events"], 84) + def test_codex_runner_records_native_usage_without_changing_tuple_result(self): with tempfile.TemporaryDirectory() as tmpdir: workspace = Path(tmpdir) @@ -367,6 +681,7 @@ def fake_run(*_args, **_kwargs): self.assertEqual(record["cached_input_tokens"], 3) self.assertEqual(record["output_tokens"], 4) self.assertEqual(record["total_tokens"], 14) + self.assertEqual(record["tool_calls"], 0) self.assertNotIn("private prompt text", json.dumps(record)) def test_role_contract_repair_records_correlated_attempts(self): diff --git a/workflows/orchestration/team_service.py b/workflows/orchestration/team_service.py index e687cac..d124c21 100644 --- a/workflows/orchestration/team_service.py +++ b/workflows/orchestration/team_service.py @@ -60,6 +60,7 @@ update_goal_stop_condition, ) from teams_runtime.shared.config import load_discord_agents_config, load_team_runtime_config +from teams_runtime.runtime.execution_policy import ModelExecutionPolicy from teams_runtime.runtime.role_result_contract import is_restart_repairable_invalid_contract_payload from teams_runtime.workflows.orchestration.relay import ( archive_internal_relay_file, @@ -383,6 +384,7 @@ INITIAL_PHASE_STEP_TODO_FINALIZATION, INITIAL_PHASE_STEPS, SPRINT_ACTIVE_BACKLOG_STATUSES, + apply_initial_plan_confirmation as apply_initial_plan_confirmation_helper, apply_sprint_planning_result as apply_sprint_planning_result_helper, archive_pending_requirement_candidates as archive_pending_requirement_candidates_helper, build_idle_current_sprint_markdown as build_idle_current_sprint_markdown_helper, @@ -947,6 +949,8 @@ def __init__( *, enable_discord_client: bool = True, relay_transport: str = RELAY_TRANSPORT_DISCORD, + model_execution_policy: ModelExecutionPolicy | None = None, + allow_external_research: bool = True, ): if role not in TEAM_ROLES: raise ValueError(f"Unsupported role: {role}") @@ -960,6 +964,8 @@ def __init__( + ", ".join(sorted(VALID_RELAY_TRANSPORTS)) ) self.relay_transport = normalized_relay_transport + self.model_execution_policy = model_execution_policy + self.allow_external_research = bool(allow_external_research) self.discord_config = load_discord_agents_config(self.paths.workspace_root) self.runtime_config = load_team_runtime_config(self.paths.workspace_root) self.agent_utilization_policy = load_agent_utilization_policy(self.paths.workspace_root) @@ -976,6 +982,7 @@ def __init__( runtime_config=self.runtime_config.role_defaults["orchestrator"], session_identity=self._local_runtime_session_identity("parser"), telemetry_config=self.runtime_config.telemetry, + execution_policy=self.model_execution_policy, ) self.goal_sourcer = GoalSourcingRuntime( paths=self.paths, @@ -983,6 +990,7 @@ def __init__( runtime_config=self.runtime_config.role_defaults["orchestrator"], session_identity=self._local_runtime_session_identity("sourcer"), telemetry_config=self.runtime_config.telemetry, + execution_policy=self.model_execution_policy, ) self.version_controller_runtime = RoleAgentRuntime( paths=self.paths, @@ -993,6 +1001,7 @@ def __init__( session_identity=self._local_runtime_session_identity("version_controller"), telemetry_config=self.runtime_config.telemetry, prompt_context_config=self.runtime_config.prompt_context, + execution_policy=self.model_execution_policy, ) self._purge_request_scoped_role_output_files() self._role_runtime_cache: dict[tuple[str, str, str], RoleAgentRuntime] = { @@ -2946,6 +2955,8 @@ def _build_role_runtime( session_identity=session_identity, telemetry_config=self.runtime_config.telemetry, prompt_context_config=self.runtime_config.prompt_context, + allow_external_research=self.allow_external_research, + execution_policy=self.model_execution_policy, ) return RoleAgentRuntime( paths=self.paths, @@ -2955,6 +2966,7 @@ def _build_role_runtime( session_identity=session_identity, telemetry_config=self.runtime_config.telemetry, prompt_context_config=self.runtime_config.prompt_context, + execution_policy=self.model_execution_policy, ) def _runtime_for_role(self, role: str, sprint_id: str) -> RoleAgentRuntime: @@ -3731,18 +3743,14 @@ async def _maybe_handle_initial_plan_feedback( now = utc_now_iso() sprint_id = str(sprint_state.get("sprint_id") or "") if interpreted_intent == "plan_confirm": - confirmation.update( - { - "status": "confirmed", - "confirmed_at": now, - "updated_at": now, - "confirmed_by": build_requester_route(message, envelope, forwarded=forwarded), - "confirmed_message_id": str(message.message_id or "").strip(), - "parser_reason": str(interpreted.params.get("parser_reason") or "").strip(), - "parser_confidence": interpreted_confidence, - } + confirmation = apply_initial_plan_confirmation_helper( + sprint_state, + confirmed_by=build_requester_route(message, envelope, forwarded=forwarded), + message_id=str(message.message_id or "").strip(), + parser_reason=str(interpreted.params.get("parser_reason") or "").strip(), + parser_confidence=interpreted_confidence, + confirmed_at=now, ) - sprint_state["initial_plan_confirmation"] = confirmation self._save_sprint_state(sprint_state) self._append_sprint_event( sprint_id, diff --git a/workflows/sprints/lifecycle.py b/workflows/sprints/lifecycle.py index 0728b74..8096e8c 100644 --- a/workflows/sprints/lifecycle.py +++ b/workflows/sprints/lifecycle.py @@ -1078,6 +1078,38 @@ def confirmed_initial_plan(sprint_state: dict[str, Any]) -> dict[str, Any]: ) +def apply_initial_plan_confirmation( + sprint_state: dict[str, Any], + *, + confirmed_by: dict[str, Any], + message_id: str = "", + parser_reason: str = "", + parser_confidence: str = "high", + confirmed_at: str = "", +) -> dict[str, Any]: + confirmation = ( + dict(sprint_state.get("initial_plan_confirmation") or {}) + if isinstance(sprint_state.get("initial_plan_confirmation"), dict) + else {} + ) + if str(confirmation.get("status") or "").strip().lower() != "pending": + raise ValueError("Initial implementation plan is not awaiting confirmation.") + now = str(confirmed_at or "").strip() or utc_now_iso() + confirmation.update( + { + "status": "confirmed", + "confirmed_at": now, + "updated_at": now, + "confirmed_by": dict(confirmed_by or {}), + "confirmed_message_id": str(message_id or "").strip(), + "parser_reason": str(parser_reason or "").strip(), + "parser_confidence": str(parser_confidence or "").strip() or "high", + } + ) + sprint_state["initial_plan_confirmation"] = confirmation + return confirmation + + def initial_plan_action_records(sprint_state: dict[str, Any]) -> list[dict[str, Any]]: plan = confirmed_initial_plan(sprint_state) return [dict(item) for item in (plan.get("plan_actions") or []) if isinstance(item, dict)] From 8a7a76f8b213d0e9a8e743a722f52ebadce83c42 Mon Sep 17 00:00:00 2001 From: rica-v3 Date: Mon, 27 Jul 2026 21:41:27 +0900 Subject: [PATCH 2/7] runner.py: add full-sprint A/B benchmark pipeline --- adapters/cli/commands.py | 97 ++ benchmarking/__init__.py | 25 + benchmarking/metrics.py | 669 +++++++++++ benchmarking/models.py | 268 +++++ benchmarking/reporting.py | 345 ++++++ benchmarking/runner.py | 455 ++++++++ benchmarking/scenario.py | 502 +++++++++ benchmarking/worker.py | 1405 ++++++++++++++++++++++++ cli.py | 86 ++ tests/test_benchmark_worker_cleanup.py | 236 ++++ tests/test_sprint_benchmark.py | 1130 +++++++++++++++++++ 11 files changed, 5218 insertions(+) create mode 100644 benchmarking/__init__.py create mode 100644 benchmarking/metrics.py create mode 100644 benchmarking/models.py create mode 100644 benchmarking/reporting.py create mode 100644 benchmarking/runner.py create mode 100644 benchmarking/scenario.py create mode 100644 benchmarking/worker.py create mode 100644 tests/test_benchmark_worker_cleanup.py create mode 100644 tests/test_sprint_benchmark.py diff --git a/adapters/cli/commands.py b/adapters/cli/commands.py index 0b94b31..b3b1b51 100644 --- a/adapters/cli/commands.py +++ b/adapters/cli/commands.py @@ -209,6 +209,84 @@ def build_parser( help=workspace_root_help_text, ) + benchmark_parser = subparsers.add_parser( + "benchmark", + help="Run an explicitly opted-in performance benchmark.", + ) + benchmark_subparsers = benchmark_parser.add_subparsers( + dest="benchmark_command", + required=True, + ) + sprint_ab_parser = benchmark_subparsers.add_parser( + "sprint-ab", + help="Compare full sprints with prompt event compaction disabled and enabled.", + ) + sprint_ab_parser.add_argument( + "--live", + action="store_true", + help="Permit live provider calls; also requires TEAMS_RUNTIME_LIVE_BENCHMARK=1.", + ) + sprint_ab_parser.add_argument( + "--runtime-config", + required=True, + help="Path to the deployed team_runtime.yaml or its workspace directory.", + ) + sprint_ab_parser.add_argument( + "--repetitions", + type=int, + default=1, + help="Number of paired runs; execution order alternates AB then BA.", + ) + sprint_ab_parser.add_argument( + "--max-invocations", + type=int, + default=20, + help="Hard physical model-invocation cap per arm.", + ) + sprint_ab_parser.add_argument( + "--call-timeout-seconds", + type=float, + default=300.0, + help="Hard provider-call timeout.", + ) + sprint_ab_parser.add_argument( + "--run-timeout-seconds", + type=float, + default=1800.0, + help="Hard full-arm timeout.", + ) + sprint_ab_parser.add_argument( + "--keep-workspaces", + choices=("none", "failures", "all"), + default="failures", + help="Retain no workspaces, failed workspaces, or every workspace.", + ) + sprint_ab_parser.add_argument( + "--rate-card-file", + default="", + help="Optional YAML rate card used only for estimated-cost reporting.", + ) + sprint_ab_parser.add_argument( + "--output-dir", + default="", + help="Optional parent directory for benchmark artifacts.", + ) + sprint_ab_parser.add_argument( + "--benchmark-id", + default="", + help="Optional stable artifact directory name.", + ) + sprint_ab_parser.add_argument( + "--allow-dirty-source", + action="store_true", + help="Allow a dirty source checkout and record its state hash in provenance.", + ) + sprint_ab_parser.add_argument( + "--json", + action="store_true", + help="Print a machine-readable result summary.", + ) + return parser @@ -237,6 +315,7 @@ def dispatch_main( cmd_goal_cancel: DispatchSyncCallback, default_relay_transport: str, cmd_metrics: DispatchSyncCallback | None = None, + cmd_benchmark_sprint_ab: DispatchSyncCallback | None = None, ) -> int: if args.command == "init": return cmd_init( @@ -341,6 +420,24 @@ def dispatch_main( return cmd_goal_resume(workspace_root) if args.goal_command in {"cancel", "terminate"}: return cmd_goal_cancel(workspace_root) + if args.command == "benchmark" and args.benchmark_command == "sprint-ab": + if cmd_benchmark_sprint_ab is None: + parser.error("Sprint benchmark command is unavailable.") + return 2 + return cmd_benchmark_sprint_ab( + live=bool(getattr(args, "live", False)), + runtime_config=str(getattr(args, "runtime_config", "") or ""), + repetitions=int(getattr(args, "repetitions", 1)), + max_invocations=int(getattr(args, "max_invocations", 20)), + call_timeout_seconds=float(getattr(args, "call_timeout_seconds", 300.0)), + run_timeout_seconds=float(getattr(args, "run_timeout_seconds", 1800.0)), + keep_workspaces=str(getattr(args, "keep_workspaces", "failures") or "failures"), + rate_card_file=str(getattr(args, "rate_card_file", "") or ""), + output_dir=str(getattr(args, "output_dir", "") or ""), + benchmark_id=str(getattr(args, "benchmark_id", "") or ""), + allow_dirty_source=bool(getattr(args, "allow_dirty_source", False)), + as_json=bool(getattr(args, "json", False)), + ) parser.error(f"Unsupported command: {args.command}") return 2 diff --git a/benchmarking/__init__.py b/benchmarking/__init__.py new file mode 100644 index 0000000..4f4944a --- /dev/null +++ b/benchmarking/__init__.py @@ -0,0 +1,25 @@ +"""Repeatable before/after benchmarks for teams_runtime.""" + +from teams_runtime.benchmarking.models import ( + ArmPlan, + BenchmarkOptions, + BenchmarkResult, + BenchmarkWorker, + QualityEvidence, + SprintEvidence, + WorkerContext, + WorkerOutcome, +) +from teams_runtime.benchmarking.runner import run_sprint_ab_benchmark + +__all__ = [ + "ArmPlan", + "BenchmarkOptions", + "BenchmarkResult", + "BenchmarkWorker", + "QualityEvidence", + "SprintEvidence", + "WorkerContext", + "WorkerOutcome", + "run_sprint_ab_benchmark", +] diff --git a/benchmarking/metrics.py b/benchmarking/metrics.py new file mode 100644 index 0000000..c099aff --- /dev/null +++ b/benchmarking/metrics.py @@ -0,0 +1,669 @@ +from __future__ import annotations + +import json +import math +from collections import defaultdict +from pathlib import Path +from typing import Any, Iterable, Mapping + +from teams_runtime.benchmarking.scenario import ( + BENCHMARK_PROMPT_CONTEXT_MAX_EVENTS, + BENCHMARK_PROMPT_CONTEXT_RECENT_EVENTS, +) +from teams_runtime.shared.prompt_context import PROMPT_EVENT_SELECTION_POLICY + + +SAFE_INVOCATION_FIELDS = frozenset( + { + "schema_version", + "invocation_id", + "operation_id", + "logical_call_id", + "attempt_index", + "attempt_kind", + "started_at", + "ended_at", + "duration_ms", + "pid", + "runtime_identity", + "role", + "purpose", + "workflow_step", + "request_id", + "sprint_id", + "todo_id", + "backlog_id", + "goal_id", + "provider", + "model", + "reasoning", + "cli_version", + "session_mode", + "session_id_hash", + "status", + "exit_code", + "error_category", + "prompt_chars", + "output_chars", + "tool_calls", + "input_tokens", + "cached_input_tokens", + "output_tokens", + "reasoning_output_tokens", + "total_tokens", + "usage_source", + "estimated_cost_usd", + "rate_card", + "prompt_context_enabled", + "prompt_context_total_events", + "prompt_context_included_events", + "prompt_context_omitted_events", + "prompt_context_recent_events", + "prompt_context_max_events", + "prompt_context_selection_policy", + "prompt_context", + } +) +_RATE_CARD_FIELDS = frozenset( + { + "input_per_million_usd", + "cached_input_per_million_usd", + "output_per_million_usd", + "per_invocation_usd", + } +) +_PROMPT_CONTEXT_COUNT_FIELDS = ( + "total_events", + "included_events", + "omitted_events", + "recent_events", + "max_events", +) + + +def _non_negative_int(value: Any) -> int | None: + if value is None or isinstance(value, bool): + return None + try: + normalized = int(value) + except (TypeError, ValueError): + return None + return normalized if normalized >= 0 else None + + +def _finite_number(value: Any) -> float | None: + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + normalized = float(value) + return normalized if math.isfinite(normalized) else None + + +def _sanitize_rate_card(value: Any) -> dict[str, float | None] | None: + if not isinstance(value, dict): + return None + sanitized: dict[str, float | None] = {} + for field_name in _RATE_CARD_FIELDS: + raw_value = value.get(field_name) + if raw_value is None: + sanitized[field_name] = None + continue + normalized = _finite_number(raw_value) + if normalized is not None and normalized >= 0: + sanitized[field_name] = normalized + return sanitized or None + + +def _sanitize_prompt_context(value: Any) -> dict[str, Any] | None: + if not isinstance(value, dict): + return None + sanitized: dict[str, Any] = {} + if isinstance(value.get("enabled"), bool): + sanitized["enabled"] = value["enabled"] + if isinstance(value.get("compacted"), bool): + sanitized["compacted"] = value["compacted"] + for field_name in _PROMPT_CONTEXT_COUNT_FIELDS: + normalized = _non_negative_int(value.get(field_name)) + if normalized is not None: + sanitized[field_name] = normalized + selection_policy = str(value.get("selection_policy") or "").strip() + if selection_policy == PROMPT_EVENT_SELECTION_POLICY: + sanitized["selection_policy"] = selection_policy + return sanitized or None + + +def sanitize_invocation_record(record: Mapping[str, Any]) -> dict[str, Any]: + sanitized = { + key: value + for key, value in record.items() + if key in SAFE_INVOCATION_FIELDS + } + rate_card = _sanitize_rate_card(sanitized.get("rate_card")) + if rate_card is None: + sanitized.pop("rate_card", None) + else: + sanitized["rate_card"] = rate_card + context = _sanitize_prompt_context(sanitized.get("prompt_context")) + if context is None: + sanitized.pop("prompt_context", None) + else: + sanitized["prompt_context"] = context + if not isinstance(sanitized.get("prompt_context_enabled"), bool): + sanitized["prompt_context_enabled"] = None + for field_name in ( + "prompt_context_total_events", + "prompt_context_included_events", + "prompt_context_omitted_events", + "prompt_context_recent_events", + "prompt_context_max_events", + ): + sanitized[field_name] = _non_negative_int(sanitized.get(field_name)) + if ( + str(sanitized.get("prompt_context_selection_policy") or "").strip() + != PROMPT_EVENT_SELECTION_POLICY + ): + sanitized["prompt_context_selection_policy"] = "" + return sanitized + + +def load_workspace_telemetry(workspace_root: Path) -> tuple[dict[str, Any], ...]: + records: list[dict[str, Any]] = [] + metrics_root = workspace_root / ".teams_runtime" / "metrics" / "model_invocations" + if not metrics_root.is_dir(): + return () + for shard in sorted(metrics_root.rglob("*.jsonl")): + try: + lines = shard.read_text(encoding="utf-8").splitlines() + except OSError: + continue + for line in lines: + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(record, dict): + records.append(sanitize_invocation_record(record)) + records.sort( + key=lambda item: ( + str(item.get("started_at") or ""), + str(item.get("invocation_id") or ""), + ) + ) + return tuple(records) + + +def _nearest_rank(values: list[int], percentile: float) -> int: + if not values: + return 0 + ordered = sorted(values) + index = max(math.ceil(percentile * len(ordered)) - 1, 0) + return ordered[index] + + +def _prompt_context(record: Mapping[str, Any]) -> dict[str, Any]: + nested = record.get("prompt_context") + if isinstance(nested, dict): + return dict(nested) + return { + "enabled": record.get("prompt_context_enabled"), + "total_events": record.get("prompt_context_total_events"), + "included_events": record.get("prompt_context_included_events"), + "omitted_events": record.get("prompt_context_omitted_events"), + "recent_events": record.get("prompt_context_recent_events"), + "max_events": record.get("prompt_context_max_events"), + "selection_policy": record.get("prompt_context_selection_policy"), + } + + +def reduce_telemetry(records: Iterable[Mapping[str, Any]]) -> dict[str, Any]: + normalized = [sanitize_invocation_record(record) for record in records] + logical_calls = { + str(record.get("logical_call_id") or "") + for record in normalized + if str(record.get("logical_call_id") or "") + } + durations: list[int] = [] + totals = { + "invocation_count": len(normalized), + "logical_call_count": len(logical_calls), + "primary_count": 0, + "contract_repair_count": 0, + "sandbox_retry_count": 0, + "completed_count": 0, + "failed_count": 0, + "tool_call_count": 0, + "prompt_chars": 0, + "output_chars": 0, + } + tokens = { + "input": 0, + "cached_input": 0, + "uncached_input": 0, + "output": 0, + "reasoning_output": 0, + "total": 0, + } + native_usage_count = 0 + tool_call_usage_count = 0 + priced_count = 0 + total_cost = 0.0 + compaction = { + "observed_invocation_count": 0, + "unobserved_invocation_count": 0, + "enabled_invocation_count": 0, + "eligible_invocation_count": 0, + "disabled_eligible_invocation_count": 0, + "compacted_invocation_count": 0, + "invalid_projection_count": 0, + "total_events": 0, + "included_events": 0, + "omitted_events": 0, + "max_observed_events": 0, + "max_included_events": 0, + "expected_recent_events": BENCHMARK_PROMPT_CONTEXT_RECENT_EVENTS, + "expected_max_events": BENCHMARK_PROMPT_CONTEXT_MAX_EVENTS, + "expected_selection_policy": PROMPT_EVENT_SELECTION_POLICY, + "selection_policies": [], + } + selection_policies: set[str] = set() + groups: dict[tuple[str, str, str, str], dict[str, Any]] = {} + for record in normalized: + attempt_kind = str(record.get("attempt_kind") or "") + if attempt_kind == "primary": + totals["primary_count"] += 1 + elif attempt_kind == "contract_repair": + totals["contract_repair_count"] += 1 + elif attempt_kind == "sandbox_retry": + totals["sandbox_retry_count"] += 1 + if str(record.get("status") or "") == "completed": + totals["completed_count"] += 1 + else: + totals["failed_count"] += 1 + duration = _non_negative_int(record.get("duration_ms")) or 0 + durations.append(duration) + tool_calls = _non_negative_int(record.get("tool_calls")) + if tool_calls is not None: + tool_call_usage_count += 1 + totals["tool_call_count"] += tool_calls + totals["prompt_chars"] += _non_negative_int(record.get("prompt_chars")) or 0 + totals["output_chars"] += _non_negative_int(record.get("output_chars")) or 0 + raw_input_tokens = _non_negative_int(record.get("input_tokens")) + raw_cached_tokens = _non_negative_int(record.get("cached_input_tokens")) + raw_output_tokens = _non_negative_int(record.get("output_tokens")) + raw_total_tokens = _non_negative_int(record.get("total_tokens")) + input_tokens = raw_input_tokens or 0 + cached_tokens = min(raw_cached_tokens or 0, input_tokens) + output_tokens = raw_output_tokens or 0 + tokens["input"] += input_tokens + tokens["cached_input"] += cached_tokens + tokens["uncached_input"] += max(input_tokens - cached_tokens, 0) + tokens["output"] += output_tokens + tokens["reasoning_output"] += _non_negative_int(record.get("reasoning_output_tokens")) or 0 + effective_total_tokens = ( + raw_total_tokens + if raw_total_tokens is not None + else input_tokens + output_tokens + ) + tokens["total"] += effective_total_tokens + complete_native_usage = ( + str(record.get("usage_source") or "") == "native" + and raw_input_tokens is not None + and raw_output_tokens is not None + and effective_total_tokens >= raw_input_tokens + raw_output_tokens + ) + if complete_native_usage: + native_usage_count += 1 + cost = _finite_number(record.get("estimated_cost_usd")) + if cost is not None: + priced_count += 1 + total_cost += cost + context = _prompt_context(record) + total_events = _non_negative_int(context.get("total_events")) + included_events = _non_negative_int(context.get("included_events")) + omitted_events = _non_negative_int(context.get("omitted_events")) + recent_events = _non_negative_int(context.get("recent_events")) + max_events = _non_negative_int(context.get("max_events")) + enabled = context.get("enabled") + selection_policy = str(context.get("selection_policy") or "").strip() + projection_candidate = isinstance(enabled, bool) or any( + value is not None + for value in ( + total_events, + included_events, + omitted_events, + recent_events, + max_events, + ) + ) or bool(selection_policy) + projection_observed = isinstance(enabled, bool) and all( + value is not None + for value in ( + total_events, + included_events, + omitted_events, + recent_events, + max_events, + ) + ) + projection_valid = False + if projection_observed: + compaction["observed_invocation_count"] += 1 + compaction["enabled_invocation_count"] += int(enabled) + if selection_policy: + selection_policies.add(selection_policy) + projection_valid = ( + max_events == BENCHMARK_PROMPT_CONTEXT_MAX_EVENTS + and recent_events == BENCHMARK_PROMPT_CONTEXT_RECENT_EVENTS + and total_events == included_events + omitted_events + and selection_policy == PROMPT_EVENT_SELECTION_POLICY + and ( + ( + enabled + and ( + (total_events <= max_events and included_events == total_events and omitted_events == 0) + or ( + total_events > max_events + and recent_events <= included_events <= max_events + and omitted_events > 0 + ) + ) + ) + or ( + not enabled + and included_events == total_events + and omitted_events == 0 + ) + ) + ) + if not projection_valid: + compaction["invalid_projection_count"] += 1 + elif projection_candidate: + compaction["invalid_projection_count"] += 1 + if projection_valid and total_events is not None: + compaction["max_observed_events"] = max(compaction["max_observed_events"], total_events) + compaction["total_events"] += total_events + if max_events is not None and total_events > max_events: + compaction["eligible_invocation_count"] += 1 + if not enabled: + compaction["disabled_eligible_invocation_count"] += 1 + if projection_valid and included_events is not None: + compaction["included_events"] += included_events + compaction["max_included_events"] = max( + compaction["max_included_events"], + included_events, + ) + if projection_valid and omitted_events is not None: + compaction["omitted_events"] += omitted_events + if enabled and omitted_events > 0: + compaction["compacted_invocation_count"] += 1 + + key = tuple( + str(record.get(field_name) or "") + for field_name in ("role", "purpose", "provider", "model") + ) + group = groups.setdefault( + key, + { + "role": key[0], + "purpose": key[1], + "provider": key[2], + "model": key[3], + "invocation_count": 0, + "failed_count": 0, + "input_tokens": 0, + "cached_input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "duration_ms": 0, + "estimated_cost_usd": None, + "_priced_count": 0, + }, + ) + group["invocation_count"] += 1 + group["failed_count"] += int(str(record.get("status") or "") != "completed") + group["input_tokens"] += input_tokens + group["cached_input_tokens"] += cached_tokens + group["output_tokens"] += output_tokens + group["total_tokens"] += effective_total_tokens + group["duration_ms"] += duration + if cost is not None: + group["_priced_count"] += 1 + group["estimated_cost_usd"] = round( + (group["estimated_cost_usd"] or 0.0) + cost, + 12, + ) + + count = len(normalized) + for group in groups.values(): + if group.pop("_priced_count") != group["invocation_count"]: + group["estimated_cost_usd"] = None + token_coverage = round(native_usage_count * 100 / count, 2) if count else 0.0 + tool_call_coverage = round(tool_call_usage_count * 100 / count, 2) if count else 0.0 + pricing_coverage = round(priced_count * 100 / count, 2) if count else 0.0 + totals["token_coverage_percent"] = token_coverage + totals["tool_call_coverage_percent"] = tool_call_coverage + totals["pricing_coverage_percent"] = pricing_coverage + totals["estimated_cost_usd"] = ( + round(total_cost, 12) if count and priced_count == count else None + ) + compaction["unobserved_invocation_count"] = ( + count - compaction["observed_invocation_count"] + ) + compaction["selection_policies"] = sorted(selection_policies) + return { + "totals": totals, + "tokens": tokens, + "latency_ms": { + "provider_total": sum(durations), + "p50": _nearest_rank(durations, 0.50), + "p95": _nearest_rank(durations, 0.95), + "max": max(durations, default=0), + }, + "compaction": compaction, + "groups": sorted( + groups.values(), + key=lambda item: ( + -int(item["total_tokens"]), + -int(item["duration_ms"]), + item["role"], + item["purpose"], + ), + ), + } + + +def _primary_groups( + records: Iterable[Mapping[str, Any]], +) -> dict[tuple[str, str, str], list[Mapping[str, Any]]]: + grouped: defaultdict[ + tuple[str, str, str], + list[Mapping[str, Any]], + ] = defaultdict(list) + for record in records: + if str(record.get("attempt_kind") or "") != "primary": + continue + base = tuple( + str(record.get(field_name) or "") + for field_name in ("role", "purpose", "workflow_step") + ) + grouped[base].append(record) + return dict(grouped) + + +def _primary_identity(record: Mapping[str, Any]) -> tuple[Any, ...]: + context = _prompt_context(record) + return ( + str(record.get("provider") or ""), + str(record.get("model") or ""), + str(record.get("reasoning") or ""), + str(record.get("status") or ""), + _non_negative_int(context.get("total_events")), + ) + + +def _delta(before: float | int | None, after: float | int | None) -> dict[str, Any]: + if before is None or after is None: + return { + "before": before, + "after": after, + "delta": None, + "change_percent": None, + "reduction": None, + "reduction_percent": None, + } + delta = after - before + reduction = before - after + return { + "before": before, + "after": after, + "delta": delta, + "change_percent": round(delta * 100 / before, 4) if before else None, + "reduction": reduction, + "reduction_percent": round(reduction * 100 / before, 4) if before else None, + } + + +def compare_metrics( + before_metrics: Mapping[str, Any], + after_metrics: Mapping[str, Any], + *, + before_wall_duration_ms: int, + after_wall_duration_ms: int, + before_records: Iterable[Mapping[str, Any]], + after_records: Iterable[Mapping[str, Any]], +) -> dict[str, Any]: + before_totals = dict(before_metrics.get("totals") or {}) + after_totals = dict(after_metrics.get("totals") or {}) + before_tokens = dict(before_metrics.get("tokens") or {}) + after_tokens = dict(after_metrics.get("tokens") or {}) + before_latency = dict(before_metrics.get("latency_ms") or {}) + after_latency = dict(after_metrics.get("latency_ms") or {}) + end_to_end = { + "invocation_count": _delta( + before_totals.get("invocation_count"), + after_totals.get("invocation_count"), + ), + "logical_call_count": _delta( + before_totals.get("logical_call_count"), + after_totals.get("logical_call_count"), + ), + "contract_repair_count": _delta( + before_totals.get("contract_repair_count"), + after_totals.get("contract_repair_count"), + ), + "sandbox_retry_count": _delta( + before_totals.get("sandbox_retry_count"), + after_totals.get("sandbox_retry_count"), + ), + "failed_count": _delta( + before_totals.get("failed_count"), + after_totals.get("failed_count"), + ), + "tool_call_count": _delta( + before_totals.get("tool_call_count"), + after_totals.get("tool_call_count"), + ), + "prompt_chars": _delta( + before_totals.get("prompt_chars"), + after_totals.get("prompt_chars"), + ), + "input_tokens": _delta( + before_tokens.get("input"), + after_tokens.get("input"), + ), + "cached_input_tokens": _delta( + before_tokens.get("cached_input"), + after_tokens.get("cached_input"), + ), + "uncached_input_tokens": _delta( + before_tokens.get("uncached_input"), + after_tokens.get("uncached_input"), + ), + "output_tokens": _delta( + before_tokens.get("output"), + after_tokens.get("output"), + ), + "reasoning_output_tokens": _delta( + before_tokens.get("reasoning_output"), + after_tokens.get("reasoning_output"), + ), + "total_tokens": _delta( + before_tokens.get("total"), + after_tokens.get("total"), + ), + "provider_duration_ms": _delta( + before_latency.get("provider_total"), + after_latency.get("provider_total"), + ), + "wall_duration_ms": _delta(before_wall_duration_ms, after_wall_duration_ms), + "estimated_cost_usd": _delta( + before_totals.get("estimated_cost_usd"), + after_totals.get("estimated_cost_usd"), + ), + } + + before_primary = _primary_groups(before_records) + after_primary = _primary_groups(after_records) + matched: list[dict[str, Any]] = [] + unmatched_before = 0 + unmatched_after = 0 + ambiguous_groups = 0 + for key in sorted(before_primary.keys() | after_primary.keys()): + before_group = before_primary.get(key, []) + after_group = after_primary.get(key, []) + unambiguous = ( + len(before_group) == 1 + and len(after_group) == 1 + and _primary_identity(before_group[0]) == _primary_identity(after_group[0]) + ) + if not unambiguous: + unmatched_before += len(before_group) + unmatched_after += len(after_group) + ambiguous_groups += int(bool(before_group) and bool(after_group)) + continue + before = before_group[0] + after = after_group[0] + matched.append( + { + "role": key[0], + "purpose": key[1], + "workflow_step": key[2], + "occurrence": 1, + "prompt_chars": _delta( + _non_negative_int(before.get("prompt_chars")), + _non_negative_int(after.get("prompt_chars")), + ), + "input_tokens": _delta( + _non_negative_int(before.get("input_tokens")), + _non_negative_int(after.get("input_tokens")), + ), + "cached_input_tokens": _delta( + _non_negative_int(before.get("cached_input_tokens")), + _non_negative_int(after.get("cached_input_tokens")), + ), + "total_tokens": _delta( + _non_negative_int(before.get("total_tokens")), + _non_negative_int(after.get("total_tokens")), + ), + "duration_ms": _delta( + _non_negative_int(before.get("duration_ms")), + _non_negative_int(after.get("duration_ms")), + ), + } + ) + return { + "end_to_end": end_to_end, + "matched_primary_invocations": matched, + "matched_primary_count": len(matched), + "unmatched_before_primary_count": unmatched_before, + "unmatched_after_primary_count": unmatched_after, + "ambiguous_primary_group_count": ambiguous_groups, + } + + +__all__ = [ + "SAFE_INVOCATION_FIELDS", + "compare_metrics", + "load_workspace_telemetry", + "reduce_telemetry", + "sanitize_invocation_record", +] diff --git a/benchmarking/models.py b/benchmarking/models.py new file mode 100644 index 0000000..1fd67a9 --- /dev/null +++ b/benchmarking/models.py @@ -0,0 +1,268 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Literal, Mapping, Protocol, Sequence + + +BenchmarkVariant = Literal["before", "after"] +KeepWorkspaces = Literal["none", "failures", "all"] +RunStatus = Literal[ + "completed", + "failed", + "timeout", + "call_budget_exhausted", + "preflight_failed", +] + + +class BenchmarkWorkerSafetyError(RuntimeError): + """Raised when benchmark worker isolation cannot be proven safe.""" + + +@dataclass(slots=True, frozen=True) +class BenchmarkOptions: + """Operator-selected controls for a sprint A/B benchmark.""" + + source_root: Path + runtime_config_path: Path + output_dir: Path | None = None + rate_card_path: Path | None = None + repetitions: int = 1 + max_invocations: int = 20 + call_timeout_seconds: float = 300.0 + run_timeout_seconds: float = 1800.0 + keep_workspaces: KeepWorkspaces = "failures" + allow_dirty_source: bool = False + live: bool = False + benchmark_id: str = "" + + def validate(self) -> None: + if self.repetitions <= 0: + raise ValueError("repetitions must be a positive integer") + if self.max_invocations <= 0: + raise ValueError("max_invocations must be a positive integer") + if self.call_timeout_seconds <= 0: + raise ValueError("call_timeout_seconds must be positive") + if self.run_timeout_seconds <= 0: + raise ValueError("run_timeout_seconds must be positive") + if self.keep_workspaces not in {"none", "failures", "all"}: + raise ValueError("keep_workspaces must be none, failures, or all") + if not self.source_root.expanduser().is_dir(): + raise FileNotFoundError(f"Source root does not exist: {self.source_root}") + config_path = self.runtime_config_path.expanduser() + if not config_path.exists(): + raise FileNotFoundError(f"Runtime config does not exist: {config_path}") + if self.rate_card_path is not None and not self.rate_card_path.expanduser().is_file(): + raise FileNotFoundError(f"Rate card does not exist: {self.rate_card_path}") + + +@dataclass(slots=True, frozen=True) +class ArmPlan: + pair_index: int + order_index: int + variant: BenchmarkVariant + run_id: str + prompt_context_enabled: bool + + +@dataclass(slots=True, frozen=True) +class SprintEvidence: + sprint_id: str = "" + status: str = "" + closeout_status: str = "" + todo_count: int = 0 + completed_todo_count: int = 0 + blocked_todo_count: int = 0 + failed_todo_count: int = 0 + commit_sha: str = "" + + def to_dict(self) -> dict[str, Any]: + return { + "sprint_id": self.sprint_id, + "status": self.status, + "closeout_status": self.closeout_status, + "todo_count": self.todo_count, + "completed_todo_count": self.completed_todo_count, + "blocked_todo_count": self.blocked_todo_count, + "failed_todo_count": self.failed_todo_count, + "commit_sha": self.commit_sha, + } + + +@dataclass(slots=True, frozen=True) +class QualityEvidence: + behavior_oracle_passed: bool = False + sprint_terminal: bool = False + closeout_verified: bool = False + protected_files_unchanged: bool = False + git_clean: bool = False + commit_created: bool = False + no_git_remotes: bool = False + blocked_todo_count: int = 0 + failed_todo_count: int = 0 + notes: tuple[str, ...] = () + + @property + def passed(self) -> bool: + return ( + self.behavior_oracle_passed + and self.sprint_terminal + and self.closeout_verified + and self.protected_files_unchanged + and self.git_clean + and self.commit_created + and self.no_git_remotes + and self.blocked_todo_count == 0 + and self.failed_todo_count == 0 + ) + + def to_dict(self) -> dict[str, Any]: + return { + "passed": self.passed, + "behavior_oracle_passed": self.behavior_oracle_passed, + "sprint_terminal": self.sprint_terminal, + "closeout_verified": self.closeout_verified, + "protected_files_unchanged": self.protected_files_unchanged, + "git_clean": self.git_clean, + "commit_created": self.commit_created, + "no_git_remotes": self.no_git_remotes, + "blocked_todo_count": self.blocked_todo_count, + "failed_todo_count": self.failed_todo_count, + "notes": list(self.notes), + } + + +@dataclass(slots=True, frozen=True) +class WorkerContext: + """Everything an orchestration worker needs for one isolated arm.""" + + benchmark_id: str + arm: ArmPlan + workspace_root: Path + run_output_dir: Path + milestone: str + history_seed: tuple[Mapping[str, Any], ...] + max_invocations: int + call_timeout_seconds: float + run_timeout_seconds: float + live: bool + + +@dataclass(slots=True, frozen=True) +class WorkerOutcome: + """Privacy-safe result returned by an injected sprint worker.""" + + status: RunStatus + sprint: SprintEvidence = field(default_factory=SprintEvidence) + quality: QualityEvidence = field(default_factory=QualityEvidence) + telemetry_records: tuple[Mapping[str, Any], ...] = () + started_at: str = "" + ended_at: str = "" + wall_duration_ms: int = 0 + worker_duration_ms: int = 0 + stop_reason: str = "" + error_category: str = "" + + +class BenchmarkWorker(Protocol): + def __call__(self, context: WorkerContext) -> WorkerOutcome: + """Run one complete sprint arm and return privacy-safe evidence.""" + + +@dataclass(slots=True, frozen=True) +class ArmResult: + arm: ArmPlan + status: RunStatus + started_at: str + ended_at: str + wall_duration_ms: int + worker_duration_ms: int + stop_reason: str + error_category: str + config_hash: str + comparable_config_hash: str + metrics: Mapping[str, Any] + quality: QualityEvidence + sprint: SprintEvidence + invocation_records: tuple[Mapping[str, Any], ...] = () + retained_workspace: str = "" + + def to_dict(self, *, include_records: bool = False) -> dict[str, Any]: + result = { + "run_id": self.arm.run_id, + "pair_index": self.arm.pair_index, + "order_index": self.arm.order_index, + "variant": self.arm.variant, + "prompt_context_enabled": self.arm.prompt_context_enabled, + "status": self.status, + "started_at": self.started_at, + "ended_at": self.ended_at, + "wall_duration_ms": self.wall_duration_ms, + "worker_duration_ms": self.worker_duration_ms, + "stop_reason": self.stop_reason, + "error_category": self.error_category, + "config_hash": self.config_hash, + "comparable_config_hash": self.comparable_config_hash, + "metrics": dict(self.metrics), + "quality": self.quality.to_dict(), + "sprint": self.sprint.to_dict(), + "retained_workspace": self.retained_workspace, + } + if include_records: + result["invocations"] = [dict(record) for record in self.invocation_records] + return result + + +@dataclass(slots=True, frozen=True) +class BenchmarkResult: + benchmark_id: str + status: Literal["comparable", "inconclusive"] + classification: Literal["preliminary_smoke", "repeated_experiment"] + output_dir: Path + report_json: Path + report_markdown: Path + runs: tuple[ArmResult, ...] + report: Mapping[str, Any] + + @property + def exit_code(self) -> int: + return 0 if self.status == "comparable" else 1 + + +def make_arm_schedule(repetitions: int) -> tuple[ArmPlan, ...]: + if repetitions <= 0: + raise ValueError("repetitions must be a positive integer") + schedule: list[ArmPlan] = [] + order_index = 0 + for pair_index in range(1, repetitions + 1): + variants: Sequence[BenchmarkVariant] = ( + ("before", "after") if pair_index % 2 else ("after", "before") + ) + for variant in variants: + order_index += 1 + schedule.append( + ArmPlan( + pair_index=pair_index, + order_index=order_index, + variant=variant, + run_id=f"pair-{pair_index:03d}-{variant}", + prompt_context_enabled=variant == "after", + ) + ) + return tuple(schedule) + + +__all__ = [ + "ArmPlan", + "ArmResult", + "BenchmarkOptions", + "BenchmarkResult", + "BenchmarkWorker", + "BenchmarkWorkerSafetyError", + "QualityEvidence", + "SprintEvidence", + "WorkerContext", + "WorkerOutcome", + "make_arm_schedule", +] diff --git a/benchmarking/reporting.py b/benchmarking/reporting.py new file mode 100644 index 0000000..7c11ceb --- /dev/null +++ b/benchmarking/reporting.py @@ -0,0 +1,345 @@ +from __future__ import annotations + +import json +import os +import statistics +import tempfile +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Iterable, Mapping + +from teams_runtime.benchmarking.metrics import compare_metrics +from teams_runtime.benchmarking.models import ArmResult, BenchmarkOptions +from teams_runtime.benchmarking.scenario import ( + BENCHMARK_PROMPT_CONTEXT_MAX_EVENTS, + BENCHMARK_PROMPT_CONTEXT_RECENT_EVENTS, + DEFAULT_HISTORY_SEED_COUNT, +) +from teams_runtime.shared.prompt_context import PROMPT_EVENT_SELECTION_POLICY + + +REPORT_SCHEMA_VERSION = 1 + + +def utc_now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def write_text_atomic(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + file_descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{path.name}.", + suffix=".tmp", + dir=path.parent, + text=True, + ) + temporary_path = Path(temporary_name) + try: + with os.fdopen(file_descriptor, "w", encoding="utf-8") as handle: + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary_path, path) + finally: + temporary_path.unlink(missing_ok=True) + + +def write_json_atomic(path: Path, payload: Mapping[str, Any]) -> None: + write_text_atomic( + path, + json.dumps(payload, ensure_ascii=True, indent=2, sort_keys=True) + "\n", + ) + + +def write_jsonl_atomic(path: Path, records: Iterable[Mapping[str, Any]]) -> None: + content = "".join( + json.dumps(record, ensure_ascii=True, sort_keys=True, separators=(",", ":")) + "\n" + for record in records + ) + write_text_atomic(path, content) + + +def write_run_artifacts(run_dir: Path, result: ArmResult) -> None: + run_dir.mkdir(parents=True, exist_ok=True) + write_json_atomic(run_dir / "run.json", result.to_dict()) + write_json_atomic(run_dir / "metrics.json", dict(result.metrics)) + write_json_atomic(run_dir / "sprint.json", result.sprint.to_dict()) + write_json_atomic(run_dir / "quality.json", result.quality.to_dict()) + write_jsonl_atomic(run_dir / "model_invocations.jsonl", result.invocation_records) + + +def _pair_comparability(before: ArmResult, after: ArmResult) -> tuple[bool, list[str]]: + reasons: list[str] = [] + if before.comparable_config_hash != after.comparable_config_hash: + reasons.append("non_feature_configuration_differs") + for label, arm in (("before", before), ("after", after)): + if arm.status != "completed": + reasons.append(f"{label}_status_{arm.status}") + if not arm.quality.passed: + reasons.append(f"{label}_quality_failed") + totals = dict(arm.metrics.get("totals") or {}) + if float(totals.get("token_coverage_percent") or 0.0) != 100.0: + reasons.append(f"{label}_native_token_coverage_incomplete") + compaction = dict(arm.metrics.get("compaction") or {}) + if int(compaction.get("invalid_projection_count") or 0): + reasons.append(f"{label}_prompt_projection_invalid") + if int(compaction.get("max_observed_events") or 0) < DEFAULT_HISTORY_SEED_COUNT: + reasons.append(f"{label}_backfill_not_observed") + if compaction.get("selection_policies") != [PROMPT_EVENT_SELECTION_POLICY]: + reasons.append(f"{label}_selection_policy_unverified") + before_compaction = dict(before.metrics.get("compaction") or {}) + after_compaction = dict(after.metrics.get("compaction") or {}) + if int(before_compaction.get("enabled_invocation_count") or 0): + reasons.append("before_compaction_unexpectedly_enabled") + if int(before_compaction.get("eligible_invocation_count") or 0) <= 0: + reasons.append("before_compaction_eligibility_not_observed") + if int(before_compaction.get("disabled_eligible_invocation_count") or 0) <= 0: + reasons.append("before_disabled_projection_not_observed") + if int(before_compaction.get("compacted_invocation_count") or 0): + reasons.append("before_compaction_unexpectedly_observed") + if int(after_compaction.get("enabled_invocation_count") or 0) <= 0: + reasons.append("after_compaction_not_enabled") + if int(after_compaction.get("enabled_invocation_count") or 0) != int( + after_compaction.get("observed_invocation_count") or 0 + ): + reasons.append("after_prompt_projection_not_uniformly_enabled") + if int(after_compaction.get("disabled_eligible_invocation_count") or 0): + reasons.append("after_disabled_projection_observed") + if int(after_compaction.get("compacted_invocation_count") or 0) <= 0: + reasons.append("after_compaction_not_observed") + return not reasons, reasons + + +def _build_pairs(runs: tuple[ArmResult, ...]) -> list[dict[str, Any]]: + pair_indexes = sorted({run.arm.pair_index for run in runs}) + pairs: list[dict[str, Any]] = [] + for pair_index in pair_indexes: + pair_runs = { + run.arm.variant: run + for run in runs + if run.arm.pair_index == pair_index + } + before = pair_runs.get("before") + after = pair_runs.get("after") + if before is None or after is None: + pairs.append( + { + "pair_index": pair_index, + "comparable": False, + "inconclusive_reasons": ["missing_arm"], + } + ) + continue + comparable, reasons = _pair_comparability(before, after) + comparison = compare_metrics( + before.metrics, + after.metrics, + before_wall_duration_ms=before.wall_duration_ms, + after_wall_duration_ms=after.wall_duration_ms, + before_records=before.invocation_records, + after_records=after.invocation_records, + ) + pairs.append( + { + "pair_index": pair_index, + "execution_order": [ + run.arm.variant + for run in sorted((before, after), key=lambda item: item.arm.order_index) + ], + "before_run_id": before.arm.run_id, + "after_run_id": after.arm.run_id, + "comparable": comparable, + "inconclusive_reasons": reasons, + "comparison": comparison, + } + ) + return pairs + + +def _aggregate_pair_metrics(pairs: list[dict[str, Any]]) -> dict[str, Any]: + comparable_pairs = [pair for pair in pairs if pair.get("comparable")] + metric_values: dict[str, list[float]] = {} + for pair in comparable_pairs: + end_to_end = dict((pair.get("comparison") or {}).get("end_to_end") or {}) + for metric_name, delta_payload in end_to_end.items(): + reduction = (delta_payload or {}).get("reduction") + if isinstance(reduction, (int, float)) and not isinstance(reduction, bool): + metric_values.setdefault(metric_name, []).append(float(reduction)) + result: dict[str, Any] = {} + for metric_name, values in sorted(metric_values.items()): + result[metric_name] = { + "pair_count": len(values), + "mean_reduction": statistics.fmean(values), + "median_reduction": statistics.median(values), + "sample_standard_deviation": ( + statistics.stdev(values) if len(values) > 1 else None + ), + } + return result + + +def build_report( + *, + benchmark_id: str, + options: BenchmarkOptions, + source_revision: Mapping[str, Any], + source_config_hash: str, + runtime_model_map: Mapping[str, Mapping[str, str]], + rate_cards: Mapping[str, Mapping[str, float | None]], + history_hash: str, + runs: tuple[ArmResult, ...], + started_at: str, + ended_at: str, +) -> dict[str, Any]: + pairs = _build_pairs(runs) + comparable = bool(pairs) and all(pair.get("comparable") for pair in pairs) + status = "comparable" if comparable else "inconclusive" + return { + "schema_version": REPORT_SCHEMA_VERSION, + "benchmark_id": benchmark_id, + "benchmark": "sprint_ab", + "classification": ( + "preliminary_smoke" if options.repetitions == 1 else "repeated_experiment" + ), + "status": status, + "started_at": started_at, + "ended_at": ended_at, + "provenance": { + "source": dict(source_revision), + "source_config_hash": source_config_hash, + "history_hash": history_hash, + "runtime_model_map": { + role: dict(values) + for role, values in runtime_model_map.items() + }, + "rate_cards": { + key: dict(values) + for key, values in rate_cards.items() + }, + }, + "controls": { + "repetitions": options.repetitions, + "max_invocations_per_arm": options.max_invocations, + "call_timeout_seconds": options.call_timeout_seconds, + "run_timeout_seconds": options.run_timeout_seconds, + "keep_workspaces": options.keep_workspaces, + "live": options.live, + "a_b_definition": { + "before": {"prompt_context_enabled": False}, + "after": { + "prompt_context_enabled": True, + "recent_events": BENCHMARK_PROMPT_CONTEXT_RECENT_EVENTS, + "max_events": BENCHMARK_PROMPT_CONTEXT_MAX_EVENTS, + }, + }, + }, + "runs": [run.to_dict() for run in runs], + "pairs": pairs, + "aggregate_reductions": _aggregate_pair_metrics(pairs), + "interpretation": { + "statistical_significance_claimed": False, + "note": ( + "A one-pair run is preliminary. Full-sprint model routing is " + "nondeterministic, so end-to-end deltas are not solely attributable " + "to prompt compaction." + ), + }, + } + + +def _display(value: Any, *, missing: str = "N/A") -> str: + if value is None: + return missing + if isinstance(value, float): + return f"{value:.4f}" + return str(value) + + +def render_markdown(report: Mapping[str, Any]) -> str: + lines = [ + "# Sprint Performance Benchmark", + "", + f"- Benchmark: `{report.get('benchmark_id', '')}`", + f"- Status: **{report.get('status', 'inconclusive')}**", + f"- Classification: `{report.get('classification', '')}`", + f"- Started: `{report.get('started_at', '')}`", + f"- Ended: `{report.get('ended_at', '')}`", + "", + "## Runs", + "", + "| Run | Variant | Status | Calls | Repairs | Input tokens | Total tokens | Wall ms | Quality |", + "| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | --- |", + ] + for run in report.get("runs") or []: + metrics = dict(run.get("metrics") or {}) + totals = dict(metrics.get("totals") or {}) + tokens = dict(metrics.get("tokens") or {}) + quality = dict(run.get("quality") or {}) + lines.append( + "| {run_id} | {variant} | {status} | {calls} | {repairs} | {input_tokens} | " + "{total_tokens} | {wall_ms} | {quality} |".format( + run_id=run.get("run_id", ""), + variant=run.get("variant", ""), + status=run.get("status", ""), + calls=totals.get("invocation_count", 0), + repairs=totals.get("contract_repair_count", 0), + input_tokens=tokens.get("input", 0), + total_tokens=tokens.get("total", 0), + wall_ms=run.get("wall_duration_ms", 0), + quality="pass" if quality.get("passed") else "fail", + ) + ) + for pair in report.get("pairs") or []: + lines.extend( + [ + "", + f"## Pair {int(pair.get('pair_index') or 0):03d}", + "", + f"- Comparable: `{str(bool(pair.get('comparable'))).lower()}`", + ] + ) + reasons = pair.get("inconclusive_reasons") or [] + if reasons: + lines.append(f"- Inconclusive reasons: `{', '.join(str(item) for item in reasons)}`") + comparison = dict(pair.get("comparison") or {}) + end_to_end = dict(comparison.get("end_to_end") or {}) + if end_to_end: + lines.extend( + [ + "", + "| Metric | Before | After | Delta | Reduction | Reduction % |", + "| --- | ---: | ---: | ---: | ---: | ---: |", + ] + ) + for metric_name, values in end_to_end.items(): + values = dict(values or {}) + missing = "unpriced" if metric_name == "estimated_cost_usd" else "N/A" + lines.append( + f"| {metric_name} | {_display(values.get('before'), missing=missing)} | " + f"{_display(values.get('after'), missing=missing)} | " + f"{_display(values.get('delta'), missing=missing)} | " + f"{_display(values.get('reduction'), missing=missing)} | " + f"{_display(values.get('reduction_percent'), missing=missing)} |" + ) + lines.extend( + [ + "", + "## Interpretation", + "", + str((report.get("interpretation") or {}).get("note") or ""), + "", + ] + ) + return "\n".join(lines) + + +__all__ = [ + "REPORT_SCHEMA_VERSION", + "build_report", + "render_markdown", + "utc_now_iso", + "write_json_atomic", + "write_jsonl_atomic", + "write_run_artifacts", + "write_text_atomic", +] diff --git a/benchmarking/runner.py b/benchmarking/runner.py new file mode 100644 index 0000000..9f05872 --- /dev/null +++ b/benchmarking/runner.py @@ -0,0 +1,455 @@ +from __future__ import annotations + +import hashlib +import os +import platform +import re +import shutil +import subprocess +import tempfile +import time +import uuid +from dataclasses import replace +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping + +from teams_runtime.benchmarking.metrics import ( + load_workspace_telemetry, + reduce_telemetry, + sanitize_invocation_record, +) +from teams_runtime.benchmarking.models import ( + ArmResult, + BenchmarkOptions, + BenchmarkResult, + BenchmarkWorker, + BenchmarkWorkerSafetyError, + QualityEvidence, + WorkerContext, + WorkerOutcome, + make_arm_schedule, +) +from teams_runtime.benchmarking.reporting import ( + build_report, + render_markdown, + utc_now_iso, + write_json_atomic, + write_run_artifacts, + write_text_atomic, +) +from teams_runtime.benchmarking.scenario import ( + SCENARIO_MILESTONE, + ScenarioWorkspace, + create_scenario_workspace, + inspect_scenario_workspace, + load_runtime_settings, +) + + +_BENCHMARK_ID_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$") +_RETAINED_BASELINE_FILES = ( + (".benchmark/scenario.json", ".benchmark/scenario.json"), + (".benchmark/history_seed.json", ".benchmark/history_seed.json"), + ("benchmark_app.py", "benchmark_app.baseline.py"), + ("tests/__init__.py", "tests/__init__.py"), + ("tests/test_benchmark_app.py", "tests/test_benchmark_app.py"), + ("BENCHMARK_TASK.md", "BENCHMARK_TASK.md"), + ("team_runtime.yaml", "team_runtime.yaml"), +) +_RETENTION_NOTICE = """# Sanitized Benchmark Snapshot + +This directory is an allowlisted diagnostic snapshot, not the execution workspace. +Baseline files are captured before any model call. The mutable implementation is +represented only by a content hash and byte count. Runtime state, model sessions, +provider output, logs, Git metadata, and unrecognized files are intentionally excluded. +""" + + +class BenchmarkPreflightError(RuntimeError): + pass + + +def _git( + root: Path, + *args: str, +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ("git", *args), + cwd=root, + text=True, + capture_output=True, + check=False, + ) + + +def _dirty_hash(porcelain: str) -> str: + return hashlib.sha256(porcelain.encode("utf-8")).hexdigest() if porcelain else "" + + +def _source_revision(source_root: Path, *, allow_dirty: bool) -> dict[str, Any]: + root = source_root.expanduser().resolve() + head = _git(root, "rev-parse", "HEAD") + if head.returncode: + raise BenchmarkPreflightError(f"Source root is not a Git repository: {root}") + status = _git(root, "status", "--porcelain=v1", "--untracked-files=all") + if status.returncode: + raise BenchmarkPreflightError("Unable to inspect source Git status") + dirty = bool(status.stdout.strip()) + if dirty and not allow_dirty: + raise BenchmarkPreflightError( + "Source worktree is dirty; commit changes or use allow_dirty_source explicitly" + ) + describe = _git(root, "describe", "--always", "--dirty", "--tags") + return { + "commit_sha": head.stdout.strip(), + "describe": describe.stdout.strip() if describe.returncode == 0 else head.stdout.strip()[:12], + "dirty": dirty, + "dirty_state_hash": _dirty_hash(status.stdout), + "python": platform.python_version(), + "platform": platform.platform(), + } + + +def _benchmark_id(options: BenchmarkOptions) -> str: + if options.benchmark_id: + if not _BENCHMARK_ID_PATTERN.fullmatch(options.benchmark_id): + raise ValueError( + "benchmark_id must be 1-96 ASCII letters, numbers, dots, underscores, or hyphens" + ) + return options.benchmark_id + timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + return f"sprint-ab-{timestamp}-{uuid.uuid4().hex[:8]}" + + +def _output_root(options: BenchmarkOptions, benchmark_id: str) -> Path: + base = ( + options.output_dir.expanduser().resolve() + if options.output_dir is not None + else options.source_root.expanduser().resolve() / ".teams_runtime" / "benchmarks" + ) + root = base / benchmark_id + root.mkdir(parents=True, mode=0o700, exist_ok=False) + root.chmod(0o700) + return root + + +def _merge_quality( + outcome: WorkerOutcome, + scenario: ScenarioWorkspace, +) -> QualityEvidence: + inspection = inspect_scenario_workspace(scenario) + worker = outcome.quality + sprint = outcome.sprint + notes = tuple(dict.fromkeys((*worker.notes, *inspection.notes))) + return QualityEvidence( + behavior_oracle_passed=inspection.behavior_oracle_passed, + sprint_terminal=worker.sprint_terminal, + closeout_verified=worker.closeout_verified, + protected_files_unchanged=inspection.protected_files_unchanged, + git_clean=inspection.git_clean, + commit_created=inspection.commit_created, + no_git_remotes=inspection.no_git_remotes, + blocked_todo_count=max(worker.blocked_todo_count, sprint.blocked_todo_count), + failed_todo_count=max(worker.failed_todo_count, sprint.failed_todo_count), + notes=notes, + ) + + +def _safe_worker_failure( + started_at: str, + started_monotonic: float, + exc: BaseException, +) -> WorkerOutcome: + return WorkerOutcome( + status="failed", + started_at=started_at, + ended_at=utc_now_iso(), + wall_duration_ms=max(int((time.monotonic() - started_monotonic) * 1000), 0), + worker_duration_ms=max(int((time.monotonic() - started_monotonic) * 1000), 0), + stop_reason="worker_exception", + error_category=type(exc).__name__, + ) + + +def _is_safe_regular_file(source_root: Path, relative_name: str) -> bool: + candidate = source_root + for component in Path(relative_name).parts: + candidate = candidate / component + if candidate.is_symlink(): + return False + if not candidate.is_file(): + return False + try: + candidate.resolve().relative_to(source_root.resolve()) + except (OSError, RuntimeError, ValueError): + return False + return True + + +def _capture_retention_baseline(source_root: Path) -> dict[str, bytes]: + baseline: dict[str, bytes] = {} + for source_name, retained_name in _RETAINED_BASELINE_FILES: + if not _is_safe_regular_file(source_root, source_name): + raise BenchmarkPreflightError( + f"Benchmark baseline file is missing or unsafe: {source_name}" + ) + baseline[retained_name] = (source_root / source_name).read_bytes() + return baseline + + +def _implementation_result_summary( + source_root: Path, + baseline: Mapping[str, bytes], +) -> dict[str, Any]: + source_name = "benchmark_app.py" + baseline_content = baseline["benchmark_app.baseline.py"] + baseline_hash = hashlib.sha256(baseline_content).hexdigest() + if not _is_safe_regular_file(source_root, source_name): + return { + "schema_version": 1, + "path": source_name, + "status": "missing_or_unsafe", + "baseline_sha256": baseline_hash, + "sha256": None, + "size_bytes": None, + "changed_from_baseline": None, + } + source = source_root / source_name + digest = hashlib.sha256() + size_bytes = 0 + with source.open("rb") as handle: + for chunk in iter(lambda: handle.read(64 * 1024), b""): + digest.update(chunk) + size_bytes += len(chunk) + result_hash = digest.hexdigest() + return { + "schema_version": 1, + "path": source_name, + "status": "hashed", + "baseline_sha256": baseline_hash, + "sha256": result_hash, + "size_bytes": size_bytes, + "changed_from_baseline": result_hash != baseline_hash, + } + + +def _retain_workspace_snapshot( + source_root: Path, + retained_root: Path, + *, + baseline: Mapping[str, bytes], +) -> None: + retained_root.mkdir(parents=True, mode=0o700, exist_ok=False) + retained_root.chmod(0o700) + for relative_name, content in baseline.items(): + destination = retained_root / relative_name + destination.parent.mkdir(parents=True, mode=0o700, exist_ok=True) + destination.write_bytes(content) + destination.chmod(0o600) + write_json_atomic( + retained_root / "benchmark_app.result.json", + _implementation_result_summary(source_root, baseline), + ) + notice_path = retained_root / "RETENTION_NOTICE.md" + notice_path.write_text( + _RETENTION_NOTICE, + encoding="utf-8", + ) + notice_path.chmod(0o600) + + +def _run_arm( + *, + benchmark_id: str, + output_root: Path, + temporary_root: Path, + options: BenchmarkOptions, + worker: BenchmarkWorker, + settings: Any, + arm: Any, +) -> ArmResult: + workspace_parent = temporary_root / arm.run_id + scenario = create_scenario_workspace( + workspace_parent, + benchmark_id=benchmark_id, + run_id=arm.run_id, + prompt_context_enabled=arm.prompt_context_enabled, + settings=settings, + ) + retention_baseline = _capture_retention_baseline(scenario.root) + run_dir = output_root / "runs" / arm.run_id + started_at = utc_now_iso() + started_monotonic = time.monotonic() + context = WorkerContext( + benchmark_id=benchmark_id, + arm=arm, + workspace_root=scenario.root, + run_output_dir=run_dir, + milestone=SCENARIO_MILESTONE, + history_seed=scenario.history_seed, + max_invocations=options.max_invocations, + call_timeout_seconds=options.call_timeout_seconds, + run_timeout_seconds=options.run_timeout_seconds, + live=options.live, + ) + try: + outcome = worker(context) + if not isinstance(outcome, WorkerOutcome): + raise TypeError("Benchmark worker must return WorkerOutcome") + except BenchmarkWorkerSafetyError: + raise + except Exception as exc: + outcome = _safe_worker_failure(started_at, started_monotonic, exc) + measured_wall_ms = max(int((time.monotonic() - started_monotonic) * 1000), 0) + if measured_wall_ms > int(options.run_timeout_seconds * 1000) and outcome.status == "completed": + outcome = replace( + outcome, + status="timeout", + stop_reason="run_timeout_exceeded", + ) + raw_records = ( + tuple(sanitize_invocation_record(record) for record in outcome.telemetry_records) + if outcome.telemetry_records + else load_workspace_telemetry(scenario.root) + ) + metrics = reduce_telemetry(raw_records) + quality = _merge_quality(outcome, scenario) + result = ArmResult( + arm=arm, + status=outcome.status, + started_at=outcome.started_at or started_at, + ended_at=outcome.ended_at or utc_now_iso(), + wall_duration_ms=outcome.wall_duration_ms or measured_wall_ms, + worker_duration_ms=outcome.worker_duration_ms or measured_wall_ms, + stop_reason=outcome.stop_reason, + error_category=outcome.error_category, + config_hash=scenario.config_hash, + comparable_config_hash=scenario.comparable_config_hash, + metrics=metrics, + quality=quality, + sprint=outcome.sprint, + invocation_records=raw_records, + ) + keep = options.keep_workspaces == "all" or ( + options.keep_workspaces == "failures" + and (result.status != "completed" or not result.quality.passed) + ) + if keep: + retained_root = output_root / "workspaces" / arm.run_id + retained_root.parent.mkdir(parents=True, exist_ok=True) + _retain_workspace_snapshot( + scenario.root, + retained_root, + baseline=retention_baseline, + ) + shutil.rmtree(scenario.root, ignore_errors=True) + result = replace( + result, + retained_workspace=f"workspaces/{arm.run_id}", + ) + else: + shutil.rmtree(scenario.root, ignore_errors=True) + write_run_artifacts(run_dir, result) + return result + + +def run_sprint_ab_benchmark( + options: BenchmarkOptions, + *, + worker: BenchmarkWorker, +) -> BenchmarkResult: + """Run isolated sprint arms and write a privacy-safe A/B report. + + The injected worker owns TeamService orchestration and provider-process + enforcement. This core owns fixtures, arm isolation, telemetry reduction, + quality checks, comparison semantics, retention, and report persistence. + """ + + options.validate() + source_revision = _source_revision( + options.source_root, + allow_dirty=options.allow_dirty_source, + ) + settings = load_runtime_settings( + options.runtime_config_path, + rate_card_path=options.rate_card_path, + ) + benchmark_id = _benchmark_id(options) + output_root = _output_root(options, benchmark_id) + started_at = utc_now_iso() + schedule = make_arm_schedule(options.repetitions) + runs: list[ArmResult] = [] + history_hash = "" + with tempfile.TemporaryDirectory(prefix=f"{benchmark_id}-") as temp_directory: + temporary_root = Path(temp_directory) + temporary_root.chmod(0o700) + for arm in schedule: + result = _run_arm( + benchmark_id=benchmark_id, + output_root=output_root, + temporary_root=temporary_root, + options=options, + worker=worker, + settings=settings, + arm=arm, + ) + runs.append(result) + if result.status == "preflight_failed": + # The paired arm uses the same source, model, and safety controls. + # Repeating a failed safety/configuration preflight cannot produce + # a valid comparison and may obscure the original failure. + break + if not history_hash: + scenario_file = ( + output_root / result.retained_workspace / ".benchmark" / "scenario.json" + if result.retained_workspace + else None + ) + if scenario_file is not None and scenario_file.is_file(): + import json + + history_hash = str( + (json.loads(scenario_file.read_text(encoding="utf-8")) or {}).get( + "history_hash" + ) + or "" + ) + if not history_hash: + from teams_runtime.benchmarking.scenario import build_history_seed, canonical_hash + + history_hash = canonical_hash(build_history_seed()) + + ended_at = utc_now_iso() + report = build_report( + benchmark_id=benchmark_id, + options=options, + source_revision=source_revision, + source_config_hash=settings.source_config_hash, + runtime_model_map=settings.role_defaults, + rate_cards=settings.rate_cards, + history_hash=history_hash, + runs=tuple(runs), + started_at=started_at, + ended_at=ended_at, + ) + report_json = output_root / "report.json" + report_markdown = output_root / "report.md" + write_json_atomic(report_json, report) + write_text_atomic(report_markdown, render_markdown(report)) + return BenchmarkResult( + benchmark_id=benchmark_id, + status=str(report["status"]), # type: ignore[arg-type] + classification=str(report["classification"]), # type: ignore[arg-type] + output_dir=output_root, + report_json=report_json, + report_markdown=report_markdown, + runs=tuple(runs), + report=report, + ) + + +__all__ = [ + "BenchmarkPreflightError", + "run_sprint_ab_benchmark", +] diff --git a/benchmarking/scenario.py b/benchmarking/scenario.py new file mode 100644 index 0000000..bc6ea7c --- /dev/null +++ b/benchmarking/scenario.py @@ -0,0 +1,502 @@ +from __future__ import annotations + +import hashlib +import json +import os +import subprocess +import sys +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Mapping + +import yaml + +from teams_runtime.core.template import scaffold_workspace +from teams_runtime.shared.models import TEAM_ROLES + + +SCENARIO_ID = "sum-positive-full-sprint-v1" +DEFAULT_HISTORY_SEED_COUNT = 48 +BENCHMARK_PROMPT_CONTEXT_RECENT_EVENTS = 8 +BENCHMARK_PROMPT_CONTEXT_MAX_EVENTS = 16 +SCENARIO_MILESTONE = ( + "Fix sum_positive(values) so it returns the sum of positive values only. " + "Preserve the public function, do not alter benchmark tests or scenario metadata, " + "run the unittest suite, and commit the completed change." +) +PROTECTED_PATHS = ( + ".benchmark/scenario.json", + ".benchmark/history_seed.json", + "tests/__init__.py", + "tests/test_benchmark_app.py", +) +_RATE_FIELDS = ( + "input_per_million_usd", + "cached_input_per_million_usd", + "output_per_million_usd", + "per_invocation_usd", +) + + +class ScenarioError(RuntimeError): + pass + + +@dataclass(slots=True, frozen=True) +class RuntimeSettings: + role_defaults: Mapping[str, Mapping[str, str]] + rate_cards: Mapping[str, Mapping[str, float | None]] + source_config_hash: str + + +@dataclass(slots=True, frozen=True) +class ScenarioWorkspace: + root: Path + initial_commit: str + initial_commit_count: int + protected_hashes: Mapping[str, str] + config_hash: str + comparable_config_hash: str + history_hash: str + history_seed: tuple[Mapping[str, Any], ...] + + +@dataclass(slots=True, frozen=True) +class WorkspaceInspection: + behavior_oracle_passed: bool + protected_files_unchanged: bool + git_clean: bool + commit_created: bool + no_git_remotes: bool + head_sha: str + notes: tuple[str, ...] + + +def _canonical_json(value: Any) -> str: + return json.dumps(value, ensure_ascii=True, sort_keys=True, separators=(",", ":")) + + +def canonical_hash(value: Any) -> str: + return hashlib.sha256(_canonical_json(value).encode("utf-8")).hexdigest() + + +def file_hash(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(64 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _read_yaml(path: Path) -> dict[str, Any]: + payload = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError(f"Expected a YAML mapping: {path}") + return payload + + +def _runtime_config_file(path: Path) -> Path: + resolved = path.expanduser().resolve() + return resolved / "team_runtime.yaml" if resolved.is_dir() else resolved + + +def _normalize_role_defaults(payload: Mapping[str, Any]) -> dict[str, dict[str, str]]: + raw_defaults = payload.get("role_defaults") + if not isinstance(raw_defaults, dict): + raise ValueError("Runtime config must define role_defaults") + normalized: dict[str, dict[str, str]] = {} + for role in TEAM_ROLES: + raw = raw_defaults.get(role) + if not isinstance(raw, dict): + raise ValueError(f"Runtime config must define role_defaults.{role}") + model = str(raw.get("model") or "").strip() + reasoning = str(raw.get("reasoning") or "").strip() + if not model or not reasoning: + raise ValueError(f"role_defaults.{role} must define model and reasoning") + normalized[role] = {"model": model, "reasoning": reasoning} + return normalized + + +def _normalize_rate(value: Any, *, field_name: str) -> float | None: + if value is None: + return None + if isinstance(value, bool): + raise ValueError(f"{field_name} must be a finite non-negative number") + try: + normalized = float(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{field_name} must be a finite non-negative number") from exc + if normalized < 0 or normalized in {float("inf"), float("-inf")} or normalized != normalized: + raise ValueError(f"{field_name} must be a finite non-negative number") + return normalized + + +def _normalize_rate_cards(payload: Mapping[str, Any]) -> dict[str, dict[str, float | None]]: + raw_cards: Any = payload.get("rate_cards") + if raw_cards is None and isinstance(payload.get("telemetry"), dict): + raw_cards = payload["telemetry"].get("rate_cards") + if raw_cards in (None, {}): + return {} + if not isinstance(raw_cards, dict): + raise ValueError("rate_cards must be a mapping") + cards: dict[str, dict[str, float | None]] = {} + for raw_key, raw_card in raw_cards.items(): + key = str(raw_key or "").strip() + if "/" not in key or not isinstance(raw_card, dict): + raise ValueError(f"Invalid rate card entry: {raw_key!r}") + normalized = { + field_name: _normalize_rate( + raw_card.get(field_name), + field_name=f"rate_cards.{key}.{field_name}", + ) + for field_name in _RATE_FIELDS + } + if normalized["per_invocation_usd"] is None and ( + normalized["input_per_million_usd"] is None + or normalized["output_per_million_usd"] is None + ): + raise ValueError( + f"rate_cards.{key} requires per_invocation_usd or both input and output rates" + ) + cards[key] = normalized + return cards + + +def load_runtime_settings( + runtime_config_path: Path, + *, + rate_card_path: Path | None = None, +) -> RuntimeSettings: + config_file = _runtime_config_file(runtime_config_path) + payload = _read_yaml(config_file) + role_defaults = _normalize_role_defaults(payload) + rate_cards: dict[str, dict[str, float | None]] = {} + if rate_card_path is not None: + rate_cards = _normalize_rate_cards(_read_yaml(rate_card_path.expanduser().resolve())) + source_snapshot = {"role_defaults": role_defaults, "rate_cards": rate_cards} + return RuntimeSettings( + role_defaults=role_defaults, + rate_cards=rate_cards, + source_config_hash=canonical_hash(source_snapshot), + ) + + +def build_history_seed( + count: int = DEFAULT_HISTORY_SEED_COUNT, +) -> tuple[Mapping[str, Any], ...]: + if count < 24: + raise ValueError("History seed must contain at least 24 events") + roles = ( + "research", + "planner", + "designer", + "architect", + "developer", + "qa", + "version_controller", + "orchestrator", + ) + started = datetime(2026, 1, 1, tzinfo=timezone.utc) + events: list[Mapping[str, Any]] = [] + for index in range(count): + timestamp = (started + timedelta(minutes=index)).isoformat() + if index < len(roles) * 2 and index % 2 == 1: + role = roles[index // 2] + event: Mapping[str, Any] = { + "created_at": timestamp, + "type": "role_report", + "actor": role, + "summary": f"Historical {role} checkpoint {index + 1:02d}.", + "payload": { + "role": role, + "status": "completed", + "summary": f"Stable benchmark evidence {index + 1:02d}.", + }, + } + else: + event = { + "created_at": timestamp, + "type": "benchmark_checkpoint", + "actor": "orchestrator", + "summary": f"Neutral historical checkpoint {index + 1:02d}.", + "payload": {"sequence": index + 1}, + } + events.append(event) + return tuple(events) + + +def _benchmark_config( + workspace_root: Path, + *, + benchmark_id: str, + run_id: str, + prompt_context_enabled: bool, + settings: RuntimeSettings, +) -> tuple[str, str]: + path = workspace_root / "team_runtime.yaml" + payload = _read_yaml(path) + sprint = dict(payload.get("sprint") or {}) + sprint.update( + { + "id": f"{benchmark_id}-{run_id.rsplit('-', 1)[0]}", + "mode": "hybrid", + "start_mode": "manual_daily", + "ingress_mode": "backlog_first", + "discovery_scope": "workspace_only", + "discovery_actions": [], + } + ) + payload["sprint"] = sprint + payload["role_defaults"] = { + role: dict(settings.role_defaults[role]) + for role in TEAM_ROLES + } + payload["research_defaults"] = { + "app": "", + "notebook": "", + "files": [], + "mode": "", + "profile_path": "", + "completion_timeout": 600, + "callback_timeout": 1200, + "cleanup": False, + "reasoning_level": "Standard", + } + payload["prompt_context"] = { + "enabled": prompt_context_enabled, + "recent_events": BENCHMARK_PROMPT_CONTEXT_RECENT_EVENTS, + "max_events": BENCHMARK_PROMPT_CONTEXT_MAX_EVENTS, + } + payload["telemetry"] = { + "enabled": True, + "rate_cards": { + key: { + field_name: value + for field_name, value in card.items() + if value is not None + } + for key, card in settings.rate_cards.items() + }, + } + payload["actions"] = {} + path.write_text( + yaml.safe_dump(payload, sort_keys=False, allow_unicode=False), + encoding="utf-8", + ) + comparable_payload = json.loads(json.dumps(payload)) + comparable_payload["prompt_context"].pop("enabled", None) + return canonical_hash(payload), canonical_hash(comparable_payload) + + +def _run_git(root: Path, *args: str, check: bool = True) -> subprocess.CompletedProcess[str]: + completed = subprocess.run( + ("git", *args), + cwd=root, + text=True, + capture_output=True, + check=False, + env={ + "HOME": os.environ.get("HOME", ""), + "PATH": os.environ.get("PATH", ""), + "LC_ALL": "C", + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_GLOBAL": os.devnull, + }, + ) + if check and completed.returncode: + raise ScenarioError(f"Git command failed: git {' '.join(args)}") + return completed + + +def _initialize_git(root: Path) -> tuple[str, int]: + _run_git(root, "init", "-b", "benchmark") + _run_git(root, "config", "--local", "user.name", "teams-runtime-benchmark") + _run_git(root, "config", "--local", "user.email", "benchmark@invalid.local") + _run_git(root, "config", "--local", "commit.gpgsign", "false") + _run_git(root, "config", "--local", "tag.gpgsign", "false") + _run_git(root, "config", "--local", "core.hooksPath", ".git/benchmark-disabled-hooks") + (root / ".git" / "benchmark-disabled-hooks").mkdir(mode=0o700, exist_ok=True) + _run_git(root, "add", "--all") + _run_git(root, "commit", "-m", "[benchmark] seed defective sum_positive scenario") + head = _run_git(root, "rev-parse", "HEAD").stdout.strip() + count = int(_run_git(root, "rev-list", "--count", "HEAD").stdout.strip()) + if _run_git(root, "remote").stdout.strip(): + raise ScenarioError("Benchmark repository unexpectedly has a Git remote") + return head, count + + +def _assert_defect_reproduces(root: Path) -> None: + result = subprocess.run( + (sys.executable, "-m", "unittest", "discover", "-s", "tests"), + cwd=root, + text=True, + capture_output=True, + check=False, + timeout=30, + env={"PATH": os.environ.get("PATH", ""), "PYTHONPATH": str(root), "LC_ALL": "C"}, + ) + if result.returncode == 0: + raise ScenarioError("Benchmark fixture must fail its baseline behavior oracle") + + +def create_scenario_workspace( + workspace_root: Path, + *, + benchmark_id: str, + run_id: str, + prompt_context_enabled: bool, + settings: RuntimeSettings, +) -> ScenarioWorkspace: + root = workspace_root.expanduser().resolve() + root.mkdir(parents=True, mode=0o700, exist_ok=False) + root.chmod(0o700) + scaffold_workspace(root) + history_seed = build_history_seed() + scenario_payload = { + "schema_version": 1, + "scenario_id": SCENARIO_ID, + "milestone": SCENARIO_MILESTONE, + "protected_paths": list(PROTECTED_PATHS), + "quality_command": ["python", "-m", "unittest", "discover", "-s", "tests"], + "history_event_count": len(history_seed), + "history_hash": canonical_hash(history_seed), + } + files = { + ".benchmark/scenario.json": json.dumps(scenario_payload, indent=2, sort_keys=True) + "\n", + ".benchmark/history_seed.json": json.dumps(history_seed, indent=2, sort_keys=True) + "\n", + "benchmark_app.py": ( + '"""Small benchmark target with an intentional defect."""\n\n' + "\n" + "def sum_positive(values):\n" + ' """Return the sum of positive numeric values."""\n' + " return sum(values)\n" + ), + "tests/__init__.py": "", + "tests/test_benchmark_app.py": ( + "import unittest\n\n" + "from benchmark_app import sum_positive\n\n\n" + "class SumPositiveTests(unittest.TestCase):\n" + " def test_mixed_values(self):\n" + " self.assertEqual(sum_positive([5, -8, 2]), 7)\n\n" + " def test_non_positive_values(self):\n" + " self.assertEqual(sum_positive([-5, 0, -3]), 0)\n\n" + " def test_empty_values(self):\n" + " self.assertEqual(sum_positive([]), 0)\n\n\n" + 'if __name__ == "__main__":\n' + " unittest.main()\n" + ), + "BENCHMARK_TASK.md": ( + "# Benchmark Task\n\n" + f"{SCENARIO_MILESTONE}\n\n" + "Acceptance command: `python -m unittest discover -s tests`\n" + ), + ".gitignore": ( + ".teams_runtime/\n" + "logs/\n" + "__pycache__/\n" + "*.py[cod]\n" + ".teams_runtime_codex_output.txt\n" + ), + } + for relative_path, content in files.items(): + target = root / relative_path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + config_hash, comparable_hash = _benchmark_config( + root, + benchmark_id=benchmark_id, + run_id=run_id, + prompt_context_enabled=prompt_context_enabled, + settings=settings, + ) + protected_hashes = { + relative_path: file_hash(root / relative_path) + for relative_path in PROTECTED_PATHS + } + _assert_defect_reproduces(root) + initial_commit, commit_count = _initialize_git(root) + return ScenarioWorkspace( + root=root, + initial_commit=initial_commit, + initial_commit_count=commit_count, + protected_hashes=protected_hashes, + config_hash=config_hash, + comparable_config_hash=comparable_hash, + history_hash=canonical_hash(history_seed), + history_seed=history_seed, + ) + + +def inspect_scenario_workspace( + scenario: ScenarioWorkspace, + *, + timeout_seconds: float = 30.0, +) -> WorkspaceInspection: + root = scenario.root + notes: list[str] = [] + try: + oracle = subprocess.run( + (sys.executable, "-m", "unittest", "discover", "-s", "tests"), + cwd=root, + text=True, + capture_output=True, + check=False, + timeout=timeout_seconds, + env={"PATH": os.environ.get("PATH", ""), "PYTHONPATH": str(root), "LC_ALL": "C"}, + ) + oracle_passed = oracle.returncode == 0 + if not oracle_passed: + notes.append("behavior_oracle_failed") + except subprocess.TimeoutExpired: + oracle_passed = False + notes.append("behavior_oracle_timeout") + + protected_unchanged = True + for relative_path, expected_hash in scenario.protected_hashes.items(): + path = root / relative_path + if not path.is_file() or file_hash(path) != expected_hash: + protected_unchanged = False + notes.append(f"protected_file_changed:{relative_path}") + + status = _run_git(root, "status", "--porcelain", "--untracked-files=all", check=False) + git_clean = status.returncode == 0 and not status.stdout.strip() + if not git_clean: + notes.append("git_worktree_not_clean") + head_result = _run_git(root, "rev-parse", "HEAD", check=False) + head_sha = head_result.stdout.strip() if head_result.returncode == 0 else "" + commit_created = bool(head_sha and head_sha != scenario.initial_commit) + if not commit_created: + notes.append("task_commit_missing") + remotes = _run_git(root, "remote", check=False) + no_git_remotes = remotes.returncode == 0 and not remotes.stdout.strip() + if not no_git_remotes: + notes.append("git_remote_detected") + return WorkspaceInspection( + behavior_oracle_passed=oracle_passed, + protected_files_unchanged=protected_unchanged, + git_clean=git_clean, + commit_created=commit_created, + no_git_remotes=no_git_remotes, + head_sha=head_sha, + notes=tuple(notes), + ) + + +__all__ = [ + "BENCHMARK_PROMPT_CONTEXT_MAX_EVENTS", + "BENCHMARK_PROMPT_CONTEXT_RECENT_EVENTS", + "DEFAULT_HISTORY_SEED_COUNT", + "PROTECTED_PATHS", + "RuntimeSettings", + "SCENARIO_ID", + "SCENARIO_MILESTONE", + "ScenarioError", + "ScenarioWorkspace", + "WorkspaceInspection", + "build_history_seed", + "canonical_hash", + "create_scenario_workspace", + "inspect_scenario_workspace", + "load_runtime_settings", +] diff --git a/benchmarking/worker.py b/benchmarking/worker.py new file mode 100644 index 0000000..08de80c --- /dev/null +++ b/benchmarking/worker.py @@ -0,0 +1,1405 @@ +from __future__ import annotations + +import argparse +import asyncio +import contextlib +import hashlib +import json +import os +import signal +import subprocess +import sys +import time +import uuid +from dataclasses import replace +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping + +from teams_runtime.benchmarking.metrics import load_workspace_telemetry +from teams_runtime.benchmarking.models import ( + ArmPlan, + BenchmarkWorkerSafetyError, + QualityEvidence, + SprintEvidence, + WorkerContext, + WorkerOutcome, +) +from teams_runtime.benchmarking.scenario import ( + DEFAULT_HISTORY_SEED_COUNT, + canonical_hash, +) +from teams_runtime.runtime.execution_policy import ( + InvocationBudget, + InvocationBudgetExceeded, + ModelExecutionPolicy, + ModelExecutionPolicyViolation, + ModelInvocationTimeout, +) +from teams_runtime.shared.models import TEAM_ROLES +from teams_runtime.shared.paths import RuntimePaths +from teams_runtime.workflows.orchestration.team_service import TeamService +from teams_runtime.workflows.sprints.lifecycle import apply_initial_plan_confirmation +from teams_runtime.workflows.state.sprint_store import iter_sprint_states + + +LIVE_BENCHMARK_ENV = "TEAMS_RUNTIME_LIVE_BENCHMARK" +_TERMINAL_SPRINT_STATUSES = frozenset({"completed", "failed", "blocked"}) +_COMPLETED_TODO_STATUSES = frozenset({"completed", "committed"}) +_MAX_RESUME_PASSES = 16 +_RELAY_POLL_SECONDS = 0.02 +_CHILD_TERMINATION_GRACE_SECONDS = 5.0 +_CHILD_ENVIRONMENT_KEYS = ( + "CODEX_API_KEY", + "CODEX_HOME", + "CURL_CA_BUNDLE", + "HOME", + "LANG", + "LC_ALL", + "LC_CTYPE", + "OPENAI_API_KEY", + "OPENAI_BASE_URL", + "OPENAI_ORG_ID", + "OPENAI_PROJECT_ID", + "PATH", + "REQUESTS_CA_BUNDLE", + "SSL_CERT_DIR", + "SSL_CERT_FILE", + "TEMP", + "TMP", + "TMPDIR", +) + + +class _BenchmarkRunTimeout(TimeoutError): + pass + + +class _SprintDidNotTerminate(RuntimeError): + pass + + +class _InitialPlanNotReady(RuntimeError): + pass + + +class _WorkerCleanupFailure(BenchmarkWorkerSafetyError): + pass + + +def _utc_now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +class _WorkerLog: + """Append-only, content-free worker diagnostics.""" + + def __init__(self, path: Path, *, reset: bool = False): + self.path = path + self.path.parent.mkdir(parents=True, mode=0o700, exist_ok=True) + try: + self.path.parent.chmod(0o700) + except OSError: + pass + if reset or not self.path.exists(): + self.path.write_text("", encoding="utf-8") + try: + self.path.chmod(0o600) + except OSError: + pass + + def append(self, event: str, **fields: Any) -> None: + safe_fields = " ".join( + f"{key}={str(value).replace(chr(10), ' ').replace(chr(13), ' ')}" + for key, value in sorted(fields.items()) + ) + line = f"{_utc_now_iso()} event={event}" + if safe_fields: + line += f" {safe_fields}" + with self.path.open("a", encoding="utf-8") as handle: + handle.write(line + "\n") + + +class _BenchmarkTeamService(TeamService): + """Production TeamService with benchmark-only outbound and history seams.""" + + def __init__( + self, + *args: Any, + benchmark_context: WorkerContext, + **kwargs: Any, + ): + self._benchmark_context = benchmark_context + self._benchmark_history_seeded = False + super().__init__(*args, **kwargs) + + def _create_internal_request_record( + self, + sprint_state: dict[str, Any], + todo: dict[str, Any], + backlog_item: dict[str, Any], + ) -> dict[str, Any]: + request_record = super()._create_internal_request_record( + sprint_state, + todo, + backlog_item, + ) + if self._benchmark_history_seeded: + return request_record + + seed = _copy_history_seed(self._benchmark_context.history_seed) + request_record["events"] = [ + *seed, + *[ + dict(event) + for event in (request_record.get("events") or []) + if isinstance(event, dict) + ], + ] + params = ( + dict(request_record.get("params") or {}) + if isinstance(request_record.get("params"), dict) + else {} + ) + params["_benchmark_history_seed"] = { + "event_count": len(seed), + "sha256": canonical_hash(seed), + } + request_record["params"] = params + self._save_request(request_record) + self._benchmark_history_seeded = True + return request_record + + def _mark_github_publish_skipped(self, sprint_state: dict[str, Any]) -> None: + sprint_state["github_issue_number"] = "" + sprint_state["github_issue_url"] = "" + sprint_state["github_issue_publish_status"] = "skipped_benchmark" + sprint_state["github_issue_publish_updated_at"] = _utc_now_iso() + sprint_state.pop("github_issue_publish_error", None) + self._save_sprint_state(sprint_state) + + def _schedule_sprint_issue_publish(self, sprint_state: dict[str, Any]) -> None: + self._mark_github_publish_skipped(sprint_state) + + async def _publish_sprint_issue_best_effort( + self, + sprint_state: dict[str, Any], + ) -> None: + self._mark_github_publish_skipped(sprint_state) + return None + + async def _publish_sprint_issue_before_terminal_reports( + self, + sprint_state: dict[str, Any], + ) -> None: + self._mark_github_publish_skipped(sprint_state) + + +def _copy_history_seed( + history_seed: tuple[Mapping[str, Any], ...], +) -> list[dict[str, Any]]: + payload = json.loads( + json.dumps( + list(history_seed), + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ) + ) + if not isinstance(payload, list) or not all(isinstance(item, dict) for item in payload): + raise ValueError("Benchmark history seed must contain JSON object events") + return [dict(item) for item in payload] + + +def _validate_context(context: WorkerContext) -> None: + if not context.live: + raise ModelExecutionPolicyViolation( + "Live benchmark worker requires WorkerContext.live=True" + ) + if os.environ.get(LIVE_BENCHMARK_ENV) != "1": + raise ModelExecutionPolicyViolation( + f"Live benchmark worker requires {LIVE_BENCHMARK_ENV}=1" + ) + workspace_root = context.workspace_root.expanduser().resolve() + if not workspace_root.is_dir(): + raise FileNotFoundError(f"Benchmark workspace does not exist: {workspace_root}") + for relative_path in ("team_runtime.yaml", ".git", ".benchmark/scenario.json"): + if not (workspace_root / relative_path).exists(): + raise FileNotFoundError( + f"Benchmark workspace is missing required path: {relative_path}" + ) + if len(context.history_seed) != DEFAULT_HISTORY_SEED_COUNT: + raise ValueError( + "Live sprint benchmark requires the fixed " + f"{DEFAULT_HISTORY_SEED_COUNT}-event history seed" + ) + if not str(context.milestone or "").strip(): + raise ValueError("Benchmark milestone must not be empty") + if context.max_invocations <= 0: + raise ValueError("Benchmark invocation budget must be positive") + if context.call_timeout_seconds <= 0: + raise ValueError("Benchmark call timeout must be positive") + if context.run_timeout_seconds <= 0: + raise ValueError("Benchmark run timeout must be positive") + + +def _source_import_root() -> Path: + # teams_runtime/benchmarking/worker.py -> import parent containing teams_runtime. + return Path(__file__).resolve().parents[2] + + +def _build_execution_policy( + context: WorkerContext, + *, + budget: InvocationBudget, +) -> ModelExecutionPolicy: + return ModelExecutionPolicy.for_benchmark( + allowed_workspace_root=context.workspace_root, + invocation_budget=budget, + call_timeout_seconds=context.call_timeout_seconds, + shell_environment={ + "LANG": "C", + "LC_ALL": "C", + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONPATH": str(_source_import_root()), + "PYTHONUNBUFFERED": "1", + }, + ) + + +def _build_services( + context: WorkerContext, + *, + policy: ModelExecutionPolicy, +) -> dict[str, _BenchmarkTeamService]: + services = { + role: _BenchmarkTeamService( + context.workspace_root, + role, + enable_discord_client=False, + relay_transport="internal", + model_execution_policy=policy, + allow_external_research=False, + benchmark_context=context, + ) + for role in TEAM_ROLES + } + configured_models = { + str(config.model or "").strip() + for service in services.values() + for config in service.runtime_config.role_defaults.values() + } + unsupported_models = sorted( + model for model in configured_models if not model or "gemini" in model.lower() + ) + if unsupported_models: + raise ModelExecutionPolicyViolation( + "Live sprint benchmark requires Codex-compatible role models" + ) + return services + + +async def _relay_pump( + services: Mapping[str, _BenchmarkTeamService], + *, + worker_log: _WorkerLog, +) -> None: + worker_log.append("relay_pump_started", role_count=len(services)) + while True: + for role in TEAM_ROLES: + await services[role]._consume_internal_relay_once() + await asyncio.sleep(_RELAY_POLL_SECONDS) + + +def _active_process_entries( + budget_or_snapshot: InvocationBudget | Mapping[str, Any], +) -> list[dict[str, Any]]: + snapshot = ( + budget_or_snapshot.snapshot() + if isinstance(budget_or_snapshot, InvocationBudget) + else budget_or_snapshot + ) + return [ + dict(entry) + for entry in (snapshot.get("entries") or []) + if isinstance(entry, dict) + and str(entry.get("state") or "") == "running" + and isinstance(entry.get("pid"), int) + ] + + +def _provider_entry_key(entry: Mapping[str, Any]) -> tuple[int, int | None]: + raw_pid = entry.get("pid") + raw_process_group_id = entry.get("process_group_id") + pid = int(raw_pid) if isinstance(raw_pid, int) and not isinstance(raw_pid, bool) else 0 + process_group_id = ( + int(raw_process_group_id) + if isinstance(raw_process_group_id, int) + and not isinstance(raw_process_group_id, bool) + else None + ) + return pid, process_group_id + + +def _merge_active_process_entries( + *snapshots: Mapping[str, Any], +) -> list[dict[str, Any]]: + merged: dict[tuple[int, int | None], dict[str, Any]] = {} + for snapshot in snapshots: + for entry in _active_process_entries(snapshot): + key = _provider_entry_key(entry) + if key[0] > 1: + merged[key] = entry + return list(merged.values()) + + +def _process_exists(pid: int) -> bool: + if pid <= 1: + return False + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return True + + +def _process_group_exists(process_group_id: int) -> bool: + if process_group_id <= 1 or not hasattr(os, "killpg"): + return False + try: + os.killpg(process_group_id, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return True + + +def _provider_entry_alive(entry: Mapping[str, Any]) -> bool: + pid, process_group_id = _provider_entry_key(entry) + if process_group_id is not None and hasattr(os, "killpg"): + return _process_group_exists(process_group_id) + return _process_exists(pid) + + +def _signal_provider_entries( + entries: list[dict[str, Any]], + process_signal: signal.Signals, +) -> int: + signaled = 0 + own_process_group = os.getpgrp() if hasattr(os, "getpgrp") else None + for entry in entries: + pid, process_group_id = _provider_entry_key(entry) + if pid <= 1: + continue + if process_group_id is not None and hasattr(os, "killpg"): + if process_group_id == own_process_group: + continue + try: + os.killpg(process_group_id, process_signal) + except ProcessLookupError: + continue + signaled += 1 + continue + try: + os.kill(pid, process_signal) + except ProcessLookupError: + continue + signaled += 1 + return signaled + + +def _signal_active_provider_processes( + budget_or_snapshot: InvocationBudget | Mapping[str, Any], + process_signal: signal.Signals, +) -> int: + return _signal_provider_entries( + _active_process_entries(budget_or_snapshot), + process_signal, + ) + + +async def _terminate_active_provider_processes( + budget: InvocationBudget, + *, + grace_seconds: float, + worker_log: _WorkerLog, +) -> None: + terminated = _signal_active_provider_processes(budget, signal.SIGTERM) + worker_log.append("provider_termination_requested", process_count=terminated) + if not terminated: + return + await asyncio.sleep(max(min(grace_seconds, 5.0), 0.0)) + killed = _signal_active_provider_processes(budget, signal.SIGKILL) + if killed: + worker_log.append("provider_kill_requested", process_count=killed) + + +def _latest_sprint_state(workspace_root: Path) -> dict[str, Any]: + paths = RuntimePaths.from_root(workspace_root) + states = [ + dict(state) + for state in iter_sprint_states(paths) + if isinstance(state, dict) and str(state.get("sprint_id") or "").strip() + ] + if not states: + return {} + return max( + states, + key=lambda state: ( + str(state.get("started_at") or ""), + str(state.get("sprint_id") or ""), + ), + ) + + +def _sprint_evidence(sprint_state: Mapping[str, Any]) -> SprintEvidence: + todos = [ + dict(todo) + for todo in (sprint_state.get("todos") or []) + if isinstance(todo, dict) + ] + statuses = [ + str(todo.get("status") or "").strip().lower() + for todo in todos + ] + return SprintEvidence( + sprint_id=str(sprint_state.get("sprint_id") or "").strip(), + status=str(sprint_state.get("status") or "").strip(), + closeout_status=str(sprint_state.get("closeout_status") or "").strip(), + todo_count=len(todos), + completed_todo_count=sum( + status in _COMPLETED_TODO_STATUSES for status in statuses + ), + blocked_todo_count=statuses.count("blocked"), + failed_todo_count=statuses.count("failed"), + commit_sha=str( + sprint_state.get("commit_sha") + or sprint_state.get("version_control_sha") + or sprint_state.get("auto_commit_sha") + or "" + ).strip(), + ) + + +def _quality_evidence( + sprint_state: Mapping[str, Any], + sprint: SprintEvidence, +) -> QualityEvidence: + notes: list[str] = [] + if not sprint.sprint_id: + notes.append("sprint_state_missing") + if sprint.status != "completed": + notes.append("sprint_not_completed") + if sprint.closeout_status != "verified": + notes.append("closeout_not_verified") + if any( + str(todo.get("status") or "").strip().lower() == "uncommitted" + for todo in (sprint_state.get("todos") or []) + if isinstance(todo, dict) + ): + notes.append("uncommitted_todo_present") + return QualityEvidence( + sprint_terminal=sprint.status == "completed", + closeout_verified=sprint.closeout_status == "verified", + blocked_todo_count=sprint.blocked_todo_count, + failed_todo_count=sprint.failed_todo_count, + notes=tuple(notes), + ) + + +def _confirmation_is_pending(sprint_state: Mapping[str, Any]) -> bool: + confirmation = sprint_state.get("initial_plan_confirmation") + return ( + isinstance(confirmation, dict) + and str(confirmation.get("status") or "").strip().lower() == "pending" + ) + + +def _confirm_initial_plan( + orchestrator: _BenchmarkTeamService, + sprint_state: dict[str, Any], + *, + worker_log: _WorkerLog, +) -> None: + confirmation = apply_initial_plan_confirmation( + sprint_state, + confirmed_by={ + "type": "benchmark_harness", + "author_id": "benchmark-harness", + "author_name": "benchmark-harness", + }, + message_id="benchmark-auto-confirm", + parser_reason="isolated benchmark auto-confirm policy", + parser_confidence="high", + confirmed_at=_utc_now_iso(), + ) + sprint_id = str(sprint_state.get("sprint_id") or "").strip() + orchestrator._save_sprint_state(sprint_state) + orchestrator._append_sprint_event( + sprint_id, + event_type="initial_plan_confirmed", + summary="Benchmark harness auto-confirmed the initial implementation plan.", + payload={ + "revision": int(confirmation.get("revision") or 0), + "confirmation_source": "benchmark_harness", + }, + ) + worker_log.append( + "initial_plan_auto_confirmed", + revision=int(confirmation.get("revision") or 0), + ) + + +async def _drive_sprint( + context: WorkerContext, + orchestrator: _BenchmarkTeamService, + *, + worker_log: _WorkerLog, +) -> None: + await orchestrator.start_sprint_lifecycle( + context.milestone, + trigger="benchmark", + resume_mode="await", + kickoff_brief=( + "Execute the deterministic local benchmark task. Use only files in the " + "workspace, preserve protected benchmark inputs, run the stated unittest " + "command, and commit the implementation." + ), + kickoff_requirements=[context.milestone], + kickoff_request_text=context.milestone, + kickoff_reference_artifacts=[ + "./BENCHMARK_TASK.md", + "./.benchmark/scenario.json", + "./tests/test_benchmark_app.py", + ], + kickoff_requester_route={ + "type": "benchmark_harness", + "author_id": "benchmark-harness", + "author_name": "benchmark-harness", + }, + ) + sprint_state = orchestrator._load_active_sprint_state() + if not sprint_state: + raise _InitialPlanNotReady("Sprint state was not created") + + sprint_id = str(sprint_state.get("sprint_id") or "").strip() + worker_log.append("sprint_created") + confirmation_performed = False + for _resume_pass in range(_MAX_RESUME_PASSES): + sprint_state = orchestrator._load_sprint_state(sprint_id) + status = str(sprint_state.get("status") or "").strip().lower() + if status in _TERMINAL_SPRINT_STATUSES: + worker_log.append( + "sprint_terminal", + closeout_status=str(sprint_state.get("closeout_status") or ""), + status=status, + ) + return + if _confirmation_is_pending(sprint_state): + if confirmation_performed: + raise _InitialPlanNotReady( + "Initial implementation plan returned to pending state" + ) + _confirm_initial_plan( + orchestrator, + sprint_state, + worker_log=worker_log, + ) + confirmation_performed = True + elif not confirmation_performed: + raise _InitialPlanNotReady( + "Initial implementation plan did not reach pending confirmation" + ) + await orchestrator._resume_active_sprint(sprint_id) + + raise _SprintDidNotTerminate( + f"Sprint did not terminate after {_MAX_RESUME_PASSES} resume passes" + ) + + +async def _execute_live_arm( + context: WorkerContext, + *, + budget: InvocationBudget, + policy: ModelExecutionPolicy, + worker_log: _WorkerLog, +) -> None: + services = _build_services(context, policy=policy) + worker_log.append( + "services_ready", + discord="disabled", + external_research="disabled", + relay="internal", + role_count=len(services), + ) + pump_task = asyncio.create_task( + _relay_pump(services, worker_log=worker_log), + name=f"benchmark-relay-{context.arm.run_id}", + ) + sprint_task = asyncio.create_task( + _drive_sprint( + context, + services["orchestrator"], + worker_log=worker_log, + ), + name=f"benchmark-sprint-{context.arm.run_id}", + ) + try: + await asyncio.wait_for( + sprint_task, + timeout=context.run_timeout_seconds, + ) + except ModelInvocationTimeout: + raise + except TimeoutError as exc: + await _terminate_active_provider_processes( + budget, + grace_seconds=policy.kill_grace_seconds, + worker_log=worker_log, + ) + raise _BenchmarkRunTimeout( + f"Sprint arm exceeded {context.run_timeout_seconds:g} seconds" + ) from exc + finally: + if not sprint_task.done(): + sprint_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await sprint_task + pump_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await pump_task + for service in services.values(): + with contextlib.suppress(Exception): + await service.discord_client.close() + + +def _classify_status( + *, + forced_status: str, + sprint: SprintEvidence, + budget_snapshot: Mapping[str, Any], +) -> tuple[str, str, str]: + if int(budget_snapshot.get("rejected_count") or 0) > 0: + return ( + "call_budget_exhausted", + "invocation_budget_exhausted", + "invocation_budget_exceeded", + ) + entries = [ + entry + for entry in (budget_snapshot.get("entries") or []) + if isinstance(entry, dict) + ] + if forced_status == "timeout" or any( + str(entry.get("state") or "") == "timeout" for entry in entries + ): + return ("timeout", "timeout", "model_or_run_timeout") + if forced_status: + return ( + forced_status, + "worker_exception", + "worker_exception", + ) + if sprint.status == "completed": + return ("completed", "sprint_completed", "") + return ( + "failed", + "sprint_not_completed", + "sprint_not_completed", + ) + + +def _preflight_failure_outcome( + *, + started_at: str, + started_monotonic: float, + worker_log: _WorkerLog, + exc: BaseException, +) -> WorkerOutcome: + worker_log.append("preflight_failed", error_category=type(exc).__name__) + duration_ms = max(int((time.monotonic() - started_monotonic) * 1000), 0) + return WorkerOutcome( + status="preflight_failed", + started_at=started_at, + ended_at=_utc_now_iso(), + wall_duration_ms=duration_ms, + worker_duration_ms=duration_ms, + stop_reason="preflight_failed", + error_category=type(exc).__name__, + ) + + +def _run_live_sprint_arm_in_child(context: WorkerContext) -> WorkerOutcome: + started_at = _utc_now_iso() + started_monotonic = time.monotonic() + run_output_dir = context.run_output_dir.expanduser().resolve() + worker_log = _WorkerLog(run_output_dir / "worker.log") + worker_log.append("child_worker_started", run_id=context.arm.run_id) + try: + _validate_context(context) + except Exception as exc: + return _preflight_failure_outcome( + started_at=started_at, + started_monotonic=started_monotonic, + worker_log=worker_log, + exc=exc, + ) + + budget = InvocationBudget( + context.max_invocations, + journal_path=run_output_dir / "call_journal.json", + ) + try: + policy = _build_execution_policy(context, budget=budget) + except Exception as exc: + return _preflight_failure_outcome( + started_at=started_at, + started_monotonic=started_monotonic, + worker_log=worker_log, + exc=exc, + ) + + forced_status = "" + forced_error_category = "" + try: + asyncio.run( + _execute_live_arm( + context, + budget=budget, + policy=policy, + worker_log=worker_log, + ) + ) + except _BenchmarkRunTimeout: + forced_status = "timeout" + forced_error_category = "run_timeout" + except ModelInvocationTimeout: + forced_status = "timeout" + forced_error_category = "model_invocation_timeout" + except InvocationBudgetExceeded: + forced_status = "call_budget_exhausted" + forced_error_category = "invocation_budget_exceeded" + except ModelExecutionPolicyViolation: + forced_status = "preflight_failed" + forced_error_category = "execution_policy_violation" + except Exception as exc: + forced_status = "failed" + forced_error_category = type(exc).__name__ + + sprint_state = _latest_sprint_state(context.workspace_root) + sprint = _sprint_evidence(sprint_state) + quality = _quality_evidence(sprint_state, sprint) + budget_snapshot = budget.snapshot() + status, stop_reason, classified_error = _classify_status( + forced_status=forced_status, + sprint=sprint, + budget_snapshot=budget_snapshot, + ) + error_category = forced_error_category or classified_error + telemetry_records = load_workspace_telemetry(context.workspace_root) + duration_ms = max(int((time.monotonic() - started_monotonic) * 1000), 0) + worker_log.append( + "worker_finished", + error_category=error_category or "none", + invocation_count=len(telemetry_records), + status=status, + ) + return WorkerOutcome( + status=status, # type: ignore[arg-type] + sprint=sprint, + quality=quality, + telemetry_records=telemetry_records, + started_at=started_at, + ended_at=_utc_now_iso(), + wall_duration_ms=duration_ms, + worker_duration_ms=duration_ms, + stop_reason=stop_reason, + error_category=error_category, + ) + + +def _sha256_text(value: str) -> str: + return hashlib.sha256(value.encode("utf-8")).hexdigest() + + +def _write_private_json(path: Path, payload: Mapping[str, Any]) -> None: + path.parent.mkdir(parents=True, mode=0o700, exist_ok=True) + temporary_path = path.parent / f".{path.name}.{uuid.uuid4().hex}.tmp" + try: + descriptor = os.open( + temporary_path, + os.O_WRONLY | os.O_CREAT | os.O_EXCL, + 0o600, + ) + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + json.dump( + payload, + handle, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ) + handle.write("\n") + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary_path, path) + try: + path.chmod(0o600) + except OSError: + pass + finally: + with contextlib.suppress(FileNotFoundError): + temporary_path.unlink() + + +def _read_json_mapping(path: Path) -> dict[str, Any]: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (FileNotFoundError, json.JSONDecodeError, OSError): + return {} + return dict(payload) if isinstance(payload, dict) else {} + + +def _manifest_payload( + context: WorkerContext, + *, + result_path: Path, +) -> dict[str, Any]: + return { + "schema_version": 1, + "benchmark_id": context.benchmark_id, + "arm": { + "pair_index": context.arm.pair_index, + "order_index": context.arm.order_index, + "variant": context.arm.variant, + "run_id": context.arm.run_id, + "prompt_context_enabled": context.arm.prompt_context_enabled, + }, + "workspace_root": str(context.workspace_root.expanduser().resolve()), + "run_output_dir": str(context.run_output_dir.expanduser().resolve()), + "result_path": str(result_path), + "controls": { + "max_invocations": context.max_invocations, + "call_timeout_seconds": context.call_timeout_seconds, + "run_timeout_seconds": context.run_timeout_seconds, + "live": context.live, + }, + "fixture_evidence": { + "milestone_sha256": _sha256_text(context.milestone), + "history_sha256": canonical_hash(context.history_seed), + "history_event_count": len(context.history_seed), + }, + } + + +def _child_context_from_manifest(payload: Mapping[str, Any]) -> tuple[WorkerContext, Path]: + if int(payload.get("schema_version") or 0) != 1: + raise ValueError("Unsupported benchmark worker manifest schema") + workspace_root = Path(str(payload.get("workspace_root") or "")).expanduser().resolve() + run_output_dir = Path(str(payload.get("run_output_dir") or "")).expanduser().resolve() + result_path = Path(str(payload.get("result_path") or "")).expanduser().resolve() + if result_path.parent != run_output_dir: + raise ValueError("Child result path must be inside the run output directory") + + scenario_payload = _read_json_mapping(workspace_root / ".benchmark" / "scenario.json") + milestone = str(scenario_payload.get("milestone") or "").strip() + try: + raw_history = json.loads( + (workspace_root / ".benchmark" / "history_seed.json").read_text( + encoding="utf-8" + ) + ) + except (FileNotFoundError, json.JSONDecodeError, OSError) as exc: + raise ValueError("Unable to load benchmark history fixture") from exc + if not isinstance(raw_history, list) or not all( + isinstance(item, dict) for item in raw_history + ): + raise ValueError("Benchmark history fixture must be a JSON object array") + history_seed = tuple(dict(item) for item in raw_history) + + expected_fixture = ( + dict(payload.get("fixture_evidence") or {}) + if isinstance(payload.get("fixture_evidence"), dict) + else {} + ) + if _sha256_text(milestone) != str( + expected_fixture.get("milestone_sha256") or "" + ): + raise ValueError("Benchmark milestone fixture hash differs from the parent context") + if canonical_hash(history_seed) != str( + expected_fixture.get("history_sha256") or "" + ): + raise ValueError("Benchmark history fixture hash differs from the parent context") + if len(history_seed) != int( + expected_fixture.get("history_event_count") or 0 + ): + raise ValueError("Benchmark history fixture count differs from the parent context") + + raw_arm = ( + dict(payload.get("arm") or {}) + if isinstance(payload.get("arm"), dict) + else {} + ) + variant = str(raw_arm.get("variant") or "") + if variant not in {"before", "after"}: + raise ValueError("Benchmark worker manifest has an invalid arm variant") + arm = ArmPlan( + pair_index=int(raw_arm.get("pair_index") or 0), + order_index=int(raw_arm.get("order_index") or 0), + variant=variant, # type: ignore[arg-type] + run_id=str(raw_arm.get("run_id") or "").strip(), + prompt_context_enabled=bool(raw_arm.get("prompt_context_enabled")), + ) + controls = ( + dict(payload.get("controls") or {}) + if isinstance(payload.get("controls"), dict) + else {} + ) + return ( + WorkerContext( + benchmark_id=str(payload.get("benchmark_id") or "").strip(), + arm=arm, + workspace_root=workspace_root, + run_output_dir=run_output_dir, + milestone=milestone, + history_seed=history_seed, + max_invocations=int(controls.get("max_invocations") or 0), + call_timeout_seconds=float(controls.get("call_timeout_seconds") or 0), + run_timeout_seconds=float(controls.get("run_timeout_seconds") or 0), + live=bool(controls.get("live")), + ), + result_path, + ) + + +def _worker_outcome_payload(outcome: WorkerOutcome) -> dict[str, Any]: + return { + "schema_version": 1, + "status": outcome.status, + "sprint": outcome.sprint.to_dict(), + "quality": outcome.quality.to_dict(), + "telemetry_records": [ + dict(record) for record in outcome.telemetry_records + ], + "started_at": outcome.started_at, + "ended_at": outcome.ended_at, + "wall_duration_ms": outcome.wall_duration_ms, + "worker_duration_ms": outcome.worker_duration_ms, + "stop_reason": outcome.stop_reason, + "error_category": outcome.error_category, + } + + +def _worker_outcome_from_payload(payload: Mapping[str, Any]) -> WorkerOutcome: + if int(payload.get("schema_version") or 0) != 1: + raise ValueError("Unsupported benchmark worker result schema") + status = str(payload.get("status") or "") + if status not in { + "completed", + "failed", + "timeout", + "call_budget_exhausted", + "preflight_failed", + }: + raise ValueError("Benchmark worker result has an invalid status") + raw_sprint = ( + dict(payload.get("sprint") or {}) + if isinstance(payload.get("sprint"), dict) + else {} + ) + sprint = SprintEvidence( + sprint_id=str(raw_sprint.get("sprint_id") or ""), + status=str(raw_sprint.get("status") or ""), + closeout_status=str(raw_sprint.get("closeout_status") or ""), + todo_count=int(raw_sprint.get("todo_count") or 0), + completed_todo_count=int(raw_sprint.get("completed_todo_count") or 0), + blocked_todo_count=int(raw_sprint.get("blocked_todo_count") or 0), + failed_todo_count=int(raw_sprint.get("failed_todo_count") or 0), + commit_sha=str(raw_sprint.get("commit_sha") or ""), + ) + raw_quality = ( + dict(payload.get("quality") or {}) + if isinstance(payload.get("quality"), dict) + else {} + ) + quality = QualityEvidence( + behavior_oracle_passed=bool(raw_quality.get("behavior_oracle_passed")), + sprint_terminal=bool(raw_quality.get("sprint_terminal")), + closeout_verified=bool(raw_quality.get("closeout_verified")), + protected_files_unchanged=bool( + raw_quality.get("protected_files_unchanged") + ), + git_clean=bool(raw_quality.get("git_clean")), + commit_created=bool(raw_quality.get("commit_created")), + no_git_remotes=bool(raw_quality.get("no_git_remotes")), + blocked_todo_count=int(raw_quality.get("blocked_todo_count") or 0), + failed_todo_count=int(raw_quality.get("failed_todo_count") or 0), + notes=tuple( + str(note) + for note in (raw_quality.get("notes") or []) + if str(note).strip() + ), + ) + telemetry_records = tuple( + dict(record) + for record in (payload.get("telemetry_records") or []) + if isinstance(record, dict) + ) + return WorkerOutcome( + status=status, # type: ignore[arg-type] + sprint=sprint, + quality=quality, + telemetry_records=telemetry_records, + started_at=str(payload.get("started_at") or ""), + ended_at=str(payload.get("ended_at") or ""), + wall_duration_ms=max(int(payload.get("wall_duration_ms") or 0), 0), + worker_duration_ms=max(int(payload.get("worker_duration_ms") or 0), 0), + stop_reason=str(payload.get("stop_reason") or ""), + error_category=str(payload.get("error_category") or ""), + ) + + +def _child_environment() -> dict[str, str]: + environment = { + key: value + for key in _CHILD_ENVIRONMENT_KEYS + if (value := os.environ.get(key)) is not None + } + environment["PATH"] = environment.get("PATH") or os.defpath + environment["PYTHONPATH"] = str(_source_import_root()) + environment["PYTHONDONTWRITEBYTECODE"] = "1" + environment["PYTHONUNBUFFERED"] = "1" + environment["NO_COLOR"] = "1" + environment[LIVE_BENCHMARK_ENV] = "1" + return environment + + +def _signal_child_group( + process: subprocess.Popen[Any], + process_signal: signal.Signals, +) -> None: + try: + if hasattr(os, "killpg"): + os.killpg(process.pid, process_signal) + elif process.poll() is None: + process.send_signal(process_signal) + except ProcessLookupError: + pass + + +def _worker_group_alive(process: subprocess.Popen[Any]) -> bool: + if hasattr(os, "killpg"): + return _process_group_exists(process.pid) + return process.poll() is None + + +def _wait_for_cleanup_confirmation( + process: subprocess.Popen[Any], + *, + provider_entries: list[dict[str, Any]], + worker_log: _WorkerLog, + timeout_seconds: float, +) -> None: + deadline = time.monotonic() + max(timeout_seconds, 0.0) + while True: + worker_alive = _worker_group_alive(process) + surviving_providers = [ + entry for entry in provider_entries if _provider_entry_alive(entry) + ] + if not worker_alive and not surviving_providers: + worker_log.append( + "worker_cleanup_confirmed", + provider_group_count=len(provider_entries), + ) + return + _signal_provider_entries(surviving_providers, signal.SIGKILL) + if worker_alive: + _signal_child_group(process, signal.SIGKILL) + if time.monotonic() >= deadline: + worker_log.append( + "worker_cleanup_failed", + provider_group_count=len(surviving_providers), + worker_group_alive=worker_alive, + ) + raise _WorkerCleanupFailure( + "Timed out confirming benchmark worker and provider process termination" + ) + time.sleep(0.05) + + +def _terminate_worker_child( + process: subprocess.Popen[Any], + *, + journal_path: Path, + worker_log: _WorkerLog, +) -> None: + initial_snapshot = _read_json_mapping(journal_path) + provider_entries = _merge_active_process_entries(initial_snapshot) + provider_count = _signal_provider_entries( + provider_entries, + signal.SIGTERM, + ) + worker_log.append( + "parent_provider_termination_requested", + process_count=provider_count, + ) + _signal_child_group(process, signal.SIGTERM) + try: + process.wait(timeout=_CHILD_TERMINATION_GRACE_SECONDS) + except subprocess.TimeoutExpired: + # The worker may reserve and launch a provider between the first journal + # read and delivery of SIGTERM. Re-read before either kill signal. + pre_kill_snapshot = _read_json_mapping(journal_path) + provider_entries = _merge_active_process_entries( + initial_snapshot, + pre_kill_snapshot, + ) + _signal_provider_entries(provider_entries, signal.SIGKILL) + _signal_child_group(process, signal.SIGKILL) + try: + process.wait(timeout=_CHILD_TERMINATION_GRACE_SECONDS) + except subprocess.TimeoutExpired as exc: + worker_log.append( + "worker_process_reap_failed", + provider_group_count=len(provider_entries), + ) + raise _WorkerCleanupFailure( + "Benchmark worker did not exit after SIGKILL" + ) from exc + else: + # Even a worker that exits during its TERM grace window may have written + # a new provider reservation after the initial snapshot. + pre_kill_snapshot = _read_json_mapping(journal_path) + provider_entries = _merge_active_process_entries( + initial_snapshot, + pre_kill_snapshot, + ) + _signal_provider_entries(provider_entries, signal.SIGKILL) + _signal_child_group(process, signal.SIGKILL) + + # Once the worker is reaped it cannot create another call. A final journal + # read closes the remaining write/read race before liveness verification. + final_snapshot = _read_json_mapping(journal_path) + provider_entries = _merge_active_process_entries( + initial_snapshot, + pre_kill_snapshot, + final_snapshot, + ) + _signal_provider_entries(provider_entries, signal.SIGKILL) + _signal_child_group(process, signal.SIGKILL) + _wait_for_cleanup_confirmation( + process, + provider_entries=provider_entries, + worker_log=worker_log, + timeout_seconds=_CHILD_TERMINATION_GRACE_SECONDS, + ) + + +def _partial_outcome( + context: WorkerContext, + *, + status: str, + started_at: str, + started_monotonic: float, + stop_reason: str, + error_category: str, +) -> WorkerOutcome: + sprint_state = _latest_sprint_state(context.workspace_root) + sprint = _sprint_evidence(sprint_state) + quality = _quality_evidence(sprint_state, sprint) + duration_ms = max(int((time.monotonic() - started_monotonic) * 1000), 0) + return WorkerOutcome( + status=status, # type: ignore[arg-type] + sprint=sprint, + quality=quality, + telemetry_records=load_workspace_telemetry(context.workspace_root), + started_at=started_at, + ended_at=_utc_now_iso(), + wall_duration_ms=duration_ms, + worker_duration_ms=duration_ms, + stop_reason=stop_reason, + error_category=error_category, + ) + + +def run_live_sprint_arm(context: WorkerContext) -> WorkerOutcome: + """Run one benchmark arm behind a hard child-process timeout boundary. + + The child executes the production sprint. The parent stores only bounded + controls in its temporary manifest and recovers privacy-safe partial evidence + if the child or one of its provider process groups must be terminated. + """ + + started_at = _utc_now_iso() + started_monotonic = time.monotonic() + run_output_dir = context.run_output_dir.expanduser().resolve() + worker_log = _WorkerLog(run_output_dir / "worker.log", reset=True) + worker_log.append("worker_parent_started", run_id=context.arm.run_id) + try: + _validate_context(context) + except Exception as exc: + return _preflight_failure_outcome( + started_at=started_at, + started_monotonic=started_monotonic, + worker_log=worker_log, + exc=exc, + ) + + nonce = uuid.uuid4().hex + manifest_path = run_output_dir / f".worker-manifest-{nonce}.json" + result_path = run_output_dir / f".worker-result-{nonce}.json" + journal_path = run_output_dir / "call_journal.json" + with contextlib.suppress(FileNotFoundError): + journal_path.unlink() + _write_private_json( + manifest_path, + _manifest_payload(context, result_path=result_path), + ) + with contextlib.suppress(FileNotFoundError): + result_path.unlink() + command = ( + sys.executable, + "-m", + "teams_runtime.benchmarking.worker", + "--child-manifest", + str(manifest_path), + ) + try: + process = subprocess.Popen( + command, + cwd=str(context.workspace_root.expanduser().resolve()), + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + env=_child_environment(), + start_new_session=True, + ) + except Exception as exc: + with contextlib.suppress(FileNotFoundError): + manifest_path.unlink() + worker_log.append("worker_child_launch_failed", error_category=type(exc).__name__) + return _partial_outcome( + context, + status="preflight_failed", + started_at=started_at, + started_monotonic=started_monotonic, + stop_reason="worker_child_launch_failed", + error_category=type(exc).__name__, + ) + + timed_out = False + try: + process.wait(timeout=context.run_timeout_seconds) + except subprocess.TimeoutExpired: + timed_out = True + worker_log.append("worker_child_timeout") + _terminate_worker_child( + process, + journal_path=journal_path, + worker_log=worker_log, + ) + finally: + with contextlib.suppress(FileNotFoundError): + manifest_path.unlink() + + if not timed_out: + # The child's own asyncio deadline can fire just before the parent's. + # Verify cleanup even when process.wait() observed a normal child exit. + _terminate_worker_child( + process, + journal_path=journal_path, + worker_log=worker_log, + ) + + if timed_out: + with contextlib.suppress(FileNotFoundError): + result_path.unlink() + outcome = _partial_outcome( + context, + status="timeout", + started_at=started_at, + started_monotonic=started_monotonic, + stop_reason="run_timeout_exceeded", + error_category="run_timeout", + ) + worker_log.append( + "worker_parent_finished", + invocation_count=len(outcome.telemetry_records), + status=outcome.status, + ) + return outcome + + result_payload = _read_json_mapping(result_path) + with contextlib.suppress(FileNotFoundError): + result_path.unlink() + if process.returncode != 0 or not result_payload: + outcome = _partial_outcome( + context, + status="failed", + started_at=started_at, + started_monotonic=started_monotonic, + stop_reason="worker_child_failed", + error_category="worker_child_failed", + ) + else: + try: + outcome = _worker_outcome_from_payload(result_payload) + except (TypeError, ValueError): + outcome = _partial_outcome( + context, + status="failed", + started_at=started_at, + started_monotonic=started_monotonic, + stop_reason="worker_result_invalid", + error_category="worker_result_invalid", + ) + + parent_duration_ms = max( + int((time.monotonic() - started_monotonic) * 1000), + 0, + ) + outcome = replace( + outcome, + started_at=started_at, + ended_at=_utc_now_iso(), + wall_duration_ms=parent_duration_ms, + ) + worker_log.append( + "worker_parent_finished", + invocation_count=len(outcome.telemetry_records), + status=outcome.status, + ) + return outcome + + +def _child_main(manifest_path: Path) -> int: + payload = _read_json_mapping(manifest_path.expanduser().resolve()) + if not payload: + return 2 + try: + context, result_path = _child_context_from_manifest(payload) + except (TypeError, ValueError): + return 2 + outcome = _run_live_sprint_arm_in_child(context) + _write_private_json(result_path, _worker_outcome_payload(outcome)) + return 0 + + +def _main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument("--child-manifest", type=Path, required=True) + args = parser.parse_args(argv) + return _child_main(args.child_manifest) + + +__all__ = [ + "LIVE_BENCHMARK_ENV", + "run_live_sprint_arm", +] + + +if __name__ == "__main__": + raise SystemExit(_main()) diff --git a/cli.py b/cli.py index 9db3f03..c4925f8 100644 --- a/cli.py +++ b/cli.py @@ -415,6 +415,91 @@ def cmd_metrics( ) +def cmd_benchmark_sprint_ab( + *, + live: bool, + runtime_config: str, + repetitions: int, + max_invocations: int, + call_timeout_seconds: float, + run_timeout_seconds: float, + keep_workspaces: str, + rate_card_file: str = "", + output_dir: str = "", + benchmark_id: str = "", + allow_dirty_source: bool = False, + as_json: bool = False, +) -> int: + from teams_runtime.benchmarking.models import BenchmarkOptions + from teams_runtime.benchmarking.runner import ( + BenchmarkPreflightError, + run_sprint_ab_benchmark, + ) + from teams_runtime.benchmarking.worker import ( + LIVE_BENCHMARK_ENV, + run_live_sprint_arm, + ) + + if not live or os.environ.get(LIVE_BENCHMARK_ENV) != "1": + print( + "Live benchmark calls require both --live and " + f"{LIVE_BENCHMARK_ENV}=1." + ) + return 2 + + normalized_runtime_config = str(runtime_config or "").strip() + if not normalized_runtime_config: + print("--runtime-config is required.") + return 2 + + options = BenchmarkOptions( + source_root=Path(__file__).resolve().parent, + runtime_config_path=Path(normalized_runtime_config).expanduser(), + output_dir=Path(output_dir).expanduser() if str(output_dir or "").strip() else None, + rate_card_path=( + Path(rate_card_file).expanduser() + if str(rate_card_file or "").strip() + else None + ), + repetitions=repetitions, + max_invocations=max_invocations, + call_timeout_seconds=call_timeout_seconds, + run_timeout_seconds=run_timeout_seconds, + keep_workspaces=keep_workspaces, # type: ignore[arg-type] + allow_dirty_source=allow_dirty_source, + live=True, + benchmark_id=str(benchmark_id or "").strip(), + ) + try: + result = run_sprint_ab_benchmark( + options, + worker=run_live_sprint_arm, + ) + except (BenchmarkPreflightError, FileNotFoundError, OSError, ValueError) as exc: + print(f"Benchmark preflight failed: {exc}") + return 2 + + summary = { + "benchmark_id": result.benchmark_id, + "status": result.status, + "classification": result.classification, + "output_dir": str(result.output_dir), + "report_json": str(result.report_json), + "report_markdown": str(result.report_markdown), + "exit_code": result.exit_code, + } + if as_json: + print(json.dumps(summary, ensure_ascii=True, indent=2, sort_keys=True)) + else: + print( + f"benchmark_id={result.benchmark_id} status={result.status} " + f"classification={result.classification}" + ) + print(f"report_json={result.report_json}") + print(f"report_markdown={result.report_markdown}") + return result.exit_code + + def cmd_config_role_set( workspace_root: Path, role: str, @@ -566,6 +651,7 @@ def main(argv: list[str] | None = None) -> int: cmd_goal_stop=cmd_goal_stop, cmd_goal_resume=cmd_goal_resume, cmd_goal_cancel=cmd_goal_cancel, + cmd_benchmark_sprint_ab=cmd_benchmark_sprint_ab, default_relay_transport=DEFAULT_RELAY_TRANSPORT, ) diff --git a/tests/test_benchmark_worker_cleanup.py b/tests/test_benchmark_worker_cleanup.py new file mode 100644 index 0000000..6cb923f --- /dev/null +++ b/tests/test_benchmark_worker_cleanup.py @@ -0,0 +1,236 @@ +from __future__ import annotations + +import signal +import subprocess +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +from teams_runtime.benchmarking import runner, worker +from teams_runtime.benchmarking.models import ( + ArmPlan, + BenchmarkOptions, + BenchmarkWorkerSafetyError, +) +from teams_runtime.benchmarking.scenario import ScenarioWorkspace + + +def _snapshot(*entries: dict[str, object]) -> dict[str, object]: + return { + "schema_version": 1, + "entries": list(entries), + } + + +def _running_entry(pid: int, process_group_id: int) -> dict[str, object]: + return { + "pid": pid, + "process_group_id": process_group_id, + "state": "running", + } + + +class _FakeProcess: + def __init__(self, *wait_results: object): + self.pid = 41001 + self._wait_results = list(wait_results) + self.wait_calls: list[float] = [] + + def wait(self, *, timeout: float) -> int: + self.wait_calls.append(timeout) + result = self._wait_results.pop(0) + if isinstance(result, BaseException): + raise result + return int(result) + + def poll(self) -> int | None: + return None + + def send_signal(self, _process_signal: signal.Signals) -> None: + return None + + +class BenchmarkWorkerCleanupTests(unittest.TestCase): + def test_timeout_cleanup_rereads_journal_before_kill(self) -> None: + first_timeout = subprocess.TimeoutExpired( + cmd=("worker",), + timeout=worker._CHILD_TERMINATION_GRACE_SECONDS, + ) + process = _FakeProcess(first_timeout, -signal.SIGKILL) + initial_entry = _running_entry(42001, 42001) + late_entry = _running_entry(42002, 42002) + snapshots = ( + _snapshot(initial_entry), + _snapshot(initial_entry, late_entry), + _snapshot(initial_entry, late_entry), + ) + log = mock.Mock() + + with ( + mock.patch.object( + worker, + "_read_json_mapping", + side_effect=snapshots, + ) as read_journal, + mock.patch.object( + worker, + "_signal_provider_entries", + return_value=0, + ) as signal_providers, + mock.patch.object(worker, "_signal_child_group") as signal_child, + mock.patch.object( + worker, + "_wait_for_cleanup_confirmation", + ) as confirm_cleanup, + ): + worker._terminate_worker_child( + process, # type: ignore[arg-type] + journal_path=Path("/content-safe/call_journal.json"), + worker_log=log, + ) + + self.assertEqual(read_journal.call_count, 3) + self.assertEqual(len(process.wait_calls), 2) + self.assertEqual(signal_providers.call_args_list[0].args[1], signal.SIGTERM) + pre_kill_entries = signal_providers.call_args_list[1].args[0] + self.assertEqual( + { + (entry["pid"], entry["process_group_id"]) + for entry in pre_kill_entries + }, + {(42001, 42001), (42002, 42002)}, + ) + self.assertEqual(signal_providers.call_args_list[1].args[1], signal.SIGKILL) + self.assertEqual(signal_child.call_args_list[0].args[1], signal.SIGTERM) + self.assertEqual(signal_child.call_args_list[1].args[1], signal.SIGKILL) + confirmed_entries = confirm_cleanup.call_args.kwargs["provider_entries"] + self.assertEqual( + { + (entry["pid"], entry["process_group_id"]) + for entry in confirmed_entries + }, + {(42001, 42001), (42002, 42002)}, + ) + + def test_second_worker_wait_timeout_fails_closed(self) -> None: + first_timeout = subprocess.TimeoutExpired( + cmd=("worker",), + timeout=worker._CHILD_TERMINATION_GRACE_SECONDS, + ) + second_timeout = subprocess.TimeoutExpired( + cmd=("worker",), + timeout=worker._CHILD_TERMINATION_GRACE_SECONDS, + ) + process = _FakeProcess(first_timeout, second_timeout) + snapshots = ( + _snapshot(_running_entry(42001, 42001)), + _snapshot(_running_entry(42001, 42001)), + ) + + with ( + mock.patch.object( + worker, + "_read_json_mapping", + side_effect=snapshots, + ), + mock.patch.object( + worker, + "_signal_provider_entries", + return_value=0, + ), + mock.patch.object(worker, "_signal_child_group"), + mock.patch.object( + worker, + "_wait_for_cleanup_confirmation", + ) as confirm_cleanup, + ): + with self.assertRaises(worker._WorkerCleanupFailure): + worker._terminate_worker_child( + process, # type: ignore[arg-type] + journal_path=Path("/content-safe/call_journal.json"), + worker_log=mock.Mock(), + ) + + self.assertEqual(len(process.wait_calls), 2) + confirm_cleanup.assert_not_called() + + def test_cleanup_confirmation_raises_while_any_group_survives(self) -> None: + process = _FakeProcess(0) + provider_entry = _running_entry(42001, 42001) + + with ( + mock.patch.object(worker, "_worker_group_alive", return_value=False), + mock.patch.object(worker, "_provider_entry_alive", return_value=True), + mock.patch.object(worker, "_signal_provider_entries", return_value=1), + mock.patch.object(worker.time, "monotonic", side_effect=(10.0, 10.0)), + ): + with self.assertRaises(worker._WorkerCleanupFailure): + worker._wait_for_cleanup_confirmation( + process, # type: ignore[arg-type] + provider_entries=[provider_entry], + worker_log=mock.Mock(), + timeout_seconds=0.0, + ) + + def test_runner_does_not_convert_worker_safety_failure_to_arm_result(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + scenario_root = root / "scenario" + scenario_root.mkdir() + scenario = ScenarioWorkspace( + root=scenario_root, + initial_commit="initial", + initial_commit_count=1, + protected_hashes={}, + config_hash="config", + comparable_config_hash="comparable", + history_hash="history", + history_seed=(), + ) + options = BenchmarkOptions( + source_root=root, + runtime_config_path=root / "runtime.yaml", + output_dir=root / "output", + live=True, + ) + arm = ArmPlan( + pair_index=1, + order_index=1, + variant="before", + run_id="pair-001-before", + prompt_context_enabled=False, + ) + + def unsafe_worker(_context: object) -> object: + raise BenchmarkWorkerSafetyError("cleanup could not be confirmed") + + with ( + mock.patch.object( + runner, + "create_scenario_workspace", + return_value=scenario, + ), + mock.patch.object( + runner, + "_capture_retention_baseline", + return_value={}, + ), + mock.patch.object(runner, "_safe_worker_failure") as safe_failure, + ): + with self.assertRaises(BenchmarkWorkerSafetyError): + runner._run_arm( + benchmark_id="safety-test", + output_root=root / "output", + temporary_root=root / "temporary", + options=options, + worker=unsafe_worker, # type: ignore[arg-type] + settings=None, + arm=arm, + ) + + safe_failure.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_sprint_benchmark.py b/tests/test_sprint_benchmark.py new file mode 100644 index 0000000..e5a82b5 --- /dev/null +++ b/tests/test_sprint_benchmark.py @@ -0,0 +1,1130 @@ +from __future__ import annotations + +import json +import io +import os +import shutil +import stat +import subprocess +import tempfile +import unittest +from contextlib import redirect_stdout +from pathlib import Path +from typing import Any +from unittest import mock + +import yaml + +from teams_runtime.benchmarking.metrics import ( + SAFE_INVOCATION_FIELDS, + compare_metrics, + reduce_telemetry, + sanitize_invocation_record, +) +from teams_runtime.benchmarking.models import ( + ArmPlan, + ArmResult, + BenchmarkOptions, + QualityEvidence, + SprintEvidence, + WorkerContext, + WorkerOutcome, + make_arm_schedule, +) +from teams_runtime.benchmarking.reporting import build_report +from teams_runtime.benchmarking.runner import run_sprint_ab_benchmark +from teams_runtime.benchmarking.scenario import ( + PROTECTED_PATHS, + RuntimeSettings, + build_history_seed, + canonical_hash, + create_scenario_workspace, +) +from teams_runtime.cli import build_parser, cmd_benchmark_sprint_ab +from teams_runtime.runtime.execution_policy import ( + InvocationBudget, + InvocationBudgetExceeded, + ModelExecutionPolicy, + ModelExecutionPolicyViolation, +) +from teams_runtime.shared.models import TEAM_ROLES +from teams_runtime.shared.prompt_context import PROMPT_EVENT_SELECTION_POLICY +from teams_runtime.workflows.sprints.lifecycle import apply_initial_plan_confirmation + + +_MISSING = object() +_HISTORY_SEED_HASH = "13024f26fb93918509533bfc5797e4fb3512944ae885234fd6da1393af72a365" + + +def _role_defaults() -> dict[str, dict[str, str]]: + return { + role: { + "model": "gpt-benchmark-test", + "reasoning": "medium", + } + for role in TEAM_ROLES + } + + +def _settings() -> RuntimeSettings: + role_defaults = _role_defaults() + return RuntimeSettings( + role_defaults=role_defaults, + rate_cards={}, + source_config_hash=canonical_hash({"role_defaults": role_defaults, "rate_cards": {}}), + ) + + +def _git(root: Path, *args: str) -> subprocess.CompletedProcess[str]: + environment = { + "HOME": os.environ.get("HOME", ""), + "PATH": os.environ.get("PATH", ""), + "LC_ALL": "C", + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", + } + return subprocess.run( + ("git", *args), + cwd=root, + text=True, + capture_output=True, + check=True, + env=environment, + ) + + +def _initialize_source_repository(root: Path) -> None: + root.mkdir(parents=True) + _git(root, "init") + _git(root, "config", "--local", "user.name", "benchmark-test") + _git(root, "config", "--local", "user.email", "benchmark-test@invalid.local") + _git(root, "config", "--local", "commit.gpgsign", "false") + (root / "source-marker.txt").write_text("benchmark source\n", encoding="utf-8") + _git(root, "add", "source-marker.txt") + _git(root, "commit", "-m", "seed benchmark source") + + +def _write_runtime_config(path: Path) -> None: + path.write_text( + yaml.safe_dump({"role_defaults": _role_defaults()}, sort_keys=False), + encoding="utf-8", + ) + + +def _telemetry_record( + *, + variant: str, + occurrence: int = 1, + native_usage: bool = True, + estimated_cost: float | object = 0.001, + compacted: bool | None = None, +) -> dict[str, Any]: + is_after = variant == "after" + if compacted is None: + compacted = is_after + input_tokens = 240 if is_after else 480 + cached_tokens = 40 if is_after else 80 + output_tokens = 30 + record: dict[str, Any] = { + "schema_version": 1, + "invocation_id": f"{variant}-invocation-{occurrence}", + "operation_id": f"{variant}-operation-{occurrence}", + "logical_call_id": f"{variant}-logical-{occurrence}", + "attempt_index": 1, + "attempt_kind": "primary", + "started_at": f"2026-07-27T00:00:0{occurrence}+00:00", + "ended_at": f"2026-07-27T00:00:1{occurrence}+00:00", + "duration_ms": 400 if is_after else 700, + "runtime_identity": "role", + "role": "developer" if occurrence == 1 else "qa", + "purpose": "implement" if occurrence == 1 else "validate", + "workflow_step": "todo_execution" if occurrence == 1 else "quality_gate", + "request_id": f"request-{occurrence}", + "sprint_id": "benchmark-sprint", + "todo_id": "todo-1", + "provider": "codex_cli", + "model": "gpt-benchmark-test", + "reasoning": "medium", + "status": "completed", + "exit_code": 0, + "prompt_chars": 1800 if is_after else 4200, + "output_chars": 300, + "tool_calls": 2, + "input_tokens": input_tokens, + "cached_input_tokens": cached_tokens, + "output_tokens": output_tokens, + "reasoning_output_tokens": 10, + "total_tokens": input_tokens + output_tokens, + "usage_source": "native" if native_usage else "", + "prompt_context": { + "enabled": is_after, + "compacted": bool(compacted), + "total_events": 48, + "included_events": 16 if compacted else 48, + "omitted_events": 32 if compacted else 0, + "recent_events": 8, + "max_events": 16, + "selection_policy": PROMPT_EVENT_SELECTION_POLICY, + "raw_history": "SENSITIVE_HISTORY_SHOULD_NOT_PERSIST", + }, + "prompt": "SENSITIVE_PROMPT_SHOULD_NOT_PERSIST", + "response": "SENSITIVE_RESPONSE_SHOULD_NOT_PERSIST", + "api_key": "SENSITIVE_API_KEY_SHOULD_NOT_PERSIST", + "session_id": "SENSITIVE_SESSION_ID_SHOULD_NOT_PERSIST", + } + if estimated_cost is not _MISSING: + record["estimated_cost_usd"] = estimated_cost + return record + + +def _passing_quality() -> QualityEvidence: + return QualityEvidence( + behavior_oracle_passed=True, + sprint_terminal=True, + closeout_verified=True, + protected_files_unchanged=True, + git_clean=True, + commit_created=True, + no_git_remotes=True, + ) + + +def _arm_result( + variant: str, + records: tuple[dict[str, Any], ...], + *, + pair_index: int = 1, + order_index: int | None = None, + comparable_config_hash: str = "same-non-feature-config", +) -> ArmResult: + return ArmResult( + arm=ArmPlan( + pair_index=pair_index, + order_index=order_index or (1 if variant == "before" else 2), + variant=variant, # type: ignore[arg-type] + run_id=f"pair-{pair_index:03d}-{variant}", + prompt_context_enabled=variant == "after", + ), + status="completed", + started_at="2026-07-27T00:00:00+00:00", + ended_at="2026-07-27T00:01:00+00:00", + wall_duration_ms=1_000 if variant == "before" else 700, + worker_duration_ms=900 if variant == "before" else 600, + stop_reason="", + error_category="", + config_hash=f"{variant}-config", + comparable_config_hash=comparable_config_hash, + metrics=reduce_telemetry(records), + quality=_passing_quality(), + sprint=SprintEvidence( + sprint_id="benchmark-sprint", + status="completed", + closeout_status="verified", + todo_count=1, + completed_todo_count=1, + commit_sha=f"{variant}-commit", + ), + invocation_records=records, + ) + + +def _report_for_runs( + runs: tuple[ArmResult, ...], + *, + repetitions: int = 1, +) -> dict[str, Any]: + options = BenchmarkOptions( + source_root=Path("/unused/source"), + runtime_config_path=Path("/unused/team_runtime.yaml"), + repetitions=repetitions, + ) + return build_report( + benchmark_id="deterministic-report", + options=options, + source_revision={"commit_sha": "source-sha", "dirty": False}, + source_config_hash="source-config-hash", + runtime_model_map=_role_defaults(), + rate_cards={}, + history_hash=_HISTORY_SEED_HASH, + runs=runs, + started_at="2026-07-27T00:00:00+00:00", + ended_at="2026-07-27T00:02:00+00:00", + ) + + +class SprintBenchmarkScenarioTests(unittest.TestCase): + def test_schedule_alternates_pair_order_and_only_after_enables_compaction(self) -> None: + schedule = make_arm_schedule(3) + + self.assertEqual( + [ + ( + arm.pair_index, + arm.order_index, + arm.variant, + arm.run_id, + arm.prompt_context_enabled, + ) + for arm in schedule + ], + [ + (1, 1, "before", "pair-001-before", False), + (1, 2, "after", "pair-001-after", True), + (2, 3, "after", "pair-002-after", True), + (2, 4, "before", "pair-002-before", False), + (3, 5, "before", "pair-003-before", False), + (3, 6, "after", "pair-003-after", True), + ], + ) + with self.assertRaisesRegex(ValueError, "positive integer"): + make_arm_schedule(0) + + def test_history_seed_is_deterministic_balanced_and_has_a_golden_hash(self) -> None: + first = build_history_seed() + second = build_history_seed() + role_reports = [event for event in first if event["type"] == "role_report"] + + self.assertEqual(first, second) + self.assertEqual(len(first), 48) + self.assertEqual(canonical_hash(first), _HISTORY_SEED_HASH) + self.assertEqual( + [event["actor"] for event in role_reports], + [ + "research", + "planner", + "designer", + "architect", + "developer", + "qa", + "version_controller", + "orchestrator", + ], + ) + self.assertEqual(first[0]["created_at"], "2026-01-01T00:00:00+00:00") + self.assertEqual(first[-1]["created_at"], "2026-01-01T00:47:00+00:00") + with self.assertRaisesRegex(ValueError, "at least 24"): + build_history_seed(23) + + def test_fixture_reproduces_defect_and_config_fingerprints_only_feature_toggle(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + before = create_scenario_workspace( + root / "before", + benchmark_id="fixture-fingerprint", + run_id="pair-001-before", + prompt_context_enabled=False, + settings=_settings(), + ) + after = create_scenario_workspace( + root / "after", + benchmark_id="fixture-fingerprint", + run_id="pair-001-after", + prompt_context_enabled=True, + settings=_settings(), + ) + + before_config = yaml.safe_load( + (before.root / "team_runtime.yaml").read_text(encoding="utf-8") + ) + after_config = yaml.safe_load( + (after.root / "team_runtime.yaml").read_text(encoding="utf-8") + ) + self.assertNotEqual(before.config_hash, after.config_hash) + self.assertEqual(before.comparable_config_hash, after.comparable_config_hash) + self.assertEqual(before.history_hash, after.history_hash) + self.assertEqual(before.history_hash, _HISTORY_SEED_HASH) + self.assertFalse(before_config["prompt_context"]["enabled"]) + self.assertTrue(after_config["prompt_context"]["enabled"]) + before_config["prompt_context"].pop("enabled") + after_config["prompt_context"].pop("enabled") + self.assertEqual(before_config, after_config) + self.assertEqual(before.initial_commit_count, 1) + self.assertEqual(set(before.protected_hashes), set(PROTECTED_PATHS)) + self.assertIn( + "return sum(values)", + (before.root / "benchmark_app.py").read_text(encoding="utf-8"), + ) + baseline = subprocess.run( + ("python", "-m", "unittest", "discover", "-s", "tests"), + cwd=before.root, + text=True, + capture_output=True, + check=False, + env={ + "PATH": os.environ.get("PATH", ""), + "PYTHONPATH": str(before.root), + "LC_ALL": "C", + }, + ) + self.assertNotEqual(baseline.returncode, 0) + self.assertEqual(_git(before.root, "remote").stdout.strip(), "") + + +class SprintBenchmarkCliTests(unittest.TestCase): + def test_parser_exposes_bounded_sprint_ab_defaults(self) -> None: + args = build_parser().parse_args( + [ + "benchmark", + "sprint-ab", + "--runtime-config", + "deployed/team_runtime.yaml", + ] + ) + + self.assertEqual(args.command, "benchmark") + self.assertEqual(args.benchmark_command, "sprint-ab") + self.assertFalse(args.live) + self.assertEqual(args.repetitions, 1) + self.assertEqual(args.max_invocations, 20) + self.assertEqual(args.call_timeout_seconds, 300.0) + self.assertEqual(args.run_timeout_seconds, 1800.0) + self.assertEqual(args.keep_workspaces, "failures") + + def test_cli_requires_both_live_opt_ins_before_calling_runner(self) -> None: + cases = ( + (False, {"TEAMS_RUNTIME_LIVE_BENCHMARK": "1"}), + (True, {}), + ) + for live, environment in cases: + with self.subTest(live=live, environment=environment): + with ( + mock.patch.dict(os.environ, environment, clear=True), + mock.patch( + "teams_runtime.benchmarking.runner.run_sprint_ab_benchmark" + ) as runner, + redirect_stdout(io.StringIO()), + ): + exit_code = cmd_benchmark_sprint_ab( + live=live, + runtime_config="unused.yaml", + repetitions=1, + max_invocations=20, + call_timeout_seconds=300, + run_timeout_seconds=1800, + keep_workspaces="failures", + ) + self.assertEqual(exit_code, 2) + runner.assert_not_called() + + +class SprintBenchmarkReportTests(unittest.TestCase): + def test_fake_worker_generates_comparable_private_full_report(self) -> None: + seen_contexts: list[dict[str, Any]] = [] + + def fake_worker(context: WorkerContext) -> WorkerOutcome: + config = yaml.safe_load( + (context.workspace_root / "team_runtime.yaml").read_text(encoding="utf-8") + ) + seen_contexts.append( + { + "run_id": context.arm.run_id, + "variant": context.arm.variant, + "prompt_context_enabled": config["prompt_context"]["enabled"], + "history_hash": canonical_hash(context.history_seed), + "max_invocations": context.max_invocations, + "call_timeout_seconds": context.call_timeout_seconds, + "run_timeout_seconds": context.run_timeout_seconds, + "live": context.live, + } + ) + target = context.workspace_root / "benchmark_app.py" + self.assertIn("return sum(values)", target.read_text(encoding="utf-8")) + target.write_text( + '"""Small benchmark target with a repaired implementation."""\n\n' + "\n" + "def sum_positive(values):\n" + ' """Return the sum of positive numeric values."""\n' + " return sum(value for value in values if value > 0)\n", + encoding="utf-8", + ) + _git(context.workspace_root, "add", "benchmark_app.py") + _git( + context.workspace_root, + "commit", + "-m", + f"repair fixture for {context.arm.variant}", + ) + commit_sha = _git(context.workspace_root, "rev-parse", "HEAD").stdout.strip() + records = tuple( + _telemetry_record( + variant=context.arm.variant, + occurrence=occurrence, + estimated_cost=0.0005 if context.arm.variant == "after" else 0.001, + ) + for occurrence in (1, 2) + ) + return WorkerOutcome( + status="completed", + sprint=SprintEvidence( + sprint_id="benchmark-sprint", + status="completed", + closeout_status="verified", + todo_count=1, + completed_todo_count=1, + commit_sha=commit_sha, + ), + quality=QualityEvidence( + sprint_terminal=True, + closeout_verified=True, + ), + telemetry_records=records, + started_at="2026-07-27T00:00:00+00:00", + ended_at="2026-07-27T00:00:02+00:00", + wall_duration_ms=1_200 if context.arm.variant == "before" else 800, + worker_duration_ms=1_100 if context.arm.variant == "before" else 700, + ) + + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + source_root = root / "source" + runtime_config = root / "runtime.yaml" + output_root = root / "reports" + _initialize_source_repository(source_root) + _write_runtime_config(runtime_config) + options = BenchmarkOptions( + source_root=source_root, + runtime_config_path=runtime_config, + output_dir=output_root, + repetitions=1, + max_invocations=20, + call_timeout_seconds=300, + run_timeout_seconds=1_800, + keep_workspaces="none", + live=False, + benchmark_id="fake-worker-full-report", + ) + + result = run_sprint_ab_benchmark(options, worker=fake_worker) + + self.assertEqual(result.status, "comparable") + self.assertEqual(result.exit_code, 0) + self.assertEqual(result.classification, "preliminary_smoke") + self.assertTrue(result.report_json.is_file()) + self.assertTrue(result.report_markdown.is_file()) + self.assertEqual( + [(item["variant"], item["prompt_context_enabled"]) for item in seen_contexts], + [("before", False), ("after", True)], + ) + self.assertEqual( + {item["history_hash"] for item in seen_contexts}, + {_HISTORY_SEED_HASH}, + ) + self.assertTrue( + all( + item["max_invocations"] == 20 + and item["call_timeout_seconds"] == 300 + and item["run_timeout_seconds"] == 1_800 + and item["live"] is False + for item in seen_contexts + ) + ) + self.assertNotEqual(result.runs[0].config_hash, result.runs[1].config_hash) + self.assertEqual( + result.runs[0].comparable_config_hash, + result.runs[1].comparable_config_hash, + ) + self.assertTrue(all(run.quality.passed for run in result.runs)) + self.assertTrue(all(not run.retained_workspace for run in result.runs)) + + report = json.loads(result.report_json.read_text(encoding="utf-8")) + pair = report["pairs"][0] + self.assertTrue(pair["comparable"]) + self.assertEqual(pair["execution_order"], ["before", "after"]) + self.assertEqual(pair["inconclusive_reasons"], []) + self.assertEqual( + pair["comparison"]["end_to_end"]["input_tokens"]["reduction"], + 480, + ) + self.assertEqual(pair["comparison"]["matched_primary_count"], 2) + self.assertEqual( + report["aggregate_reductions"]["input_tokens"]["pair_count"], + 1, + ) + self.assertIsNone( + report["aggregate_reductions"]["input_tokens"][ + "sample_standard_deviation" + ] + ) + self.assertFalse( + report["interpretation"]["statistical_significance_claimed"] + ) + self.assertIn("preliminary", report["interpretation"]["note"].lower()) + + for run in result.runs: + run_dir = result.output_dir / "runs" / run.arm.run_id + self.assertTrue((run_dir / "run.json").is_file()) + self.assertTrue((run_dir / "metrics.json").is_file()) + self.assertTrue((run_dir / "model_invocations.jsonl").is_file()) + invocation_lines = ( + run_dir / "model_invocations.jsonl" + ).read_text(encoding="utf-8").splitlines() + self.assertEqual(len(invocation_lines), 2) + persisted = json.loads(invocation_lines[0]) + self.assertLessEqual(set(persisted), SAFE_INVOCATION_FIELDS) + self.assertNotIn("raw_history", persisted["prompt_context"]) + self.assertNotIn( + "invocations", + json.loads((run_dir / "run.json").read_text(encoding="utf-8")), + ) + + for artifact in result.output_dir.rglob("*"): + if artifact.is_file(): + content = artifact.read_text(encoding="utf-8") + self.assertNotIn("SENSITIVE_", content, artifact) + + def test_missing_usage_is_inconclusive_but_missing_price_is_explicitly_unpriced(self) -> None: + before_records = ( + _telemetry_record(variant="before", estimated_cost=0.001), + ) + after_unpriced_records = ( + _telemetry_record(variant="after", estimated_cost=_MISSING), + ) + unpriced_report = _report_for_runs( + ( + _arm_result("before", before_records), + _arm_result("after", after_unpriced_records), + ) + ) + + self.assertEqual(unpriced_report["status"], "comparable") + self.assertEqual( + unpriced_report["runs"][1]["metrics"]["totals"]["pricing_coverage_percent"], + 0.0, + ) + self.assertIsNone( + unpriced_report["runs"][1]["metrics"]["totals"]["estimated_cost_usd"] + ) + cost_delta = unpriced_report["pairs"][0]["comparison"]["end_to_end"][ + "estimated_cost_usd" + ] + self.assertIsNone(cost_delta["delta"]) + self.assertIsNone(cost_delta["reduction_percent"]) + + after_missing_usage = ( + _telemetry_record( + variant="after", + native_usage=False, + estimated_cost=_MISSING, + ), + ) + missing_usage_report = _report_for_runs( + ( + _arm_result("before", before_records), + _arm_result("after", after_missing_usage), + ) + ) + self.assertEqual(missing_usage_report["status"], "inconclusive") + self.assertFalse(missing_usage_report["pairs"][0]["comparable"]) + self.assertIn( + "after_native_token_coverage_incomplete", + missing_usage_report["pairs"][0]["inconclusive_reasons"], + ) + + def test_reducer_reports_partial_coverage_without_inventing_cost_or_usage(self) -> None: + native_priced = _telemetry_record( + variant="before", + occurrence=1, + estimated_cost=0.002, + ) + missing = _telemetry_record( + variant="before", + occurrence=2, + native_usage=False, + estimated_cost=_MISSING, + ) + for field in ( + "input_tokens", + "cached_input_tokens", + "output_tokens", + "reasoning_output_tokens", + "total_tokens", + ): + missing.pop(field) + + metrics = reduce_telemetry((native_priced, missing)) + + self.assertEqual(metrics["totals"]["invocation_count"], 2) + self.assertEqual(metrics["totals"]["token_coverage_percent"], 50.0) + self.assertEqual(metrics["totals"]["pricing_coverage_percent"], 50.0) + self.assertIsNone(metrics["totals"]["estimated_cost_usd"]) + self.assertEqual(metrics["tokens"]["input"], native_priced["input_tokens"]) + self.assertEqual( + {group["estimated_cost_usd"] for group in metrics["groups"]}, + {None, 0.002}, + ) + + def test_native_usage_requires_complete_consistent_provider_counts(self) -> None: + complete = _telemetry_record(variant="before") + missing_output = _telemetry_record(variant="before") + missing_output.pop("output_tokens") + inconsistent_total = _telemetry_record(variant="before") + inconsistent_total["total_tokens"] = ( + inconsistent_total["input_tokens"] + + inconsistent_total["output_tokens"] + - 1 + ) + derived_total = _telemetry_record(variant="before") + derived_total.pop("total_tokens") + + cases = ( + ("complete", complete, 100.0), + ("missing_output", missing_output, 0.0), + ("inconsistent_total", inconsistent_total, 0.0), + ("derived_total", derived_total, 100.0), + ) + for label, record, expected_coverage in cases: + with self.subTest(label=label): + metrics = reduce_telemetry((record,)) + self.assertEqual( + metrics["totals"]["token_coverage_percent"], + expected_coverage, + ) + self.assertEqual( + reduce_telemetry((derived_total,))["tokens"]["total"], + derived_total["input_tokens"] + derived_total["output_tokens"], + ) + + def test_invalid_compaction_projection_cannot_make_a_pair_comparable(self) -> None: + invalid_cases: dict[str, dict[str, Any]] = {} + inconsistent_total = _telemetry_record(variant="after") + inconsistent_total["prompt_context"]["total_events"] = 49 + invalid_cases["inconsistent_total"] = inconsistent_total + below_recent_tail = _telemetry_record(variant="after") + below_recent_tail["prompt_context"].update( + {"included_events": 7, "omitted_events": 41} + ) + invalid_cases["below_recent_tail"] = below_recent_tail + wrong_recent_limit = _telemetry_record(variant="after") + wrong_recent_limit["prompt_context"]["recent_events"] = 7 + invalid_cases["wrong_recent_limit"] = wrong_recent_limit + wrong_max_limit = _telemetry_record(variant="after") + wrong_max_limit["prompt_context"]["max_events"] = 17 + invalid_cases["wrong_max_limit"] = wrong_max_limit + + for label, invalid_after in invalid_cases.items(): + with self.subTest(label=label): + compaction = reduce_telemetry((invalid_after,))["compaction"] + self.assertEqual(compaction["observed_invocation_count"], 1) + self.assertEqual(compaction["invalid_projection_count"], 1) + self.assertEqual(compaction["compacted_invocation_count"], 0) + self.assertEqual(compaction["omitted_events"], 0) + + report = _report_for_runs( + ( + _arm_result( + "before", + (_telemetry_record(variant="before"),), + ), + _arm_result("after", (inconsistent_total,)), + ) + ) + self.assertEqual(report["status"], "inconclusive") + self.assertFalse(report["pairs"][0]["comparable"]) + self.assertIn( + "after_compaction_not_observed", + report["pairs"][0]["inconclusive_reasons"], + ) + + def test_after_arm_rejects_mixed_enabled_and_disabled_projections(self) -> None: + enabled = _telemetry_record(variant="after", occurrence=1) + disabled_short_history = _telemetry_record( + variant="after", + occurrence=2, + compacted=False, + ) + disabled_short_history["prompt_context"].update( + { + "enabled": False, + "total_events": 4, + "included_events": 4, + "omitted_events": 0, + } + ) + + report = _report_for_runs( + ( + _arm_result( + "before", + (_telemetry_record(variant="before"),), + ), + _arm_result( + "after", + (enabled, disabled_short_history), + ), + ) + ) + + self.assertEqual(report["status"], "inconclusive") + reasons = report["pairs"][0]["inconclusive_reasons"] + self.assertIn( + "after_prompt_projection_not_uniformly_enabled", + reasons, + ) + self.assertNotIn("after_disabled_projection_observed", reasons) + + def test_repeated_primary_groups_are_left_unmatched_as_ambiguous(self) -> None: + before_records = [ + _telemetry_record(variant="before", occurrence=index) + for index in (1, 2) + ] + after_records = [ + _telemetry_record(variant="after", occurrence=index) + for index in (1, 2) + ] + for record in (*before_records, *after_records): + record.update( + { + "role": "developer", + "purpose": "implement", + "workflow_step": "todo_execution", + } + ) + + comparison = compare_metrics( + reduce_telemetry(before_records), + reduce_telemetry(after_records), + before_wall_duration_ms=1_000, + after_wall_duration_ms=900, + before_records=before_records, + after_records=after_records, + ) + + self.assertEqual(comparison["matched_primary_count"], 0) + self.assertEqual(comparison["unmatched_before_primary_count"], 2) + self.assertEqual(comparison["unmatched_after_primary_count"], 2) + self.assertEqual(comparison["ambiguous_primary_group_count"], 1) + + def test_sanitizer_drops_prompt_response_secrets_and_nested_history(self) -> None: + sanitized = sanitize_invocation_record(_telemetry_record(variant="after")) + + self.assertLessEqual(set(sanitized), SAFE_INVOCATION_FIELDS) + self.assertNotIn("prompt", sanitized) + self.assertNotIn("response", sanitized) + self.assertNotIn("api_key", sanitized) + self.assertNotIn("session_id", sanitized) + self.assertNotIn("raw_history", sanitized["prompt_context"]) + self.assertEqual(sanitized["prompt_context"]["omitted_events"], 32) + + def test_sanitizer_allowlists_nested_rate_card_fields(self) -> None: + record = _telemetry_record(variant="after") + record["rate_card"] = { + "input_per_million_usd": 1.25, + "cached_input_per_million_usd": 0.25, + "output_per_million_usd": 2.5, + "per_invocation_usd": None, + "api_key": "SENSITIVE_RATE_CARD_SECRET", + "metadata": {"authorization": "SENSITIVE_NESTED_SECRET"}, + } + + sanitized = sanitize_invocation_record(record) + + self.assertEqual( + sanitized["rate_card"], + { + "input_per_million_usd": 1.25, + "cached_input_per_million_usd": 0.25, + "output_per_million_usd": 2.5, + "per_invocation_usd": None, + }, + ) + self.assertNotIn( + "SENSITIVE_", + json.dumps(sanitized["rate_card"], sort_keys=True), + ) + + +class SprintBenchmarkExecutionSafetyTests(unittest.TestCase): + def test_invocation_budget_rejects_the_twenty_first_call_and_journals_no_content(self) -> None: + class InvocationContext: + invocation_id = "safe-invocation-id" + operation_id = "safe-operation-id" + logical_call_id = "safe-logical-id" + attempt_index = 1 + attempt_kind = "primary" + role = "developer" + purpose = "implementation" + workflow_step = "todo_execution" + prompt = "SENSITIVE_BUDGET_PROMPT" + + with tempfile.TemporaryDirectory() as temporary_directory: + journal = Path(temporary_directory) / "budget" / "journal.json" + budget = InvocationBudget(20, journal_path=journal) + reservations = [ + budget.reserve(InvocationContext(), provider="codex_cli") + for _ in range(20) + ] + + self.assertEqual(len({item.reservation_id for item in reservations}), 20) + self.assertEqual(budget.reserved_count, 20) + self.assertEqual(budget.remaining, 0) + with self.assertRaises(InvocationBudgetExceeded) as raised: + budget.reserve(InvocationContext(), provider="codex_cli") + self.assertEqual(raised.exception.max_invocations, 20) + self.assertEqual(raised.exception.reserved_count, 20) + self.assertEqual(budget.rejected_count, 1) + + persisted_text = journal.read_text(encoding="utf-8") + persisted = json.loads(persisted_text) + self.assertEqual(persisted["reserved_count"], 20) + self.assertEqual(persisted["rejected_count"], 1) + self.assertEqual(len(persisted["entries"]), 20) + self.assertNotIn("SENSITIVE_BUDGET_PROMPT", persisted_text) + if os.name == "posix": + self.assertEqual( + stat.S_IMODE(journal.stat().st_mode), + 0o600, + ) + + def test_execution_policy_rejects_secret_environment_and_workspace_escape(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + allowed = root / "allowed" + outside = root / "outside" + allowed.mkdir() + outside.mkdir() + budget = InvocationBudget(20) + + for name in ("OPENAI_API_KEY", "GH_TOKEN", "DATABASE_PASSWORD"): + with self.subTest(name=name): + with self.assertRaisesRegex(ValueError, "secret-bearing"): + ModelExecutionPolicy.for_benchmark( + allowed_workspace_root=allowed, + invocation_budget=budget, + call_timeout_seconds=30, + shell_environment={name: "must-not-leak"}, + ) + + policy = ModelExecutionPolicy.for_benchmark( + allowed_workspace_root=allowed, + invocation_budget=budget, + call_timeout_seconds=30, + shell_environment={"PYTHONPATH": str(allowed)}, + ) + nested = allowed / "nested" + nested.mkdir() + policy.assert_workspace_allowed(nested) + with self.assertRaises(ModelExecutionPolicyViolation): + policy.assert_workspace_allowed(outside) + with self.assertRaises(ModelExecutionPolicyViolation): + policy.assert_workspace_allowed(allowed / ".." / "outside") + if hasattr(os, "symlink"): + escape_link = allowed / "escape-link" + escape_link.symlink_to(outside, target_is_directory=True) + with self.assertRaises(ModelExecutionPolicyViolation): + policy.assert_workspace_allowed(escape_link) + + def test_retained_workspaces_are_allowlisted_sanitized_snapshots(self) -> None: + sensitive_marker = "SENSITIVE_RAW_PROVIDER_AND_SESSION_STATE" + expected_files = { + ".benchmark/history_seed.json", + ".benchmark/scenario.json", + "BENCHMARK_TASK.md", + "RETENTION_NOTICE.md", + "benchmark_app.baseline.py", + "benchmark_app.result.json", + "team_runtime.yaml", + "tests/__init__.py", + "tests/test_benchmark_app.py", + } + + def failing_worker(context: WorkerContext) -> WorkerOutcome: + (context.workspace_root / ".teams_runtime_codex_output.txt").write_text( + sensitive_marker, + encoding="utf-8", + ) + session_file = ( + context.workspace_root + / ".teams_runtime" + / "role_sessions" + / "developer.json" + ) + session_file.parent.mkdir(parents=True, exist_ok=True) + session_file.write_text( + json.dumps({"session_id": sensitive_marker}), + encoding="utf-8", + ) + log_file = context.workspace_root / "logs" / "provider.log" + log_file.parent.mkdir(parents=True, exist_ok=True) + log_file.write_text(sensitive_marker, encoding="utf-8") + (context.workspace_root / "unknown-model-note.txt").write_text( + sensitive_marker, + encoding="utf-8", + ) + for relative_name in ( + ".benchmark/history_seed.json", + ".benchmark/scenario.json", + "BENCHMARK_TASK.md", + "team_runtime.yaml", + ): + (context.workspace_root / relative_name).write_text( + sensitive_marker, + encoding="utf-8", + ) + shutil.rmtree(context.workspace_root / "tests") + redirected_tests = ( + context.workspace_root / ".teams_runtime" / "redirected-tests" + ) + redirected_tests.mkdir(parents=True) + for filename in ("__init__.py", "test_benchmark_app.py"): + (redirected_tests / filename).write_text( + f"# {sensitive_marker}\n", + encoding="utf-8", + ) + (context.workspace_root / "tests").symlink_to( + redirected_tests, + target_is_directory=True, + ) + (context.workspace_root / "benchmark_app.py").unlink() + (context.workspace_root / "benchmark_app.py").symlink_to( + session_file, + ) + return WorkerOutcome( + status="failed", + stop_reason="fixture_failure", + error_category="FixtureFailure", + ) + + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + source_root = root / "source" + runtime_config = root / "runtime.yaml" + _initialize_source_repository(source_root) + _write_runtime_config(runtime_config) + result = run_sprint_ab_benchmark( + BenchmarkOptions( + source_root=source_root, + runtime_config_path=runtime_config, + output_dir=root / "reports", + repetitions=1, + keep_workspaces="failures", + benchmark_id="sanitized-retention", + ), + worker=failing_worker, + ) + + self.assertEqual(len(result.runs), 2) + for run in result.runs: + retained_root = result.output_dir / run.retained_workspace + retained_files = { + path.relative_to(retained_root).as_posix() + for path in retained_root.rglob("*") + if path.is_file() + } + self.assertEqual(retained_files, expected_files) + self.assertFalse((retained_root / ".git").exists()) + self.assertFalse((retained_root / ".teams_runtime").exists()) + self.assertFalse((retained_root / "logs").exists()) + self.assertIn( + "return sum(values)", + (retained_root / "benchmark_app.baseline.py").read_text( + encoding="utf-8" + ), + ) + self.assertEqual( + json.loads( + (retained_root / "benchmark_app.result.json").read_text( + encoding="utf-8" + ) + )["status"], + "missing_or_unsafe", + ) + self.assertIn( + "allowlisted diagnostic snapshot", + (retained_root / "RETENTION_NOTICE.md").read_text( + encoding="utf-8" + ), + ) + + for artifact in result.output_dir.rglob("*"): + if artifact.is_file(): + self.assertNotIn( + sensitive_marker, + artifact.read_text(encoding="utf-8"), + artifact, + ) + + def test_preflight_failure_stops_all_subsequent_arms(self) -> None: + seen_variants: list[str] = [] + + def preflight_failure(context: WorkerContext) -> WorkerOutcome: + seen_variants.append(context.arm.variant) + return WorkerOutcome( + status="preflight_failed", + stop_reason="provider_preflight_failed", + error_category="BenchmarkPreflightError", + ) + + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + source_root = root / "source" + runtime_config = root / "runtime.yaml" + _initialize_source_repository(source_root) + _write_runtime_config(runtime_config) + result = run_sprint_ab_benchmark( + BenchmarkOptions( + source_root=source_root, + runtime_config_path=runtime_config, + output_dir=root / "reports", + repetitions=3, + keep_workspaces="none", + benchmark_id="preflight-short-circuit", + ), + worker=preflight_failure, + ) + + self.assertEqual(seen_variants, ["before"]) + self.assertEqual(len(result.runs), 1) + self.assertEqual(result.runs[0].status, "preflight_failed") + self.assertEqual(result.status, "inconclusive") + self.assertEqual( + result.report["pairs"][0]["inconclusive_reasons"], + ["missing_arm"], + ) + + +class SprintBenchmarkLifecycleTests(unittest.TestCase): + def test_initial_plan_auto_confirmation_preserves_draft_and_records_actor(self) -> None: + draft = { + "revision": 3, + "milestone_title": "Fix deterministic fixture", + "plan_actions": [{"plan_action_id": "PLAN-001", "title": "Repair defect"}], + } + state = { + "initial_plan_confirmation": { + "status": "pending", + "revision": 3, + "draft_proposal": draft, + "plan_artifact": "shared_workspace/sprints/test/implementation_plan.md", + "created_at": "2026-07-27T00:00:00+00:00", + } + } + actor = { + "type": "benchmark_auto_approval", + "id": "sprint-ab-harness", + "name": "Sprint A/B harness", + } + + confirmation = apply_initial_plan_confirmation( + state, + confirmed_by=actor, + message_id="benchmark-auto-confirm", + parser_reason="isolated benchmark policy", + parser_confidence="high", + confirmed_at="2026-07-27T00:01:00+00:00", + ) + + self.assertIs(confirmation, state["initial_plan_confirmation"]) + self.assertEqual(confirmation["status"], "confirmed") + self.assertEqual(confirmation["confirmed_at"], "2026-07-27T00:01:00+00:00") + self.assertEqual(confirmation["updated_at"], "2026-07-27T00:01:00+00:00") + self.assertEqual(confirmation["confirmed_by"], actor) + self.assertEqual(confirmation["confirmed_message_id"], "benchmark-auto-confirm") + self.assertEqual(confirmation["parser_reason"], "isolated benchmark policy") + self.assertEqual(confirmation["parser_confidence"], "high") + self.assertEqual(confirmation["draft_proposal"], draft) + with self.assertRaisesRegex(ValueError, "not awaiting confirmation"): + apply_initial_plan_confirmation(state, confirmed_by=actor) + + +if __name__ == "__main__": + unittest.main() From db1bb759b924f7328dfc2e072acd2094b56a1f2a Mon Sep 17 00:00:00 2001 From: rica-v3 Date: Mon, 27 Jul 2026 21:41:33 +0900 Subject: [PATCH 3/7] performance_benchmarking.md: document sprint cost measurement --- docs/performance_benchmarking.md | 379 +++++++++++++++++++++++++++++++ 1 file changed, 379 insertions(+) create mode 100644 docs/performance_benchmarking.md diff --git a/docs/performance_benchmarking.md b/docs/performance_benchmarking.md new file mode 100644 index 0000000..3a0e939 --- /dev/null +++ b/docs/performance_benchmarking.md @@ -0,0 +1,379 @@ +# Full-Sprint Performance Benchmarking + +This guide defines the repeatable integration benchmark used to quantify model-call +cost and performance changes in `teams_runtime`. It is intentionally separate from +the README because it describes an operator-only live experiment, not normal runtime +startup. + +## Objective + +The benchmark answers a narrow question: + +> For the same complete sprint, how do call count, duration, prompt size, native token +> usage, and optional estimated cost change when request-event prompt compaction is +> enabled? + +The first experiment targets the optimization merged in PR #5. Both arms run the same +merged source revision and deployed role/model configuration: + +| Arm | `prompt_context.enabled` | Event projection | +| --- | --- | --- | +| Before | `false` | The complete request `events` array is embedded in each applicable prompt. | +| After | `true` | At most 16 events are embedded: the latest 8 events plus the latest older evidence for roles not represented in that tail. | + +This is a feature-toggle comparison, not a comparison between two Git commits. Keeping +the source revision fixed removes unrelated code changes from the experiment. + +The default run is one paired smoke test. A single pair can reveal large regressions +and validate measurement coverage, but it is not statistically significant. Provider +routing, tool use, cache state, and model behavior are nondeterministic, so end-to-end +deltas are not attributable to compaction alone. Use repeated pairs before making a +capacity or budget commitment. + +## Full-Sprint Scenario + +Each arm receives a fresh, isolated, no-remote Git repository. The benchmark scaffolds +the normal team workspace and adds a deliberately defective Python function: + +```python +def sum_positive(values): + return sum(values) +``` + +The protected test oracle is: + +```python +assert sum_positive([5, -8, 2]) == 7 +assert sum_positive([-5, 0, -3]) == 0 +assert sum_positive([]) == 0 +``` + +The initial test run must fail. The sprint milestone asks the team to preserve the +public function, fix the behavior, run the `unittest` suite, leave protected benchmark +files unchanged, and commit the result. The complete production sprint workflow is +used: planning, explicit benchmark auto-confirmation, backlog/TODO execution, governed +role handoffs, QA, version control, and closeout. + +An arm passes its behavior and workflow gates only when all of the following are true: + +- the initial defective fixture was reproduced before the arm started +- the final protected behavior oracle passes +- protected scenario and test files have the same SHA-256 hashes +- the sprint reaches a terminal completed state +- closeout is verified +- no TODO is blocked or failed +- at least one task commit exists +- the final Git worktree is clean +- the isolated repository still has no Git remotes +- every persisted invocation has native token usage +- the After arm records at least one real compacted prompt projection +- the Before and After non-feature configuration fingerprints match + +## Backfill + +In this benchmark, **Backfill** means adding a deterministic history prefix to the +request that exercises the sprint TODO. It does not mean importing production +telemetry, replaying customer data, or modifying an existing workspace. + +The benchmark generates 48 content-safe historical events. They contain neutral +checkpoints and one evidence checkpoint for each workflow role: + +```json +[ + { + "created_at": "2026-01-01T00:01:00+00:00", + "type": "role_report", + "actor": "research", + "summary": "Historical research checkpoint 02.", + "payload": { + "role": "research", + "status": "completed", + "summary": "Stable benchmark evidence 02." + } + }, + { + "created_at": "2026-01-01T00:02:00+00:00", + "type": "benchmark_checkpoint", + "actor": "orchestrator", + "summary": "Neutral historical checkpoint 03.", + "payload": {"sequence": 3} + } +] +``` + +The full 48-event sequence and its canonical SHA-256 hash are identical in both arms. +It is prepended only to the internal execution request, before that request's normal +events. This produces a realistic long-history prompt without changing planning +inputs or introducing facts that could change the desired implementation. + +Backfill serves three purposes: + +1. It guarantees that the request exceeds the 16-event compaction threshold. +2. It gives the selector older evidence from every role to preserve. +3. It makes prompt-size and token deltas reproducible across paired runs. + +The generated history is saved as `.benchmark/history_seed.json` inside a retained, +sanitized arm snapshot. Reports store only its hash and counts. The benchmark never +backfills the normal telemetry store with fabricated model invocations. + +## Compaction + +Compaction is executed immediately before an applicable role prompt is built. The +canonical request JSON on disk remains complete. Only the request projection embedded +in the model prompt changes. + +The selection policy is +`recent_tail_plus_latest_role_evidence`: + +1. If compaction is disabled, or the request has at most `max_events`, include every + event. +2. Select the final `recent_events` events. The benchmark uses 8. +3. Record which roles already have evidence in that recent tail. +4. Scan older events from newest to oldest. +5. For each role not yet represented, include its most recent `role_report` evidence. +6. Stop when every available role is represented or `max_events` is reached. The + benchmark uses 16. +7. Restore selected events to chronological order and embed their complete objects, + not summaries. +8. Add a projection notice with total, included, and omitted counts plus the path to + the complete canonical request. + +For example, assume the 48-event backfill is followed by one normal request-created +event. Before compaction, the prompt input contains: + +```json +{ + "request_id": "req-example", + "events": [ + {"type": "role_report", "payload": {"role": "research", "status": "completed"}}, + "... 47 additional deterministic historical events ...", + {"type": "created", "actor": "orchestrator"} + ] +} +``` + +The Before arm embeds all 49 events. The After arm's projection contains the latest +8 events and the most recent older evidence for up to 8 missing roles: + +```json +{ + "compacted": true, + "total_events": 49, + "included_events": 16, + "omitted_events": 33, + "recent_events": 8, + "max_events": 16, + "selection": "recent_tail_plus_latest_role_evidence", + "canonical_request": "./.teams_runtime/requests/req-example.json" +} +``` + +The projected `events` array contains 16 complete event objects in chronological +order. The other 33 events still exist in the canonical request. The prompt tells the +role to open that file only when a decision requires evidence missing from the +projection. + +Telemetry records the projection policy and counts on the physical provider attempt. +This proves that compaction was eligible and executed; token reduction by itself is +not accepted as proof. + +## Running The Benchmark + +Run from the `teams_runtime` source checkout: + +```bash +TEAMS_RUNTIME_LIVE_BENCHMARK=1 \ +python -m teams_runtime benchmark sprint-ab \ + --live \ + --runtime-config ../teams_generated/team_runtime.yaml \ + --repetitions 1 \ + --max-invocations 20 \ + --call-timeout-seconds 300 \ + --run-timeout-seconds 1800 \ + --keep-workspaces failures +``` + +Live calls require both the environment variable and `--live`. Omitting either is a +preflight failure and makes no model call. + +Options: + +| Option | Default | Meaning | +| --- | ---: | --- | +| `--runtime-config PATH` | required | Deployed `team_runtime.yaml` or the workspace directory containing it. | +| `--repetitions N` | `1` | Number of paired experiments. Pair 1 runs Before/After; pair 2 runs After/Before, then order alternates. | +| `--max-invocations N` | `20` | Hard physical model-call cap for each arm, including retries and contract repairs. | +| `--call-timeout-seconds N` | `300` | Hard timeout for one provider process group. | +| `--run-timeout-seconds N` | `1800` | Hard wall-clock timeout for one complete arm. | +| `--keep-workspaces MODE` | `failures` | `none`, `failures`, or `all`. | +| `--rate-card-file PATH` | unset | Optional YAML pricing snapshot. | +| `--output-dir PATH` | `.teams_runtime/benchmarks` | Parent directory for benchmark artifacts. | +| `--benchmark-id ID` | generated | Stable 1-96 character artifact directory name. | +| `--allow-dirty-source` | false | Permit a dirty checkout and record its content-free state hash. | +| `--json` | false | Print a machine-readable completion summary. | + +Exit codes: + +| Code | Meaning | +| ---: | --- | +| `0` | Every pair passed quality and comparability gates. | +| `1` | Artifacts were preserved, but at least one pair is partial or inconclusive. | +| `2` | Usage or preflight failure; no valid benchmark was started. | + +## Execution Safety + +The live worker is deliberately narrower than normal runtime execution: + +- every role uses the internal file relay; no Discord listener or message is used +- sprint GitHub issue publication is replaced with a local `skipped_benchmark` record +- external deep research is disabled and remains an unresolved risk if requested +- the fixture repository has no remote and safe Git configuration disables prompts, + signing, system/global configuration, and repository hooks +- benchmark model execution supports Codex only; Gemini CLI requests are rejected +- approval policy is `never` and sandbox mode is `workspace-write` +- dangerous sandbox-bypass requests and automatic bypass retries are rejected +- MCP servers, web search, plugins, hooks, computer use, and multi-agent features are + disabled for the provider process +- the provider receives only the authentication and transport variables needed by the + outer Codex CLI +- tool shells inherit no outer environment and receive an explicit non-secret + allowlist +- writable paths must resolve inside the isolated arm root +- a shared atomic journal reserves a call before launch, so concurrent roles cannot + exceed the arm's physical-call budget +- provider processes run in dedicated process groups and are terminated on call + timeout +- an arm timeout terminates active provider groups before the worker is stopped + +When a call cap or timeout is reached, the benchmark does not silently raise the +limit. It preserves partial telemetry, marks the arm inconclusive, and proceeds to the +other arm when doing so remains safe. + +Live benchmarks are never run automatically in CI. Deterministic fake-worker +integration tests exercise scheduling, fixtures, aggregation, reporting, and failure +paths without credentials or provider calls. + +## Reports + +Artifacts are written under: + +```text +.teams_runtime/benchmarks// +├── report.json +├── report.md +├── runs/ +│ ├── pair-001-before/ +│ │ ├── run.json +│ │ ├── metrics.json +│ │ ├── model_invocations.jsonl +│ │ ├── sprint.json +│ │ ├── quality.json +│ │ ├── call_journal.json +│ │ └── worker.log +│ └── pair-001-after/ +└── workspaces/ + └── ... sanitized snapshots retained according to --keep-workspaces +``` + +The execution report includes: + +- physical invocation and logical-call counts +- primary, contract-repair, sandbox-retry, failed, and completed counts +- tool-call count and coverage +- provider and end-to-end wall duration, including p50 and p95 provider latency +- prompt and output character counts +- input, cached input, uncached input, output, reasoning-output, and total tokens +- native-token coverage +- optional estimated cost and pricing coverage +- compaction eligibility, executions, total/included/omitted events, and maximum + included events +- per-role/provider/model groups +- matched primary calls keyed by role, purpose, workflow step, and occurrence +- full-sprint Before/After deltas and reduction percentages + +Reports are content-safe: they do not contain prompts, model responses, tool output, +raw errors, raw session IDs, credentials, or environment values. + +### Optional Rate Card + +Pricing is operator-supplied and never fetched automatically. A rate-card file can +use a top-level `rate_cards` mapping: + +```yaml +rate_cards: + "codex_cli/gpt-5.5": + input_per_million_usd: 0.00 + cached_input_per_million_usd: 0.00 + output_per_million_usd: 0.00 + "codex_cli/gpt-5.3-codex-spark": + input_per_million_usd: 0.00 + cached_input_per_million_usd: 0.00 + output_per_million_usd: 0.00 +``` + +Replace zero placeholders with the pricing agreement applicable to the deployment. +Cost is reported only when every invocation in both arms has a matching rate and the +native usage required by that rate. Otherwise cost remains `null`/`unpriced`; partial +coverage is never presented as a complete total. + +## Interpreting Results + +Prioritize evidence in this order: + +1. Verify both arms passed all quality and comparability gates. +2. Verify the same history hash and non-feature configuration hash were used. +3. Verify 100% native token coverage. +4. Verify the After arm recorded actual omissions and the Before arm did not. +5. Compare matched primary calls to isolate prompts that exercised the same role, + purpose, and workflow step. +6. Compare full-sprint totals to capture routing, retries, and downstream effects. +7. Treat cost as an estimate only when pricing coverage is 100%. + +A one-pair report is labeled `preliminary_smoke`; its sample standard deviation is +`null`. For a stronger estimate, rerun with at least three pairs. Execution order +alternates automatically to reduce a simple warm-cache or time-order bias. Do not +combine reports from different source revisions, deployed model maps, history hashes, +or rate-card snapshots. + +## Troubleshooting + +`preflight_failed`: + +- confirm both live opt-ins are present +- confirm the runtime config exists and defines every role model/reasoning pair +- commit source changes, or deliberately use `--allow-dirty-source` +- choose a new benchmark ID or remove only an explicitly disposable old output + +`call_budget_exhausted`: + +- inspect `call_journal.json` and the role/purpose aggregates +- do not increase the cap to make an individual result appear comparable +- simplify the fixture only by creating a new scenario version, or deliberately + schedule a separately documented higher-budget experiment + +`timeout`: + +- inspect `worker.log`, terminal journal entries, and partial invocation telemetry +- distinguish a single provider timeout from the full-arm timeout +- rerun the unchanged pair after a transient provider incident; do not merge a + replacement arm into the old pair + +`native_token_coverage_incomplete`: + +- confirm the installed Codex CLI emits terminal usage in JSON mode +- retain the result as evidence of a measurement gap +- do not substitute character-count estimates for native tokens in a comparable pair + +`after_compaction_not_observed`: + +- verify the deterministic history hash +- inspect the sanitized `model_invocations.jsonl` projection metadata in the arm report +- treat the pair as inconclusive even if input tokens decreased + +Retained paths are sanitized, allowlisted snapshots rather than execution workspaces. +They include only pre-execution benchmark metadata, deterministic fixture/test files, +the baseline `benchmark_app.py`, the benchmark task, and the arm configuration. The +post-execution implementation is represented only by a SHA-256 digest, byte count, and +changed/not-changed flag. Git metadata, `.teams_runtime`, logs, model sessions, provider +output, role workspaces, model-mutated file content, and all unrecognized files are +excluded. Remove snapshots according to the project's normal retention policy. From 2ee886aa234b8b9644fa42339f1fa52b5165e20e Mon Sep 17 00:00:00 2001 From: rica-v3 Date: Mon, 27 Jul 2026 23:53:23 +0900 Subject: [PATCH 4/7] cli.py: restore benchmark JSON output --- cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cli.py b/cli.py index c4925f8..cc56979 100644 --- a/cli.py +++ b/cli.py @@ -2,6 +2,7 @@ import argparse import asyncio +import json import logging import os from collections.abc import Awaitable From b29f2ae0c18fe164ecbdd85279fecee19e77d70f Mon Sep 17 00:00:00 2001 From: rica-v3 Date: Mon, 27 Jul 2026 23:53:35 +0900 Subject: [PATCH 5/7] worker.py: harden live sprint measurement --- benchmarking/metrics.py | 96 ++- benchmarking/models.py | 114 +++- benchmarking/reporting.py | 68 ++- benchmarking/runner.py | 227 ++++++- benchmarking/scenario.py | 2 +- benchmarking/worker.py | 596 ++++++++++++++++-- runtime/benchmark_launcher.py | 39 ++ runtime/codex_runner.py | 76 ++- runtime/execution_policy.py | 10 +- tests/test_benchmark_worker_cleanup.py | 384 +++++++++++- tests/test_execution_policy.py | 145 +++++ tests/test_sprint_benchmark.py | 795 ++++++++++++++++++++++++- 12 files changed, 2448 insertions(+), 104 deletions(-) create mode 100644 runtime/benchmark_launcher.py diff --git a/benchmarking/metrics.py b/benchmarking/metrics.py index c099aff..156c173 100644 --- a/benchmarking/metrics.py +++ b/benchmarking/metrics.py @@ -214,8 +214,34 @@ def _prompt_context(record: Mapping[str, Any]) -> dict[str, Any]: } -def reduce_telemetry(records: Iterable[Mapping[str, Any]]) -> dict[str, Any]: +def reduce_telemetry( + records: Iterable[Mapping[str, Any]], + *, + expected_invocation_count: int | None = None, + coverage_available: bool = True, +) -> dict[str, Any]: normalized = [sanitize_invocation_record(record) for record in records] + if not isinstance(coverage_available, bool): + raise ValueError("coverage_available must be a boolean") + if expected_invocation_count is not None and ( + isinstance(expected_invocation_count, bool) + or not isinstance(expected_invocation_count, int) + or expected_invocation_count < 0 + ): + raise ValueError("expected_invocation_count must be a non-negative integer") + if not coverage_available and expected_invocation_count is not None: + raise ValueError( + "expected_invocation_count must be omitted when coverage is unavailable" + ) + observed_invocation_count = len(normalized) + coverage_denominator = max( + observed_invocation_count, + ( + expected_invocation_count + if expected_invocation_count is not None + else observed_invocation_count + ), + ) logical_calls = { str(record.get("logical_call_id") or "") for record in normalized @@ -223,7 +249,24 @@ def reduce_telemetry(records: Iterable[Mapping[str, Any]]) -> dict[str, Any]: } durations: list[int] = [] totals = { - "invocation_count": len(normalized), + "invocation_count": observed_invocation_count, + "coverage_basis": ( + "call_journal" + if coverage_available and expected_invocation_count is not None + else ( + "observed_telemetry" + if coverage_available + else "unavailable_untrusted_call_journal" + ) + ), + "expected_invocation_count": ( + coverage_denominator if coverage_available else None + ), + "unobserved_invocation_count": ( + max(coverage_denominator - observed_invocation_count, 0) + if coverage_available + else None + ), "logical_call_count": len(logical_calls), "primary_count": 0, "contract_repair_count": 0, @@ -432,21 +475,50 @@ def reduce_telemetry(records: Iterable[Mapping[str, Any]]) -> dict[str, Any]: 12, ) - count = len(normalized) + count = observed_invocation_count for group in groups.values(): - if group.pop("_priced_count") != group["invocation_count"]: + if ( + group.pop("_priced_count") != group["invocation_count"] + or not coverage_available + or count != coverage_denominator + ): group["estimated_cost_usd"] = None - token_coverage = round(native_usage_count * 100 / count, 2) if count else 0.0 - tool_call_coverage = round(tool_call_usage_count * 100 / count, 2) if count else 0.0 - pricing_coverage = round(priced_count * 100 / count, 2) if count else 0.0 - totals["token_coverage_percent"] = token_coverage - totals["tool_call_coverage_percent"] = tool_call_coverage - totals["pricing_coverage_percent"] = pricing_coverage + token_coverage = ( + round(native_usage_count * 100 / coverage_denominator, 2) + if coverage_denominator + else 0.0 + ) + tool_call_coverage = ( + round(tool_call_usage_count * 100 / coverage_denominator, 2) + if coverage_denominator + else 0.0 + ) + pricing_coverage = ( + round(priced_count * 100 / coverage_denominator, 2) + if coverage_denominator + else 0.0 + ) + totals["token_coverage_percent"] = ( + token_coverage if coverage_available else None + ) + totals["tool_call_coverage_percent"] = ( + tool_call_coverage if coverage_available else None + ) + totals["pricing_coverage_percent"] = ( + pricing_coverage if coverage_available else None + ) totals["estimated_cost_usd"] = ( - round(total_cost, 12) if count and priced_count == count else None + round(total_cost, 12) + if coverage_available + and coverage_denominator + and count == coverage_denominator + and priced_count == coverage_denominator + else None ) compaction["unobserved_invocation_count"] = ( - count - compaction["observed_invocation_count"] + coverage_denominator - compaction["observed_invocation_count"] + if coverage_available + else None ) compaction["selection_policies"] = sorted(selection_policies) return { diff --git a/benchmarking/models.py b/benchmarking/models.py index 1fd67a9..4315b50 100644 --- a/benchmarking/models.py +++ b/benchmarking/models.py @@ -1,8 +1,11 @@ from __future__ import annotations +import hashlib +import json +import math from dataclasses import dataclass, field from pathlib import Path -from typing import Any, Literal, Mapping, Protocol, Sequence +from typing import Any, Iterable, Literal, Mapping, Protocol, Sequence BenchmarkVariant = Literal["before", "after"] @@ -14,6 +17,108 @@ "call_budget_exhausted", "preflight_failed", ] +_INVOCATION_ATTEMPT_BOOLEAN_FIELDS = frozenset( + { + "journal_available", + "identity_reconciled", + "reconciled", + } +) +_INVOCATION_ATTEMPT_INTEGER_FIELDS = frozenset( + { + "schema_version", + "journal_schema_version", + "max_invocations", + "reserved_count", + "entry_count", + "telemetry_record_count", + "unobserved_attempt_count", + "telemetry_overage_count", + "completed_count", + "failed_count", + "timeout_count", + "launch_failed_count", + "terminated_count", + "active_count", + "unknown_state_count", + "malformed_entry_count", + "unaccounted_count", + "overaccounted_count", + "rejected_count", + "remaining_budget", + "journal_invocation_id_missing_count", + "journal_invocation_id_duplicate_count", + "telemetry_invocation_id_missing_count", + "telemetry_invocation_id_duplicate_count", + "telemetry_invocation_id_unmatched_count", + "journal_invocation_id_unobserved_count", + } +) +_INVOCATION_ATTEMPT_FLOAT_FIELDS = frozenset( + { + "telemetry_coverage_percent", + } +) +_INVOCATION_ATTEMPT_HASH_FIELDS = frozenset( + { + "journal_invocation_ids_sha256", + } +) + + +def invocation_identity_digest(values: Iterable[Any]) -> str: + normalized = sorted( + str(value or "").strip() + for value in values + ) + canonical = json.dumps( + normalized, + ensure_ascii=True, + separators=(",", ":"), + ) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def sanitize_invocation_attempts( + value: Any, +) -> dict[str, bool | float | int | str]: + """Keep only bounded count fields at the worker/report privacy boundary.""" + + if not isinstance(value, Mapping): + return {} + sanitized: dict[str, bool | float | int | str] = {} + for field_name in _INVOCATION_ATTEMPT_BOOLEAN_FIELDS: + raw_value = value.get(field_name) + if isinstance(raw_value, bool): + sanitized[field_name] = raw_value + for field_name in _INVOCATION_ATTEMPT_INTEGER_FIELDS: + raw_value = value.get(field_name) + if raw_value is None or isinstance(raw_value, bool): + continue + try: + normalized = int(raw_value) + except (TypeError, ValueError): + continue + if normalized >= 0: + sanitized[field_name] = normalized + for field_name in _INVOCATION_ATTEMPT_FLOAT_FIELDS: + raw_value = value.get(field_name) + if isinstance(raw_value, bool): + continue + try: + normalized = float(raw_value) + except (TypeError, ValueError): + continue + if math.isfinite(normalized) and 0.0 <= normalized <= 100.0: + sanitized[field_name] = normalized + for field_name in _INVOCATION_ATTEMPT_HASH_FIELDS: + normalized = str(value.get(field_name) or "").strip().lower() + if ( + len(normalized) == 64 + and all(character in "0123456789abcdef" for character in normalized) + ): + sanitized[field_name] = normalized + return dict(sorted(sanitized.items())) class BenchmarkWorkerSafetyError(RuntimeError): @@ -157,6 +262,7 @@ class WorkerOutcome: sprint: SprintEvidence = field(default_factory=SprintEvidence) quality: QualityEvidence = field(default_factory=QualityEvidence) telemetry_records: tuple[Mapping[str, Any], ...] = () + invocation_attempts: Mapping[str, Any] = field(default_factory=dict) started_at: str = "" ended_at: str = "" wall_duration_ms: int = 0 @@ -185,6 +291,7 @@ class ArmResult: metrics: Mapping[str, Any] quality: QualityEvidence sprint: SprintEvidence + invocation_attempts: Mapping[str, Any] = field(default_factory=dict) invocation_records: tuple[Mapping[str, Any], ...] = () retained_workspace: str = "" @@ -207,6 +314,9 @@ def to_dict(self, *, include_records: bool = False) -> dict[str, Any]: "metrics": dict(self.metrics), "quality": self.quality.to_dict(), "sprint": self.sprint.to_dict(), + "invocation_attempts": sanitize_invocation_attempts( + self.invocation_attempts + ), "retained_workspace": self.retained_workspace, } if include_records: @@ -264,5 +374,7 @@ def make_arm_schedule(repetitions: int) -> tuple[ArmPlan, ...]: "SprintEvidence", "WorkerContext", "WorkerOutcome", + "invocation_identity_digest", "make_arm_schedule", + "sanitize_invocation_attempts", ] diff --git a/benchmarking/reporting.py b/benchmarking/reporting.py index 7c11ceb..53f9ff8 100644 --- a/benchmarking/reporting.py +++ b/benchmarking/reporting.py @@ -18,7 +18,7 @@ from teams_runtime.shared.prompt_context import PROMPT_EVENT_SELECTION_POLICY -REPORT_SCHEMA_VERSION = 1 +REPORT_SCHEMA_VERSION = 2 def utc_now_iso() -> str: @@ -77,6 +77,51 @@ def _pair_comparability(before: ArmResult, after: ArmResult) -> tuple[bool, list reasons.append(f"{label}_status_{arm.status}") if not arm.quality.passed: reasons.append(f"{label}_quality_failed") + attempts = dict(arm.invocation_attempts) + if attempts.get("journal_available") is not True: + reasons.append(f"{label}_call_journal_missing") + elif int(attempts.get("journal_schema_version") or 0) not in {1, 2}: + reasons.append(f"{label}_call_journal_schema_unsupported") + else: + if attempts.get("reconciled") is not True: + reasons.append(f"{label}_call_journal_not_reconciled") + if attempts.get("identity_reconciled") is not True: + reasons.append( + f"{label}_invocation_identity_not_reconciled" + ) + for field_name, reason_suffix in ( + ("active_count", "active_attempts_present"), + ("unknown_state_count", "unknown_attempt_states"), + ("malformed_entry_count", "malformed_attempt_entries"), + ("unaccounted_count", "unaccounted_attempts"), + ("overaccounted_count", "overaccounted_attempts"), + ("telemetry_overage_count", "telemetry_attempt_overage"), + ("unobserved_attempt_count", "unobserved_attempts"), + ("terminated_count", "terminated_attempts"), + ("rejected_count", "rejected_attempts"), + ( + "journal_invocation_id_missing_count", + "journal_invocation_ids_missing", + ), + ( + "journal_invocation_id_duplicate_count", + "journal_invocation_ids_duplicated", + ), + ( + "telemetry_invocation_id_missing_count", + "telemetry_invocation_ids_missing", + ), + ( + "telemetry_invocation_id_duplicate_count", + "telemetry_invocation_ids_duplicated", + ), + ( + "telemetry_invocation_id_unmatched_count", + "telemetry_invocation_ids_unmatched", + ), + ): + if int(attempts.get(field_name) or 0) > 0: + reasons.append(f"{label}_{reason_suffix}") totals = dict(arm.metrics.get("totals") or {}) if float(totals.get("token_coverage_percent") or 0.0) != 100.0: reasons.append(f"{label}_native_token_coverage_incomplete") @@ -267,21 +312,32 @@ def render_markdown(report: Mapping[str, Any]) -> str: "", "## Runs", "", - "| Run | Variant | Status | Calls | Repairs | Input tokens | Total tokens | Wall ms | Quality |", - "| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | --- |", + "| Run | Variant | Status | Reserved | Telemetry | Completed | Failed | Timed out | Launch failed | Terminated | Active | Rejected | Repairs | Input tokens | Total tokens | Wall ms | Quality |", + "| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- |", ] for run in report.get("runs") or []: metrics = dict(run.get("metrics") or {}) totals = dict(metrics.get("totals") or {}) tokens = dict(metrics.get("tokens") or {}) + attempts = dict(run.get("invocation_attempts") or {}) quality = dict(run.get("quality") or {}) lines.append( - "| {run_id} | {variant} | {status} | {calls} | {repairs} | {input_tokens} | " - "{total_tokens} | {wall_ms} | {quality} |".format( + "| {run_id} | {variant} | {status} | {reserved} | {telemetry} | " + "{completed} | {failed} | {timed_out} | {launch_failed} | {terminated} | " + "{active} | {rejected} | {repairs} | {input_tokens} | {total_tokens} | " + "{wall_ms} | {quality} |".format( run_id=run.get("run_id", ""), variant=run.get("variant", ""), status=run.get("status", ""), - calls=totals.get("invocation_count", 0), + reserved=_display(attempts.get("reserved_count")), + telemetry=totals.get("invocation_count", 0), + completed=_display(attempts.get("completed_count")), + failed=_display(attempts.get("failed_count")), + timed_out=_display(attempts.get("timeout_count")), + launch_failed=_display(attempts.get("launch_failed_count")), + terminated=_display(attempts.get("terminated_count")), + active=_display(attempts.get("active_count")), + rejected=_display(attempts.get("rejected_count")), repairs=totals.get("contract_repair_count", 0), input_tokens=tokens.get("input", 0), total_tokens=tokens.get("total", 0), diff --git a/benchmarking/runner.py b/benchmarking/runner.py index 9f05872..46e8717 100644 --- a/benchmarking/runner.py +++ b/benchmarking/runner.py @@ -28,7 +28,9 @@ QualityEvidence, WorkerContext, WorkerOutcome, + invocation_identity_digest, make_arm_schedule, + sanitize_invocation_attempts, ) from teams_runtime.benchmarking.reporting import ( build_report, @@ -156,6 +158,115 @@ def _merge_quality( ) +_REQUIRED_INVOCATION_ATTEMPT_FIELDS = frozenset( + { + "schema_version", + "journal_schema_version", + "max_invocations", + "reserved_count", + "entry_count", + "telemetry_record_count", + "unobserved_attempt_count", + "telemetry_overage_count", + "completed_count", + "failed_count", + "timeout_count", + "launch_failed_count", + "terminated_count", + "active_count", + "unknown_state_count", + "malformed_entry_count", + "unaccounted_count", + "overaccounted_count", + "rejected_count", + "remaining_budget", + "journal_invocation_ids_sha256", + "journal_invocation_id_missing_count", + "journal_invocation_id_duplicate_count", + "telemetry_invocation_id_missing_count", + "telemetry_invocation_id_duplicate_count", + "telemetry_invocation_id_unmatched_count", + "journal_invocation_id_unobserved_count", + } +) +_INVOCATION_ATTEMPT_STATE_COUNTS = ( + "completed_count", + "failed_count", + "timeout_count", + "launch_failed_count", + "terminated_count", + "active_count", + "unknown_state_count", + "malformed_entry_count", +) + + +def _journal_coverage_available( + invocation_attempts: Mapping[str, Any], + *, + expected_max_invocations: int, +) -> bool: + if ( + invocation_attempts.get("journal_available") is not True + or invocation_attempts.get("reconciled") is not True + or invocation_attempts.get("identity_reconciled") is not True + or not _REQUIRED_INVOCATION_ATTEMPT_FIELDS.issubset( + invocation_attempts + ) + ): + return False + summary_schema = int(invocation_attempts["schema_version"]) + journal_schema = int(invocation_attempts["journal_schema_version"]) + maximum = int(invocation_attempts["max_invocations"]) + reserved = int(invocation_attempts["reserved_count"]) + entries = int(invocation_attempts["entry_count"]) + observed = int(invocation_attempts["telemetry_record_count"]) + accounted = sum( + int(invocation_attempts[field_name]) + for field_name in _INVOCATION_ATTEMPT_STATE_COUNTS + ) + return ( + summary_schema == 1 + and journal_schema in {1, 2} + and maximum == expected_max_invocations + and reserved <= maximum + and int(invocation_attempts["remaining_budget"]) + == maximum - reserved + and reserved == entries == accounted + and int(invocation_attempts["unknown_state_count"]) == 0 + and int(invocation_attempts["malformed_entry_count"]) == 0 + and int(invocation_attempts["unaccounted_count"]) == 0 + and int(invocation_attempts["overaccounted_count"]) == 0 + and int( + invocation_attempts["journal_invocation_id_missing_count"] + ) + == 0 + and int( + invocation_attempts["journal_invocation_id_duplicate_count"] + ) + == 0 + and int( + invocation_attempts["telemetry_invocation_id_missing_count"] + ) + == 0 + and int( + invocation_attempts["telemetry_invocation_id_duplicate_count"] + ) + == 0 + and int( + invocation_attempts["telemetry_invocation_id_unmatched_count"] + ) + == 0 + and int( + invocation_attempts["journal_invocation_id_unobserved_count"] + ) + == max(reserved - observed, 0) + and int(invocation_attempts["unobserved_attempt_count"]) + == max(reserved - observed, 0) + and int(invocation_attempts["telemetry_overage_count"]) == 0 + ) + + def _safe_worker_failure( started_at: str, started_monotonic: float, @@ -313,7 +424,120 @@ def _run_arm( if outcome.telemetry_records else load_workspace_telemetry(scenario.root) ) - metrics = reduce_telemetry(raw_records) + invocation_attempts = sanitize_invocation_attempts( + outcome.invocation_attempts + ) + if invocation_attempts.get("journal_available") is True: + reserved_count = int(invocation_attempts.get("reserved_count") or 0) + telemetry_record_count = len(raw_records) + telemetry_invocation_ids = [ + str(record.get("invocation_id") or "").strip() + for record in raw_records + ] + telemetry_nonempty_ids = [ + invocation_id + for invocation_id in telemetry_invocation_ids + if invocation_id + ] + telemetry_missing_id_count = ( + len(telemetry_invocation_ids) - len(telemetry_nonempty_ids) + ) + telemetry_duplicate_id_count = ( + len(telemetry_nonempty_ids) + - len(set(telemetry_nonempty_ids)) + ) + telemetry_identity_digest = invocation_identity_digest( + telemetry_invocation_ids + ) + invocation_attempts.update( + { + "telemetry_record_count": telemetry_record_count, + "unobserved_attempt_count": max( + reserved_count - telemetry_record_count, + 0, + ), + "telemetry_overage_count": max( + telemetry_record_count - reserved_count, + 0, + ), + "telemetry_coverage_percent": ( + round( + min( + telemetry_record_count * 100 / reserved_count, + 100.0, + ), + 2, + ) + if reserved_count + else 0.0 + ), + "telemetry_invocation_id_missing_count": ( + telemetry_missing_id_count + ), + "telemetry_invocation_id_duplicate_count": ( + telemetry_duplicate_id_count + ), + "identity_reconciled": bool( + invocation_attempts.get("identity_reconciled") + and telemetry_missing_id_count == 0 + and telemetry_duplicate_id_count == 0 + and int( + invocation_attempts.get( + "telemetry_invocation_id_unmatched_count" + ) + or 0 + ) + == 0 + and int( + invocation_attempts.get( + "journal_invocation_id_unobserved_count" + ) + or 0 + ) + == max( + reserved_count - telemetry_record_count, + 0, + ) + and ( + telemetry_record_count < reserved_count + or invocation_attempts.get( + "journal_invocation_ids_sha256" + ) + == telemetry_identity_digest + ) + ), + } + ) + accounted_count = sum( + int(invocation_attempts.get(field_name) or 0) + for field_name in _INVOCATION_ATTEMPT_STATE_COUNTS + ) + invocation_attempts["reconciled"] = bool( + invocation_attempts.get("reconciled") + and int(invocation_attempts.get("max_invocations") or 0) + == options.max_invocations + and reserved_count + == int(invocation_attempts.get("entry_count") or 0) + == accounted_count + and int(invocation_attempts.get("unaccounted_count") or 0) == 0 + and int(invocation_attempts.get("overaccounted_count") or 0) == 0 + ) + coverage_available = _journal_coverage_available( + invocation_attempts, + expected_max_invocations=options.max_invocations, + ) + if not coverage_available: + invocation_attempts.pop("telemetry_coverage_percent", None) + expected_invocation_count = ( + int(invocation_attempts.get("reserved_count") or 0) + if coverage_available + else None + ) + metrics = reduce_telemetry( + raw_records, + expected_invocation_count=expected_invocation_count, + coverage_available=coverage_available, + ) quality = _merge_quality(outcome, scenario) result = ArmResult( arm=arm, @@ -329,6 +553,7 @@ def _run_arm( metrics=metrics, quality=quality, sprint=outcome.sprint, + invocation_attempts=invocation_attempts, invocation_records=raw_records, ) keep = options.keep_workspaces == "all" or ( diff --git a/benchmarking/scenario.py b/benchmarking/scenario.py index bc6ea7c..ac82cb4 100644 --- a/benchmarking/scenario.py +++ b/benchmarking/scenario.py @@ -16,7 +16,7 @@ from teams_runtime.shared.models import TEAM_ROLES -SCENARIO_ID = "sum-positive-full-sprint-v1" +SCENARIO_ID = "sum-positive-full-sprint-v2" DEFAULT_HISTORY_SEED_COUNT = 48 BENCHMARK_PROMPT_CONTEXT_RECENT_EVENTS = 8 BENCHMARK_PROMPT_CONTEXT_MAX_EVENTS = 16 diff --git a/benchmarking/worker.py b/benchmarking/worker.py index 08de80c..8a9aea2 100644 --- a/benchmarking/worker.py +++ b/benchmarking/worker.py @@ -9,12 +9,13 @@ import signal import subprocess import sys +import threading import time import uuid from dataclasses import replace from datetime import datetime, timezone from pathlib import Path -from typing import Any, Mapping +from typing import Any, Iterable, Mapping from teams_runtime.benchmarking.metrics import load_workspace_telemetry from teams_runtime.benchmarking.models import ( @@ -24,6 +25,8 @@ SprintEvidence, WorkerContext, WorkerOutcome, + invocation_identity_digest, + sanitize_invocation_attempts, ) from teams_runtime.benchmarking.scenario import ( DEFAULT_HISTORY_SEED_COUNT, @@ -39,7 +42,10 @@ from teams_runtime.shared.models import TEAM_ROLES from teams_runtime.shared.paths import RuntimePaths from teams_runtime.workflows.orchestration.team_service import TeamService -from teams_runtime.workflows.sprints.lifecycle import apply_initial_plan_confirmation +from teams_runtime.workflows.sprints.lifecycle import ( + INITIAL_PHASE_STEP_MILESTONE_REFINEMENT, + apply_initial_plan_confirmation, +) from teams_runtime.workflows.state.sprint_store import iter_sprint_states @@ -49,6 +55,15 @@ _MAX_RESUME_PASSES = 16 _RELAY_POLL_SECONDS = 0.02 _CHILD_TERMINATION_GRACE_SECONDS = 5.0 +_CALL_JOURNAL_STATES = ( + "reserved", + "running", + "completed", + "failed", + "timeout", + "launch_failed", + "terminated", +) _CHILD_ENVIRONMENT_KEYS = ( "CODEX_API_KEY", "CODEX_HOME", @@ -87,6 +102,12 @@ class _WorkerCleanupFailure(BenchmarkWorkerSafetyError): pass +class _BenchmarkHistorySeedState: + def __init__(self) -> None: + self.lock = threading.RLock() + self.request_id = "" + + def _utc_now_iso() -> str: return datetime.now(timezone.utc).isoformat() @@ -127,27 +148,60 @@ def __init__( self, *args: Any, benchmark_context: WorkerContext, + benchmark_history_state: _BenchmarkHistorySeedState | None = None, **kwargs: Any, ): self._benchmark_context = benchmark_context + self._benchmark_history_state = ( + benchmark_history_state or _BenchmarkHistorySeedState() + ) self._benchmark_history_seeded = False super().__init__(*args, **kwargs) - def _create_internal_request_record( - self, - sprint_state: dict[str, Any], - todo: dict[str, Any], - backlog_item: dict[str, Any], - ) -> dict[str, Any]: - request_record = super()._create_internal_request_record( - sprint_state, - todo, - backlog_item, + def _prepare_benchmark_history(self, request_record: dict[str, Any]) -> bool: + params = ( + dict(request_record.get("params") or {}) + if isinstance(request_record.get("params"), dict) + else {} ) - if self._benchmark_history_seeded: - return request_record - + if ( + str(params.get("_teams_kind") or "").strip() != "sprint_internal" + or str(params.get("sprint_phase") or "").strip() != "initial" + or str(params.get("initial_phase_step") or "").strip() + != INITIAL_PHASE_STEP_MILESTONE_REFINEMENT + ): + return False + + request_id = str(request_record.get("request_id") or "").strip() + if not request_id: + raise ValueError("Benchmark history target request must have an id") seed = _copy_history_seed(self._benchmark_context.history_seed) + expected_marker = { + "event_count": len(seed), + "sha256": _history_seed_hash(seed), + } + existing_marker = params.get("_benchmark_history_seed") + if existing_marker is not None: + events = [ + dict(event) + for event in (request_record.get("events") or []) + if isinstance(event, dict) + ] + if ( + existing_marker != expected_marker + or _history_seed_hash(events[: len(seed)]) + != expected_marker["sha256"] + ): + raise ValueError("Benchmark request contains an invalid history seed marker") + seeded_request_id = self._benchmark_history_state.request_id + if seeded_request_id and seeded_request_id != request_id: + raise ValueError( + "Benchmark history seed marker appears on multiple requests" + ) + return True + if self._benchmark_history_state.request_id: + return False + request_record["events"] = [ *seed, *[ @@ -156,19 +210,21 @@ def _create_internal_request_record( if isinstance(event, dict) ], ] - params = ( - dict(request_record.get("params") or {}) - if isinstance(request_record.get("params"), dict) - else {} - ) - params["_benchmark_history_seed"] = { - "event_count": len(seed), - "sha256": canonical_hash(seed), - } + params["_benchmark_history_seed"] = expected_marker request_record["params"] = params - self._save_request(request_record) - self._benchmark_history_seeded = True - return request_record + return True + + def _save_request(self, request_record: dict[str, Any]) -> None: + with self._benchmark_history_state.lock: + history_prepared = self._prepare_benchmark_history(request_record) + super()._save_request(request_record) + if history_prepared: + self._benchmark_history_state.request_id = str( + request_record.get("request_id") or "" + ).strip() + self._benchmark_history_seeded = bool( + self._benchmark_history_state.request_id + ) def _mark_github_publish_skipped(self, sprint_state: dict[str, Any]) -> None: sprint_state["github_issue_number"] = "" @@ -211,6 +267,20 @@ def _copy_history_seed( return [dict(item) for item in payload] +def _history_seed_hash(history_seed: list[dict[str, Any]]) -> str: + normalized: list[dict[str, Any]] = [] + for raw_event in history_seed: + event = dict(raw_event) + created_at = str(event.get("created_at") or "").strip() + if created_at: + parsed = datetime.fromisoformat(created_at.replace("Z", "+00:00")) + if parsed.tzinfo is None: + raise ValueError("Benchmark history timestamps must include a timezone") + event["created_at"] = parsed.astimezone(timezone.utc).isoformat() + normalized.append(event) + return canonical_hash(normalized) + + def _validate_context(context: WorkerContext) -> None: if not context.live: raise ModelExecutionPolicyViolation( @@ -272,6 +342,7 @@ def _build_services( *, policy: ModelExecutionPolicy, ) -> dict[str, _BenchmarkTeamService]: + history_state = _BenchmarkHistorySeedState() services = { role: _BenchmarkTeamService( context.workspace_root, @@ -281,6 +352,7 @@ def _build_services( model_execution_policy=policy, allow_external_research=False, benchmark_context=context, + benchmark_history_state=history_state, ) for role in TEAM_ROLES } @@ -353,6 +425,183 @@ def _merge_active_process_entries( return list(merged.values()) +def _journal_non_negative_int(value: Any, *, default: int = 0) -> int: + if value is None or isinstance(value, bool): + return default + try: + normalized = int(value) + except (TypeError, ValueError): + return default + return normalized if normalized >= 0 else default + + +def _summarize_call_journal( + snapshot: Mapping[str, Any], + *, + telemetry_records: Iterable[Mapping[str, Any]], +) -> dict[str, Any]: + raw_entries = snapshot.get("entries") + raw_entry_list = list(raw_entries) if isinstance(raw_entries, list) else [] + entries = [dict(entry) for entry in raw_entry_list if isinstance(entry, dict)] + malformed_entry_count = len(raw_entry_list) - len(entries) + state_counts = {state: 0 for state in _CALL_JOURNAL_STATES} + unknown_count = 0 + for entry in entries: + state = str(entry.get("state") or "").strip() + if state in state_counts: + state_counts[state] += 1 + else: + unknown_count += 1 + + reserved_count = _journal_non_negative_int(snapshot.get("reserved_count")) + entry_count = len(raw_entry_list) + telemetry_record_list = tuple(telemetry_records) + observed_count = len(telemetry_record_list) + journal_invocation_ids = [ + str(entry.get("invocation_id") or "").strip() + for entry in entries + ] + telemetry_invocation_ids = [ + str(record.get("invocation_id") or "").strip() + for record in telemetry_record_list + ] + journal_nonempty_ids = [ + invocation_id + for invocation_id in journal_invocation_ids + if invocation_id + ] + telemetry_nonempty_ids = [ + invocation_id + for invocation_id in telemetry_invocation_ids + if invocation_id + ] + journal_missing_id_count = ( + len(journal_invocation_ids) - len(journal_nonempty_ids) + ) + telemetry_missing_id_count = ( + len(telemetry_invocation_ids) - len(telemetry_nonempty_ids) + ) + journal_duplicate_id_count = ( + len(journal_nonempty_ids) - len(set(journal_nonempty_ids)) + ) + telemetry_duplicate_id_count = ( + len(telemetry_nonempty_ids) - len(set(telemetry_nonempty_ids)) + ) + journal_id_set = set(journal_nonempty_ids) + telemetry_id_set = set(telemetry_nonempty_ids) + telemetry_unmatched_id_count = len( + telemetry_id_set - journal_id_set + ) + journal_unobserved_id_count = len( + journal_id_set - telemetry_id_set + ) + identity_reconciled = ( + journal_missing_id_count == 0 + and telemetry_missing_id_count == 0 + and journal_duplicate_id_count == 0 + and telemetry_duplicate_id_count == 0 + and telemetry_unmatched_id_count == 0 + ) + unaccounted_count = max(reserved_count - entry_count, 0) + overaccounted_count = max(entry_count - reserved_count, 0) + return { + "schema_version": 1, + "journal_available": bool(snapshot), + "journal_schema_version": _journal_non_negative_int( + snapshot.get("schema_version") + ), + "max_invocations": _journal_non_negative_int( + snapshot.get("max_invocations") + ), + "reserved_count": reserved_count, + "entry_count": entry_count, + "telemetry_record_count": observed_count, + "unobserved_attempt_count": max(reserved_count - observed_count, 0), + "telemetry_overage_count": max(observed_count - reserved_count, 0), + "telemetry_coverage_percent": ( + round(min(observed_count * 100 / reserved_count, 100.0), 2) + if reserved_count + else 0.0 + ), + "identity_reconciled": identity_reconciled, + "journal_invocation_ids_sha256": invocation_identity_digest( + journal_invocation_ids + ), + "journal_invocation_id_missing_count": journal_missing_id_count, + "journal_invocation_id_duplicate_count": journal_duplicate_id_count, + "telemetry_invocation_id_missing_count": telemetry_missing_id_count, + "telemetry_invocation_id_duplicate_count": telemetry_duplicate_id_count, + "telemetry_invocation_id_unmatched_count": ( + telemetry_unmatched_id_count + ), + "journal_invocation_id_unobserved_count": ( + journal_unobserved_id_count + ), + "completed_count": state_counts["completed"], + "failed_count": state_counts["failed"], + "timeout_count": state_counts["timeout"], + "launch_failed_count": state_counts["launch_failed"], + "terminated_count": state_counts["terminated"], + "active_count": state_counts["reserved"] + state_counts["running"], + "unknown_state_count": unknown_count, + "malformed_entry_count": malformed_entry_count, + "unaccounted_count": unaccounted_count, + "overaccounted_count": overaccounted_count, + "reconciled": ( + reserved_count == entry_count + and malformed_entry_count == 0 + and unknown_count == 0 + ), + "rejected_count": _journal_non_negative_int( + snapshot.get("rejected_count") + ), + "remaining_budget": _journal_non_negative_int(snapshot.get("remaining")), + } + + +def _finalize_call_journal_after_cleanup( + journal_path: Path, + snapshot: Mapping[str, Any], + *, + stop_reason: str, +) -> dict[str, Any]: + if not snapshot: + return {} + normalized = dict(snapshot) + entries: list[dict[str, Any]] = [] + changed = False + completed_at = _utc_now_iso() + for raw_entry in (snapshot.get("entries") or []): + if not isinstance(raw_entry, dict): + continue + entry = dict(raw_entry) + if str(entry.get("state") or "").strip() in {"reserved", "running"}: + entry.update( + { + "state": "terminated", + "completed_at": completed_at, + "exit_code": None, + "stop_reason": str(stop_reason or "worker_cleanup").strip(), + } + ) + changed = True + entries.append(entry) + normalized["entries"] = entries + normalized["reserved_count"] = max( + _journal_non_negative_int(snapshot.get("reserved_count")), + len(entries), + ) + if changed: + normalized["schema_version"] = 2 + try: + _write_private_json(journal_path, normalized) + except (OSError, TypeError, ValueError) as exc: + raise _WorkerCleanupFailure( + "Failed to finalize benchmark call journal after cleanup" + ) from exc + return normalized + + def _process_exists(pid: int) -> bool: if pid <= 1: return False @@ -362,6 +611,8 @@ def _process_exists(pid: int) -> bool: return False except PermissionError: return True + except OSError: + return True return True @@ -374,6 +625,8 @@ def _process_group_exists(process_group_id: int) -> bool: return False except PermissionError: return True + except OSError: + return True return True @@ -401,12 +654,18 @@ def _signal_provider_entries( os.killpg(process_group_id, process_signal) except ProcessLookupError: continue + except OSError: + # Cleanup confirmation will fail closed if the group survives. + continue signaled += 1 continue try: os.kill(pid, process_signal) except ProcessLookupError: continue + except OSError: + # Continue so one inaccessible process cannot hide later entries. + continue signaled += 1 return signaled @@ -812,6 +1071,10 @@ def _run_live_sprint_arm_in_child(context: WorkerContext) -> WorkerOutcome: sprint=sprint, quality=quality, telemetry_records=telemetry_records, + invocation_attempts=_summarize_call_journal( + budget_snapshot, + telemetry_records=telemetry_records, + ), started_at=started_at, ended_at=_utc_now_iso(), wall_duration_ms=duration_ms, @@ -863,6 +1126,80 @@ def _read_json_mapping(path: Path) -> dict[str, Any]: return dict(payload) if isinstance(payload, dict) else {} +def _read_call_journal_strict( + path: Path, + *, + required: bool = False, +) -> dict[str, Any]: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError: + if required: + raise _WorkerCleanupFailure( + "Required benchmark call journal is missing" + ) + return {} + except (json.JSONDecodeError, OSError) as exc: + raise _WorkerCleanupFailure( + "Existing benchmark call journal is unreadable or malformed" + ) from exc + if not isinstance(payload, dict): + raise _WorkerCleanupFailure("Benchmark call journal must be a JSON object") + snapshot = dict(payload) + if _journal_non_negative_int(snapshot.get("schema_version")) not in {1, 2}: + raise _WorkerCleanupFailure("Benchmark call journal schema is invalid") + entries = snapshot.get("entries") + if not isinstance(entries, list) or not all( + isinstance(entry, dict) for entry in entries + ): + raise _WorkerCleanupFailure("Benchmark call journal entries are invalid") + for field_name in ( + "max_invocations", + "reserved_count", + "remaining", + "rejected_count", + ): + raw_value = snapshot.get(field_name) + if ( + raw_value is None + or isinstance(raw_value, bool) + or not isinstance(raw_value, int) + or raw_value < 0 + ): + raise _WorkerCleanupFailure( + f"Benchmark call journal {field_name} is invalid" + ) + max_invocations = int(snapshot["max_invocations"]) + reserved_count = int(snapshot["reserved_count"]) + if ( + max_invocations <= 0 + or reserved_count != len(entries) + or reserved_count > max_invocations + or int(snapshot["remaining"]) != max(max_invocations - reserved_count, 0) + ): + raise _WorkerCleanupFailure("Benchmark call journal counts do not reconcile") + reservation_ids: set[str] = set() + for entry in entries: + reservation_id = str(entry.get("reservation_id") or "").strip() + if not reservation_id or reservation_id in reservation_ids: + raise _WorkerCleanupFailure( + "Benchmark call journal reservation ids are missing or duplicated" + ) + reservation_ids.add(reservation_id) + state = str(entry.get("state") or "").strip() + if state not in _CALL_JOURNAL_STATES: + raise _WorkerCleanupFailure( + "Benchmark call journal contains an invalid invocation state" + ) + if state == "running": + pid = entry.get("pid") + if not isinstance(pid, int) or isinstance(pid, bool) or pid <= 1: + raise _WorkerCleanupFailure( + "Running benchmark journal entry has an invalid process id" + ) + return snapshot + + def _manifest_payload( context: WorkerContext, *, @@ -984,6 +1321,9 @@ def _worker_outcome_payload(outcome: WorkerOutcome) -> dict[str, Any]: "telemetry_records": [ dict(record) for record in outcome.telemetry_records ], + "invocation_attempts": sanitize_invocation_attempts( + outcome.invocation_attempts + ), "started_at": outcome.started_at, "ended_at": outcome.ended_at, "wall_duration_ms": outcome.wall_duration_ms, @@ -1048,11 +1388,15 @@ def _worker_outcome_from_payload(payload: Mapping[str, Any]) -> WorkerOutcome: for record in (payload.get("telemetry_records") or []) if isinstance(record, dict) ) + invocation_attempts = sanitize_invocation_attempts( + payload.get("invocation_attempts") + ) return WorkerOutcome( status=status, # type: ignore[arg-type] sprint=sprint, quality=quality, telemetry_records=telemetry_records, + invocation_attempts=invocation_attempts, started_at=str(payload.get("started_at") or ""), ended_at=str(payload.get("ended_at") or ""), wall_duration_ms=max(int(payload.get("wall_duration_ms") or 0), 0), @@ -1086,16 +1430,32 @@ def _signal_child_group( os.killpg(process.pid, process_signal) elif process.poll() is None: process.send_signal(process_signal) - except ProcessLookupError: + except OSError: pass def _worker_group_alive(process: subprocess.Popen[Any]) -> bool: if hasattr(os, "killpg"): + try: + process.poll() + except OSError: + return True return _process_group_exists(process.pid) return process.poll() is None +def _append_cleanup_log( + worker_log: _WorkerLog, + event: str, + **fields: Any, +) -> None: + try: + worker_log.append(event, **fields) + except OSError: + # Diagnostics must not prevent process termination. + pass + + def _wait_for_cleanup_confirmation( process: subprocess.Popen[Any], *, @@ -1110,7 +1470,8 @@ def _wait_for_cleanup_confirmation( entry for entry in provider_entries if _provider_entry_alive(entry) ] if not worker_alive and not surviving_providers: - worker_log.append( + _append_cleanup_log( + worker_log, "worker_cleanup_confirmed", provider_group_count=len(provider_entries), ) @@ -1119,7 +1480,8 @@ def _wait_for_cleanup_confirmation( if worker_alive: _signal_child_group(process, signal.SIGKILL) if time.monotonic() >= deadline: - worker_log.append( + _append_cleanup_log( + worker_log, "worker_cleanup_failed", provider_group_count=len(surviving_providers), worker_group_alive=worker_alive, @@ -1135,54 +1497,104 @@ def _terminate_worker_child( *, journal_path: Path, worker_log: _WorkerLog, -) -> None: - initial_snapshot = _read_json_mapping(journal_path) + stop_reason: str = "worker_cleanup", +) -> dict[str, Any]: + cleanup_failures: list[tuple[str, _WorkerCleanupFailure]] = [] + + def defer_failure(stage: str, failure: _WorkerCleanupFailure) -> None: + cleanup_failures.append((stage, failure)) + _append_cleanup_log( + worker_log, + "worker_cleanup_failure_deferred", + stage=stage, + error_category=type(failure).__name__, + ) + + def read_journal(stage: str) -> dict[str, Any]: + try: + snapshot = _read_call_journal_strict( + journal_path, + required=True, + ) + except _WorkerCleanupFailure as exc: + defer_failure(stage, exc) + return {} + if any( + str(entry.get("state") or "").strip() == "reserved" + for entry in (snapshot.get("entries") or []) + if isinstance(entry, dict) + ): + defer_failure( + f"{stage}_reserved_attempt", + _WorkerCleanupFailure( + "Benchmark provider launch registration was incomplete" + ), + ) + return snapshot + + initial_snapshot = read_journal("initial_journal") provider_entries = _merge_active_process_entries(initial_snapshot) provider_count = _signal_provider_entries( provider_entries, signal.SIGTERM, ) - worker_log.append( + _append_cleanup_log( + worker_log, "parent_provider_termination_requested", process_count=provider_count, ) _signal_child_group(process, signal.SIGTERM) + term_timed_out = False try: process.wait(timeout=_CHILD_TERMINATION_GRACE_SECONDS) except subprocess.TimeoutExpired: - # The worker may reserve and launch a provider between the first journal - # read and delivery of SIGTERM. Re-read before either kill signal. - pre_kill_snapshot = _read_json_mapping(journal_path) - provider_entries = _merge_active_process_entries( - initial_snapshot, - pre_kill_snapshot, + term_timed_out = True + except OSError: + term_timed_out = True + defer_failure( + "worker_term_wait", + _WorkerCleanupFailure( + "Unable to reap benchmark worker after SIGTERM" + ), ) - _signal_provider_entries(provider_entries, signal.SIGKILL) - _signal_child_group(process, signal.SIGKILL) + + # The worker may reserve and launch a provider between the first journal + # read and delivery of SIGTERM. This read is required regardless of whether + # the worker exited during its grace window. + pre_kill_snapshot = read_journal("pre_kill_journal") + provider_entries = _merge_active_process_entries( + initial_snapshot, + pre_kill_snapshot, + ) + _signal_provider_entries(provider_entries, signal.SIGKILL) + _signal_child_group(process, signal.SIGKILL) + if term_timed_out: try: process.wait(timeout=_CHILD_TERMINATION_GRACE_SECONDS) - except subprocess.TimeoutExpired as exc: - worker_log.append( + except subprocess.TimeoutExpired: + _append_cleanup_log( + worker_log, "worker_process_reap_failed", provider_group_count=len(provider_entries), ) - raise _WorkerCleanupFailure( - "Benchmark worker did not exit after SIGKILL" - ) from exc - else: - # Even a worker that exits during its TERM grace window may have written - # a new provider reservation after the initial snapshot. - pre_kill_snapshot = _read_json_mapping(journal_path) - provider_entries = _merge_active_process_entries( - initial_snapshot, - pre_kill_snapshot, - ) - _signal_provider_entries(provider_entries, signal.SIGKILL) - _signal_child_group(process, signal.SIGKILL) + defer_failure( + "worker_kill_wait", + _WorkerCleanupFailure( + "Benchmark worker did not exit after SIGKILL" + ), + ) + except OSError: + defer_failure( + "worker_kill_wait", + _WorkerCleanupFailure( + "Unable to reap benchmark worker after SIGKILL" + ), + ) - # Once the worker is reaped it cannot create another call. A final journal - # read closes the remaining write/read race before liveness verification. - final_snapshot = _read_json_mapping(journal_path) + # Once the worker has received SIGKILL it cannot intentionally launch + # another call. A final read closes the remaining write/read race before + # liveness verification. + final_snapshot = read_journal("final_journal") provider_entries = _merge_active_process_entries( initial_snapshot, pre_kill_snapshot, @@ -1197,6 +1609,30 @@ def _terminate_worker_child( timeout_seconds=_CHILD_TERMINATION_GRACE_SECONDS, ) + finalized_snapshot: dict[str, Any] = {} + if final_snapshot: + try: + finalized_snapshot = _finalize_call_journal_after_cleanup( + journal_path, + final_snapshot, + stop_reason=stop_reason, + ) + except _WorkerCleanupFailure as exc: + defer_failure("journal_finalization", exc) + + if cleanup_failures: + stages = ",".join(stage for stage, _failure in cleanup_failures) + _append_cleanup_log( + worker_log, + "worker_cleanup_failed_closed", + failure_count=len(cleanup_failures), + stages=stages, + ) + raise _WorkerCleanupFailure( + f"Benchmark worker cleanup could not be verified: {stages}" + ) from cleanup_failures[0][1] + return finalized_snapshot + def _partial_outcome( context: WorkerContext, @@ -1211,11 +1647,20 @@ def _partial_outcome( sprint = _sprint_evidence(sprint_state) quality = _quality_evidence(sprint_state, sprint) duration_ms = max(int((time.monotonic() - started_monotonic) * 1000), 0) + telemetry_records = load_workspace_telemetry(context.workspace_root) + journal_snapshot = _read_call_journal_strict( + context.run_output_dir.expanduser().resolve() / "call_journal.json", + required=True, + ) return WorkerOutcome( status=status, # type: ignore[arg-type] sprint=sprint, quality=quality, - telemetry_records=load_workspace_telemetry(context.workspace_root), + telemetry_records=telemetry_records, + invocation_attempts=_summarize_call_journal( + journal_snapshot, + telemetry_records=telemetry_records, + ), started_at=started_at, ended_at=_utc_now_iso(), wall_duration_ms=duration_ms, @@ -1254,6 +1699,22 @@ def run_live_sprint_arm(context: WorkerContext) -> WorkerOutcome: journal_path = run_output_dir / "call_journal.json" with contextlib.suppress(FileNotFoundError): journal_path.unlink() + try: + _write_private_json( + journal_path, + { + "schema_version": 2, + "max_invocations": context.max_invocations, + "reserved_count": 0, + "remaining": context.max_invocations, + "rejected_count": 0, + "entries": [], + }, + ) + except (OSError, TypeError, ValueError) as exc: + raise _WorkerCleanupFailure( + "Failed to initialize benchmark call journal" + ) from exc _write_private_json( manifest_path, _manifest_payload(context, result_path=result_path), @@ -1291,15 +1752,17 @@ def run_live_sprint_arm(context: WorkerContext) -> WorkerOutcome: ) timed_out = False + final_journal_snapshot: dict[str, Any] = {} try: process.wait(timeout=context.run_timeout_seconds) except subprocess.TimeoutExpired: timed_out = True worker_log.append("worker_child_timeout") - _terminate_worker_child( + final_journal_snapshot = _terminate_worker_child( process, journal_path=journal_path, worker_log=worker_log, + stop_reason="run_timeout_exceeded", ) finally: with contextlib.suppress(FileNotFoundError): @@ -1308,10 +1771,11 @@ def run_live_sprint_arm(context: WorkerContext) -> WorkerOutcome: if not timed_out: # The child's own asyncio deadline can fire just before the parent's. # Verify cleanup even when process.wait() observed a normal child exit. - _terminate_worker_child( + final_journal_snapshot = _terminate_worker_child( process, journal_path=journal_path, worker_log=worker_log, + stop_reason="worker_exit_cleanup", ) if timed_out: @@ -1366,6 +1830,14 @@ def run_live_sprint_arm(context: WorkerContext) -> WorkerOutcome: started_at=started_at, ended_at=_utc_now_iso(), wall_duration_ms=parent_duration_ms, + invocation_attempts=( + _summarize_call_journal( + final_journal_snapshot, + telemetry_records=outcome.telemetry_records, + ) + if final_journal_snapshot + else outcome.invocation_attempts + ), ) worker_log.append( "worker_parent_finished", diff --git a/runtime/benchmark_launcher.py b/runtime/benchmark_launcher.py new file mode 100644 index 0000000..1070bed --- /dev/null +++ b/runtime/benchmark_launcher.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import os +import sys + + +_READY_BYTE = b"\x01" + + +def main(argv: list[str] | None = None) -> int: + arguments = list(sys.argv[1:] if argv is None else argv) + if len(arguments) < 4 or arguments[0] != "--ready-fd": + return 64 + try: + ready_fd = int(arguments[1]) + except ValueError: + return 64 + if arguments[2] != "--": + return 64 + command = arguments[3:] + if not command: + return 64 + + try: + with os.fdopen(ready_fd, "rb", closefd=True) as ready_pipe: + ready = ready_pipe.read(1) + except OSError: + return 70 + if ready != _READY_BYTE: + return 70 + + try: + os.execvpe(command[0], command, os.environ) + except OSError: + return 71 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/runtime/codex_runner.py b/runtime/codex_runner.py index 0edec83..62a4645 100644 --- a/runtime/codex_runner.py +++ b/runtime/codex_runner.py @@ -6,6 +6,7 @@ import re import signal import subprocess +import sys import time from datetime import datetime from pathlib import Path @@ -75,6 +76,10 @@ "TMP", "TMPDIR", ) +_BENCHMARK_LAUNCHER_PATH = Path(__file__).with_name( + "benchmark_launcher.py" +) +_BENCHMARK_LAUNCH_READY_BYTE = b"\x01" def _nested_mapping(payload: Any, *keys: str) -> dict[str, Any]: @@ -423,33 +428,76 @@ def _run_benchmark_process( env: dict[str, str], reservation: InvocationReservation, ) -> subprocess.CompletedProcess[str]: - process = subprocess.Popen( - command, - cwd=str(cwd), - stdin=subprocess.PIPE if stdin_input is not None else None, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - env=env, - start_new_session=True, - ) - process_group_id: int | None + if os.name != "posix": + raise ModelExecutionPolicyViolation( + "Benchmark provider launch requires POSIX file-descriptor handoff." + ) + ready_read_fd, ready_write_fd = os.pipe() try: - process_group_id = os.getpgid(process.pid) if hasattr(os, "getpgid") else None - except ProcessLookupError: - process_group_id = process.pid + try: + process = subprocess.Popen( + [ + sys.executable, + str(_BENCHMARK_LAUNCHER_PATH), + "--ready-fd", + str(ready_read_fd), + "--", + *command, + ], + cwd=str(cwd), + stdin=( + subprocess.PIPE + if stdin_input is not None + else None + ), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + start_new_session=True, + pass_fds=(ready_read_fd,), + ) + finally: + os.close(ready_read_fd) + except BaseException: + try: + os.close(ready_write_fd) + except OSError: + pass + raise + + # The launcher is the future provider process: exec preserves both its + # PID and its session/process-group identity. + process_group_id = process.pid try: reservation.mark_started( pid=process.pid, process_group_id=process_group_id, ) + if ( + os.write( + ready_write_fd, + _BENCHMARK_LAUNCH_READY_BYTE, + ) + != len(_BENCHMARK_LAUNCH_READY_BYTE) + ): + raise OSError("Benchmark provider launch handoff was incomplete.") except BaseException: + try: + os.close(ready_write_fd) + except OSError: + pass self._terminate_process_group( process, process_group_id=process_group_id, grace_seconds=float(self.execution_policy.kill_grace_seconds), ) raise + else: + try: + os.close(ready_write_fd) + except OSError: + pass timeout_seconds = float(self.execution_policy.call_timeout_seconds or 0) try: diff --git a/runtime/execution_policy.py b/runtime/execution_policy.py index 82c9479..cdbe933 100644 --- a/runtime/execution_policy.py +++ b/runtime/execution_policy.py @@ -19,7 +19,13 @@ r"(?:^|_)(?:API_?KEY|AUTH|CREDENTIALS?|PASSWORD|SECRET|TOKEN)(?:$|_)", re.IGNORECASE, ) -_TERMINAL_STATES = {"completed", "failed", "timeout", "launch_failed"} +_TERMINAL_STATES = { + "completed", + "failed", + "timeout", + "launch_failed", + "terminated", +} def _utc_timestamp() -> str: @@ -196,7 +202,7 @@ def snapshot(self) -> dict[str, Any]: def _snapshot_locked(self) -> dict[str, Any]: return { - "schema_version": 1, + "schema_version": 2, "max_invocations": self.max_invocations, "reserved_count": len(self._entries), "remaining": max(self.max_invocations - len(self._entries), 0), diff --git a/tests/test_benchmark_worker_cleanup.py b/tests/test_benchmark_worker_cleanup.py index 6cb923f..4f9e8ec 100644 --- a/tests/test_benchmark_worker_cleanup.py +++ b/tests/test_benchmark_worker_cleanup.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json import signal import subprocess import tempfile @@ -70,7 +71,7 @@ def test_timeout_cleanup_rereads_journal_before_kill(self) -> None: with ( mock.patch.object( worker, - "_read_json_mapping", + "_read_call_journal_strict", side_effect=snapshots, ) as read_journal, mock.patch.object( @@ -83,11 +84,16 @@ def test_timeout_cleanup_rereads_journal_before_kill(self) -> None: worker, "_wait_for_cleanup_confirmation", ) as confirm_cleanup, + mock.patch.object( + worker, + "_write_private_json", + ) as write_journal, ): - worker._terminate_worker_child( + final_snapshot = worker._terminate_worker_child( process, # type: ignore[arg-type] journal_path=Path("/content-safe/call_journal.json"), worker_log=log, + stop_reason="run_timeout_exceeded", ) self.assertEqual(read_journal.call_count, 3) @@ -101,6 +107,20 @@ def test_timeout_cleanup_rereads_journal_before_kill(self) -> None: }, {(42001, 42001), (42002, 42002)}, ) + self.assertEqual( + [entry["state"] for entry in final_snapshot["entries"]], + ["terminated", "terminated"], + ) + self.assertTrue( + all( + entry["stop_reason"] == "run_timeout_exceeded" + for entry in final_snapshot["entries"] + ) + ) + write_journal.assert_called_once_with( + Path("/content-safe/call_journal.json"), + final_snapshot, + ) self.assertEqual(signal_providers.call_args_list[1].args[1], signal.SIGKILL) self.assertEqual(signal_child.call_args_list[0].args[1], signal.SIGTERM) self.assertEqual(signal_child.call_args_list[1].args[1], signal.SIGKILL) @@ -126,12 +146,13 @@ def test_second_worker_wait_timeout_fails_closed(self) -> None: snapshots = ( _snapshot(_running_entry(42001, 42001)), _snapshot(_running_entry(42001, 42001)), + _snapshot(_running_entry(42001, 42001)), ) with ( mock.patch.object( worker, - "_read_json_mapping", + "_read_call_journal_strict", side_effect=snapshots, ), mock.patch.object( @@ -144,6 +165,10 @@ def test_second_worker_wait_timeout_fails_closed(self) -> None: worker, "_wait_for_cleanup_confirmation", ) as confirm_cleanup, + mock.patch.object( + worker, + "_write_private_json", + ) as write_journal, ): with self.assertRaises(worker._WorkerCleanupFailure): worker._terminate_worker_child( @@ -153,7 +178,358 @@ def test_second_worker_wait_timeout_fails_closed(self) -> None: ) self.assertEqual(len(process.wait_calls), 2) - confirm_cleanup.assert_not_called() + confirm_cleanup.assert_called_once() + write_journal.assert_called_once() + + def test_journal_read_failures_are_deferred_until_cleanup_is_confirmed( + self, + ) -> None: + entries = ( + _running_entry(42001, 42001), + _running_entry(42002, 42002), + _running_entry(42003, 42003), + ) + + for failed_read_index in range(3): + with self.subTest(failed_read_index=failed_read_index): + process = _FakeProcess(0) + side_effects: list[object] = [ + _snapshot(entries[0]), + _snapshot(entries[1]), + _snapshot(entries[2]), + ] + side_effects[failed_read_index] = ( + worker._WorkerCleanupFailure("journal unavailable") + ) + + with ( + mock.patch.object( + worker, + "_read_call_journal_strict", + side_effect=side_effects, + ) as read_journal, + mock.patch.object( + worker, + "_signal_provider_entries", + return_value=0, + ) as signal_providers, + mock.patch.object( + worker, + "_signal_child_group", + ) as signal_child, + mock.patch.object( + worker, + "_wait_for_cleanup_confirmation", + ) as confirm_cleanup, + mock.patch.object( + worker, + "_write_private_json", + ) as write_journal, + ): + with self.assertRaisesRegex( + worker._WorkerCleanupFailure, + "cleanup could not be verified", + ): + worker._terminate_worker_child( + process, # type: ignore[arg-type] + journal_path=Path( + "/content-safe/call_journal.json" + ), + worker_log=mock.Mock(), + ) + + self.assertEqual(read_journal.call_count, 3) + self.assertTrue( + all( + call.kwargs["required"] is True + for call in read_journal.call_args_list + ) + ) + self.assertEqual(len(process.wait_calls), 1) + child_signals = [ + call.args[1] + for call in signal_child.call_args_list + ] + self.assertEqual(child_signals[0], signal.SIGTERM) + self.assertIn(signal.SIGKILL, child_signals) + confirm_cleanup.assert_called_once() + confirmed_entries = confirm_cleanup.call_args.kwargs[ + "provider_entries" + ] + self.assertEqual( + { + (entry["pid"], entry["process_group_id"]) + for entry in confirmed_entries + }, + { + (entry["pid"], entry["process_group_id"]) + for index, entry in enumerate(entries) + if index != failed_read_index + }, + ) + final_provider_sweep = ( + signal_providers.call_args_list[-1] + ) + self.assertEqual( + final_provider_sweep.args[1], + signal.SIGKILL, + ) + self.assertEqual( + { + (entry["pid"], entry["process_group_id"]) + for entry in final_provider_sweep.args[0] + }, + { + (entry["pid"], entry["process_group_id"]) + for index, entry in enumerate(entries) + if index != failed_read_index + }, + ) + if failed_read_index == 2: + write_journal.assert_not_called() + else: + write_journal.assert_called_once() + + def test_all_journal_reads_failing_still_cleans_up_worker_group(self) -> None: + process = _FakeProcess(0) + journal_failure = worker._WorkerCleanupFailure( + "journal unavailable" + ) + + with ( + mock.patch.object( + worker, + "_read_call_journal_strict", + side_effect=( + journal_failure, + journal_failure, + journal_failure, + ), + ) as read_journal, + mock.patch.object( + worker, + "_signal_provider_entries", + return_value=0, + ), + mock.patch.object( + worker, + "_signal_child_group", + ) as signal_child, + mock.patch.object( + worker, + "_wait_for_cleanup_confirmation", + ) as confirm_cleanup, + mock.patch.object( + worker, + "_write_private_json", + ) as write_journal, + ): + with self.assertRaisesRegex( + worker._WorkerCleanupFailure, + "initial_journal,pre_kill_journal,final_journal", + ): + worker._terminate_worker_child( + process, # type: ignore[arg-type] + journal_path=Path( + "/content-safe/call_journal.json" + ), + worker_log=mock.Mock(), + ) + + self.assertEqual(read_journal.call_count, 3) + self.assertEqual(len(process.wait_calls), 1) + self.assertEqual( + [call.args[1] for call in signal_child.call_args_list], + [signal.SIGTERM, signal.SIGKILL, signal.SIGKILL], + ) + confirm_cleanup.assert_called_once() + self.assertEqual( + confirm_cleanup.call_args.kwargs["provider_entries"], + [], + ) + write_journal.assert_not_called() + + def test_reserved_launch_observed_during_cleanup_fails_closed(self) -> None: + process = _FakeProcess(0) + reserved_entry = { + "reservation_id": "reservation-1", + "state": "reserved", + } + snapshot = _snapshot(reserved_entry) + + with ( + mock.patch.object( + worker, + "_read_call_journal_strict", + side_effect=(snapshot, snapshot, snapshot), + ), + mock.patch.object( + worker, + "_signal_provider_entries", + return_value=0, + ), + mock.patch.object( + worker, + "_signal_child_group", + ) as signal_child, + mock.patch.object( + worker, + "_wait_for_cleanup_confirmation", + ) as confirm_cleanup, + mock.patch.object( + worker, + "_write_private_json", + ) as write_journal, + ): + with self.assertRaisesRegex( + worker._WorkerCleanupFailure, + "reserved_attempt", + ): + worker._terminate_worker_child( + process, # type: ignore[arg-type] + journal_path=Path( + "/content-safe/call_journal.json" + ), + worker_log=mock.Mock(), + ) + + self.assertEqual( + [call.args[1] for call in signal_child.call_args_list], + [signal.SIGTERM, signal.SIGKILL, signal.SIGKILL], + ) + confirm_cleanup.assert_called_once() + write_journal.assert_called_once() + + def test_existing_malformed_journal_fails_closed(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + journal_path = Path(temporary_directory) / "call_journal.json" + journal_path.write_text("{not-json", encoding="utf-8") + + with self.assertRaisesRegex( + worker._WorkerCleanupFailure, + "unreadable or malformed", + ): + worker._read_call_journal_strict(journal_path) + + journal_path.unlink() + self.assertEqual( + worker._read_call_journal_strict(journal_path), + {}, + ) + with self.assertRaisesRegex( + worker._WorkerCleanupFailure, + "Required benchmark call journal is missing", + ): + worker._read_call_journal_strict( + journal_path, + required=True, + ) + + def test_journal_reservation_ids_must_be_unique_and_present(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + journal_path = Path(temporary_directory) / "call_journal.json" + cases = { + "duplicate": [ + {"reservation_id": "same", "state": "completed"}, + {"reservation_id": "same", "state": "completed"}, + ], + "missing": [ + {"reservation_id": "first", "state": "completed"}, + {"state": "completed"}, + ], + } + for label, entries in cases.items(): + with self.subTest(label=label): + journal_path.write_text( + json.dumps( + { + "schema_version": 2, + "max_invocations": 2, + "reserved_count": 2, + "remaining": 0, + "rejected_count": 0, + "entries": entries, + } + ), + encoding="utf-8", + ) + with self.assertRaisesRegex( + worker._WorkerCleanupFailure, + "reservation ids", + ): + worker._read_call_journal_strict(journal_path) + + def test_call_journal_summary_keeps_attempt_and_telemetry_counts_distinct(self) -> None: + states = [ + *(["completed"] * 5), + *(["timeout"] * 2), + "terminated", + ] + summary = worker._summarize_call_journal( + { + "schema_version": 2, + "max_invocations": 20, + "reserved_count": 8, + "remaining": 12, + "rejected_count": 1, + "entries": [ + { + "state": state, + "invocation_id": f"invocation-{index}", + } + for index, state in enumerate(states) + ], + }, + telemetry_records=tuple( + {"invocation_id": f"invocation-{index}"} + for index in range(7) + ), + ) + + self.assertEqual(summary["reserved_count"], 8) + self.assertEqual(summary["telemetry_record_count"], 7) + self.assertEqual(summary["unobserved_attempt_count"], 1) + self.assertEqual(summary["telemetry_coverage_percent"], 87.5) + self.assertEqual(summary["completed_count"], 5) + self.assertEqual(summary["timeout_count"], 2) + self.assertEqual(summary["terminated_count"], 1) + self.assertEqual(summary["active_count"], 0) + self.assertEqual(summary["rejected_count"], 1) + self.assertTrue(summary["identity_reconciled"]) + self.assertEqual( + summary["journal_invocation_id_unobserved_count"], + 1, + ) + + def test_journal_finalization_failure_fails_closed(self) -> None: + snapshot = { + "schema_version": 2, + "max_invocations": 1, + "reserved_count": 1, + "remaining": 0, + "rejected_count": 0, + "entries": [ + { + "reservation_id": "reservation-1", + "state": "running", + } + ], + } + + with mock.patch.object( + worker, + "_write_private_json", + side_effect=OSError("disk full"), + ): + with self.assertRaisesRegex( + worker._WorkerCleanupFailure, + "Failed to finalize", + ): + worker._finalize_call_journal_after_cleanup( + Path("/content-safe/call_journal.json"), + snapshot, + stop_reason="run_timeout_exceeded", + ) def test_cleanup_confirmation_raises_while_any_group_survives(self) -> None: process = _FakeProcess(0) diff --git a/tests/test_execution_policy.py b/tests/test_execution_policy.py index d728c06..ddabf16 100644 --- a/tests/test_execution_policy.py +++ b/tests/test_execution_policy.py @@ -8,6 +8,7 @@ from pathlib import Path from unittest import mock +from teams_runtime.runtime import benchmark_launcher from teams_runtime.runtime.codex_runner import CodexRunner from teams_runtime.runtime.execution_policy import ( InvocationBudget, @@ -156,8 +157,152 @@ def test_real_timeout_marks_journal_and_terminates_process_group(self) -> None: persisted = json.loads( (root / "call_journal.json").read_text(encoding="utf-8") ) + self.assertEqual(persisted["schema_version"], 2) self.assertEqual(persisted["entries"][0]["state"], "timeout") + def test_provider_launch_waits_for_durable_pid_registration(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory).resolve() + policy, _budget = self._policy(root) + runner = CodexRunner( + RoleRuntimeConfig(model="gpt-benchmark"), + execution_policy=policy, + ) + process = mock.Mock(pid=43210, returncode=0) + process.communicate.return_value = ("provider output", "") + events: list[str] = [] + reservation = mock.Mock() + reservation.mark_started.side_effect = ( + lambda **_kwargs: events.append("registered") + ) + + def release_provider( + file_descriptor: int, + payload: bytes, + ) -> int: + self.assertEqual(file_descriptor, 12) + self.assertEqual(payload, b"\x01") + events.append("released") + return len(payload) + + command = [sys.executable, "-c", "print('provider')"] + with ( + mock.patch( + "teams_runtime.runtime.codex_runner.os.pipe", + return_value=(11, 12), + ), + mock.patch( + "teams_runtime.runtime.codex_runner.os.close", + ) as close_fd, + mock.patch( + "teams_runtime.runtime.codex_runner.os.write", + side_effect=release_provider, + ), + mock.patch( + "teams_runtime.runtime.codex_runner.subprocess.Popen", + return_value=process, + ) as popen, + ): + completed = runner._run_benchmark_process( + command, + cwd=root, + stdin_input=None, + env={"PATH": os.defpath}, + reservation=reservation, + ) + + self.assertEqual(events, ["registered", "released"]) + reservation.mark_started.assert_called_once_with( + pid=43210, + process_group_id=43210, + ) + launch_command = popen.call_args.args[0] + self.assertEqual( + launch_command[-len(command):], + command, + ) + self.assertEqual( + popen.call_args.kwargs["pass_fds"], + (11,), + ) + self.assertEqual(close_fd.call_args_list[0].args, (11,)) + self.assertEqual(close_fd.call_args_list[-1].args, (12,)) + self.assertEqual(completed.returncode, 0) + + def test_registration_failure_never_releases_provider(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory).resolve() + policy, _budget = self._policy(root) + runner = CodexRunner( + RoleRuntimeConfig(model="gpt-benchmark"), + execution_policy=policy, + ) + process = mock.Mock(pid=43210, returncode=None) + reservation = mock.Mock() + reservation.mark_started.side_effect = OSError( + "journal unavailable" + ) + + with ( + mock.patch( + "teams_runtime.runtime.codex_runner.os.pipe", + return_value=(11, 12), + ), + mock.patch( + "teams_runtime.runtime.codex_runner.os.close", + ) as close_fd, + mock.patch( + "teams_runtime.runtime.codex_runner.os.write", + ) as release_provider, + mock.patch( + "teams_runtime.runtime.codex_runner.subprocess.Popen", + return_value=process, + ), + mock.patch.object( + runner, + "_terminate_process_group", + return_value=("", ""), + ) as terminate_process, + ): + with self.assertRaisesRegex( + OSError, + "journal unavailable", + ): + runner._run_benchmark_process( + [sys.executable, "-c", "print('provider')"], + cwd=root, + stdin_input=None, + env={"PATH": os.defpath}, + reservation=reservation, + ) + + release_provider.assert_not_called() + self.assertIn(mock.call(11), close_fd.call_args_list) + self.assertIn(mock.call(12), close_fd.call_args_list) + terminate_process.assert_called_once() + + def test_launcher_exits_without_exec_when_parent_closes_gate(self) -> None: + ready_read_fd, ready_write_fd = os.pipe() + os.close(ready_write_fd) + + with mock.patch.object( + benchmark_launcher.os, + "execvpe", + ) as execute_provider: + exit_code = benchmark_launcher.main( + [ + "--ready-fd", + str(ready_read_fd), + "--", + sys.executable, + "-c", + "print('must not run')", + ] + ) + + self.assertEqual(exit_code, 70) + execute_provider.assert_not_called() + if __name__ == "__main__": unittest.main() diff --git a/tests/test_sprint_benchmark.py b/tests/test_sprint_benchmark.py index e5a82b5..0130eee 100644 --- a/tests/test_sprint_benchmark.py +++ b/tests/test_sprint_benchmark.py @@ -9,6 +9,7 @@ import tempfile import unittest from contextlib import redirect_stdout +from dataclasses import replace from pathlib import Path from typing import Any from unittest import mock @@ -29,10 +30,14 @@ SprintEvidence, WorkerContext, WorkerOutcome, + invocation_identity_digest, make_arm_schedule, ) from teams_runtime.benchmarking.reporting import build_report -from teams_runtime.benchmarking.runner import run_sprint_ab_benchmark +from teams_runtime.benchmarking.runner import ( + _journal_coverage_available, + run_sprint_ab_benchmark, +) from teams_runtime.benchmarking.scenario import ( PROTECTED_PATHS, RuntimeSettings, @@ -40,6 +45,11 @@ canonical_hash, create_scenario_workspace, ) +from teams_runtime.benchmarking.worker import ( + _BenchmarkHistorySeedState, + _BenchmarkTeamService, + _history_seed_hash, +) from teams_runtime.cli import build_parser, cmd_benchmark_sprint_ab from teams_runtime.runtime.execution_policy import ( InvocationBudget, @@ -49,6 +59,7 @@ ) from teams_runtime.shared.models import TEAM_ROLES from teams_runtime.shared.prompt_context import PROMPT_EVENT_SELECTION_POLICY +from teams_runtime.workflows.orchestration.team_service import TeamService from teams_runtime.workflows.sprints.lifecycle import apply_initial_plan_confirmation @@ -197,6 +208,15 @@ def _arm_result( order_index: int | None = None, comparable_config_hash: str = "same-non-feature-config", ) -> ArmResult: + invocation_count = len(records) + completed_count = sum( + str(record.get("status") or "") == "completed" + for record in records + ) + invocation_ids = [ + str(record.get("invocation_id") or "") + for record in records + ] return ArmResult( arm=ArmPlan( pair_index=pair_index, @@ -224,6 +244,40 @@ def _arm_result( completed_todo_count=1, commit_sha=f"{variant}-commit", ), + invocation_attempts={ + "schema_version": 1, + "journal_available": True, + "journal_schema_version": 2, + "reconciled": True, + "identity_reconciled": True, + "max_invocations": 20, + "reserved_count": invocation_count, + "entry_count": invocation_count, + "telemetry_record_count": invocation_count, + "unobserved_attempt_count": 0, + "telemetry_overage_count": 0, + "completed_count": completed_count, + "failed_count": invocation_count - completed_count, + "timeout_count": 0, + "launch_failed_count": 0, + "terminated_count": 0, + "active_count": 0, + "unknown_state_count": 0, + "malformed_entry_count": 0, + "unaccounted_count": 0, + "overaccounted_count": 0, + "rejected_count": 0, + "remaining_budget": 20 - invocation_count, + "journal_invocation_ids_sha256": ( + invocation_identity_digest(invocation_ids) + ), + "journal_invocation_id_missing_count": 0, + "journal_invocation_id_duplicate_count": 0, + "telemetry_invocation_id_missing_count": 0, + "telemetry_invocation_id_duplicate_count": 0, + "telemetry_invocation_id_unmatched_count": 0, + "journal_invocation_id_unobserved_count": 0, + }, invocation_records=records, ) @@ -333,6 +387,12 @@ def test_fixture_reproduces_defect_and_config_fingerprints_only_feature_toggle(s self.assertEqual(before.comparable_config_hash, after.comparable_config_hash) self.assertEqual(before.history_hash, after.history_hash) self.assertEqual(before.history_hash, _HISTORY_SEED_HASH) + scenario = json.loads( + (before.root / ".benchmark" / "scenario.json").read_text( + encoding="utf-8" + ) + ) + self.assertEqual(scenario["scenario_id"], "sum-positive-full-sprint-v2") self.assertFalse(before_config["prompt_context"]["enabled"]) self.assertTrue(after_config["prompt_context"]["enabled"]) before_config["prompt_context"].pop("enabled") @@ -360,6 +420,250 @@ def test_fixture_reproduces_defect_and_config_fingerprints_only_feature_toggle(s self.assertEqual(_git(before.root, "remote").stdout.strip(), "") +class SprintBenchmarkBackfillTests(unittest.TestCase): + @staticmethod + def _service( + state: _BenchmarkHistorySeedState | None = None, + ) -> _BenchmarkTeamService: + service = object.__new__(_BenchmarkTeamService) + service._benchmark_context = mock.Mock(history_seed=build_history_seed()) + service._benchmark_history_state = state or _BenchmarkHistorySeedState() + service._benchmark_history_seeded = False + return service + + def test_first_sprint_request_is_seeded_before_persistence(self) -> None: + service = self._service() + persisted: list[dict[str, Any]] = [] + request_record = { + "request_id": "planning-request", + "params": { + "_teams_kind": "sprint_internal", + "sprint_phase": "initial", + "initial_phase_step": "milestone_refinement", + }, + "events": [{"type": "created", "actor": "sprint_runner"}], + } + second_request = { + "request_id": "todo-request", + "params": {"_teams_kind": "sprint_internal"}, + "events": [{"type": "created", "actor": "sprint_runner"}], + } + + def capture(_service: TeamService, record: dict[str, Any]) -> None: + persisted.append(json.loads(json.dumps(record))) + + with mock.patch.object( + TeamService, + "_save_request", + autospec=True, + side_effect=capture, + ): + service._save_request(request_record) + service._save_request(second_request) + + seed = build_history_seed() + self.assertEqual(len(persisted), 2) + self.assertEqual(persisted[0]["events"][: len(seed)], list(seed)) + self.assertEqual( + canonical_hash(persisted[0]["events"][: len(seed)]), + _HISTORY_SEED_HASH, + ) + self.assertEqual( + persisted[0]["params"]["_benchmark_history_seed"], + { + "event_count": len(seed), + "sha256": _HISTORY_SEED_HASH, + }, + ) + self.assertEqual( + persisted[0]["events"][-1], + {"type": "created", "actor": "sprint_runner"}, + ) + self.assertNotIn("_benchmark_history_seed", persisted[1]["params"]) + self.assertEqual(len(persisted[1]["events"]), 1) + + def test_persisted_seed_is_idempotent_across_role_services(self) -> None: + shared_state = _BenchmarkHistorySeedState() + first_service = self._service(shared_state) + relay_service = self._service(shared_state) + later_service = self._service(shared_state) + seed = build_history_seed() + request_record = { + "request_id": "planning-request", + "params": { + "_teams_kind": "sprint_internal", + "sprint_phase": "initial", + "initial_phase_step": "milestone_refinement", + }, + "events": [{"type": "created", "actor": "sprint_runner"}], + } + + with mock.patch.object(TeamService, "_save_request", autospec=True): + first_service._save_request(request_record) + first_event_count = len(request_record["events"]) + relay_service._save_request(request_record) + later_request = { + "request_id": "later-planning-request", + "params": { + "_teams_kind": "sprint_internal", + "sprint_phase": "initial", + "initial_phase_step": "milestone_refinement", + }, + "events": [{"type": "created", "actor": "sprint_runner"}], + } + later_service._save_request(later_request) + + self.assertEqual(first_event_count, len(seed) + 1) + self.assertEqual(len(request_record["events"]), first_event_count) + self.assertTrue(relay_service._benchmark_history_seeded) + self.assertEqual(len(later_request["events"]), 1) + self.assertNotIn( + "_benchmark_history_seed", + later_request["params"], + ) + + def test_non_initial_sprint_request_cannot_consume_backfill(self) -> None: + service = self._service() + request_record = { + "request_id": "todo-request", + "params": {"_teams_kind": "sprint_internal"}, + "events": [{"type": "created", "actor": "sprint_runner"}], + } + + with mock.patch.object(TeamService, "_save_request", autospec=True): + service._save_request(request_record) + + self.assertEqual(len(request_record["events"]), 1) + self.assertNotIn( + "_benchmark_history_seed", + request_record["params"], + ) + self.assertFalse(service._benchmark_history_seeded) + + def test_seed_marker_conflicts_fail_before_persistence(self) -> None: + service = self._service() + request_record = { + "request_id": "planning-request", + "params": { + "_teams_kind": "sprint_internal", + "sprint_phase": "initial", + "initial_phase_step": "milestone_refinement", + "_benchmark_history_seed": { + "event_count": 48, + "sha256": "wrong-hash", + }, + }, + "events": list(build_history_seed()), + } + + with mock.patch.object( + TeamService, + "_save_request", + autospec=True, + ) as save_request: + with self.assertRaisesRegex(ValueError, "invalid history seed marker"): + service._save_request(request_record) + + save_request.assert_not_called() + self.assertFalse(service._benchmark_history_seeded) + + def test_failed_persistence_does_not_mark_seed_as_complete(self) -> None: + service = self._service() + request_record = { + "request_id": "planning-request", + "params": { + "_teams_kind": "sprint_internal", + "sprint_phase": "initial", + "initial_phase_step": "milestone_refinement", + }, + "events": [{"type": "created", "actor": "sprint_runner"}], + } + + with mock.patch.object( + TeamService, + "_save_request", + autospec=True, + side_effect=OSError("write failed"), + ): + with self.assertRaisesRegex(OSError, "write failed"): + service._save_request(request_record) + + self.assertFalse(service._benchmark_history_seeded) + + def test_real_planning_builder_persists_seed_before_return(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + scenario = create_scenario_workspace( + root / "arm", + benchmark_id="planning-seed-boundary", + run_id="pair-001-before", + prompt_context_enabled=False, + settings=_settings(), + ) + context = WorkerContext( + benchmark_id="planning-seed-boundary", + arm=ArmPlan( + pair_index=1, + order_index=1, + variant="before", + run_id="pair-001-before", + prompt_context_enabled=False, + ), + workspace_root=scenario.root, + run_output_dir=root / "run", + milestone="Verify deterministic Backfill persistence.", + history_seed=scenario.history_seed, + max_invocations=1, + call_timeout_seconds=1, + run_timeout_seconds=1, + live=True, + ) + service = _BenchmarkTeamService( + scenario.root, + "orchestrator", + enable_discord_client=False, + relay_transport="internal", + allow_external_research=False, + benchmark_context=context, + ) + sprint_state = service._build_manual_sprint_state( + milestone_title="Verify deterministic Backfill persistence.", + trigger="benchmark", + ) + + request_record = service._build_sprint_planning_request_record( + sprint_state, + phase="initial", + iteration=1, + step="milestone_refinement", + ) + persisted = service._load_request(request_record["request_id"]) + relay_service = _BenchmarkTeamService( + scenario.root, + "research", + enable_discord_client=False, + relay_transport="internal", + allow_external_research=False, + benchmark_context=context, + benchmark_history_state=service._benchmark_history_state, + ) + relay_service._save_request(persisted) + persisted = relay_service._load_request(request_record["request_id"]) + + seed_count = len(build_history_seed()) + persisted_prefix = list(persisted["events"][:seed_count]) + self.assertEqual(len(persisted["events"]), seed_count + 1) + self.assertEqual(_history_seed_hash(persisted_prefix), _HISTORY_SEED_HASH) + self.assertEqual( + persisted["params"]["_benchmark_history_seed"], + { + "event_count": seed_count, + "sha256": _HISTORY_SEED_HASH, + }, + ) + self.assertEqual(persisted["events"][-1]["type"], "created") + + class SprintBenchmarkCliTests(unittest.TestCase): def test_parser_exposes_bounded_sprint_ab_defaults(self) -> None: args = build_parser().parse_args( @@ -406,6 +710,56 @@ def test_cli_requires_both_live_opt_ins_before_calling_runner(self) -> None: self.assertEqual(exit_code, 2) runner.assert_not_called() + def test_cli_renders_successful_result_as_json(self) -> None: + result = mock.Mock( + benchmark_id="json-success", + status="comparable", + classification="preliminary_smoke", + output_dir=Path("/tmp/json-success"), + report_json=Path("/tmp/json-success/report.json"), + report_markdown=Path("/tmp/json-success/report.md"), + exit_code=0, + ) + output = io.StringIO() + + with ( + mock.patch.dict( + os.environ, + {"TEAMS_RUNTIME_LIVE_BENCHMARK": "1"}, + clear=True, + ), + mock.patch( + "teams_runtime.benchmarking.runner.run_sprint_ab_benchmark", + return_value=result, + ) as runner, + redirect_stdout(output), + ): + exit_code = cmd_benchmark_sprint_ab( + live=True, + runtime_config="unused.yaml", + repetitions=1, + max_invocations=20, + call_timeout_seconds=300, + run_timeout_seconds=1800, + keep_workspaces="failures", + as_json=True, + ) + + self.assertEqual(exit_code, 0) + self.assertEqual( + json.loads(output.getvalue()), + { + "benchmark_id": "json-success", + "status": "comparable", + "classification": "preliminary_smoke", + "output_dir": "/tmp/json-success", + "report_json": "/tmp/json-success/report.json", + "report_markdown": "/tmp/json-success/report.md", + "exit_code": 0, + }, + ) + runner.assert_called_once() + class SprintBenchmarkReportTests(unittest.TestCase): def test_fake_worker_generates_comparable_private_full_report(self) -> None: @@ -468,6 +822,42 @@ def fake_worker(context: WorkerContext) -> WorkerOutcome: closeout_verified=True, ), telemetry_records=records, + invocation_attempts={ + "schema_version": 1, + "journal_available": True, + "journal_schema_version": 2, + "reconciled": True, + "identity_reconciled": True, + "max_invocations": 20, + "reserved_count": 2, + "entry_count": 2, + "telemetry_record_count": 2, + "completed_count": 2, + "failed_count": 0, + "timeout_count": 0, + "launch_failed_count": 0, + "terminated_count": 0, + "active_count": 0, + "unknown_state_count": 0, + "malformed_entry_count": 0, + "unaccounted_count": 0, + "overaccounted_count": 0, + "rejected_count": 0, + "remaining_budget": 18, + "journal_invocation_ids_sha256": ( + invocation_identity_digest( + record["invocation_id"] + for record in records + ) + ), + "journal_invocation_id_missing_count": 0, + "journal_invocation_id_duplicate_count": 0, + "telemetry_invocation_id_missing_count": 0, + "telemetry_invocation_id_duplicate_count": 0, + "telemetry_invocation_id_unmatched_count": 0, + "journal_invocation_id_unobserved_count": 0, + "prompt": "SENSITIVE_ATTEMPT_SUMMARY_SHOULD_NOT_PERSIST", + }, started_at="2026-07-27T00:00:00+00:00", ended_at="2026-07-27T00:00:02+00:00", wall_duration_ms=1_200 if context.arm.variant == "before" else 800, @@ -527,6 +917,15 @@ def fake_worker(context: WorkerContext) -> WorkerOutcome: self.assertTrue(all(not run.retained_workspace for run in result.runs)) report = json.loads(result.report_json.read_text(encoding="utf-8")) + self.assertEqual(report["schema_version"], 2) + self.assertEqual( + report["runs"][0]["invocation_attempts"]["reserved_count"], + 2, + ) + self.assertNotIn( + "prompt", + report["runs"][0]["invocation_attempts"], + ) pair = report["pairs"][0] self.assertTrue(pair["comparable"]) self.assertEqual(pair["execution_order"], ["before", "after"]) @@ -549,6 +948,8 @@ def fake_worker(context: WorkerContext) -> WorkerOutcome: report["interpretation"]["statistical_significance_claimed"] ) self.assertIn("preliminary", report["interpretation"]["note"].lower()) + markdown = result.report_markdown.read_text(encoding="utf-8") + self.assertIn("| Reserved | Telemetry | Completed |", markdown) for run in result.runs: run_dir = result.output_dir / "runs" / run.arm.run_id @@ -572,6 +973,178 @@ def fake_worker(context: WorkerContext) -> WorkerOutcome: content = artifact.read_text(encoding="utf-8") self.assertNotIn("SENSITIVE_", content, artifact) + def test_untrusted_journal_never_persists_coverage_or_cost(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + source_root = root / "source" + runtime_config = root / "runtime.yaml" + output_root = root / "reports" + _initialize_source_repository(source_root) + _write_runtime_config(runtime_config) + + for case_name in ( + "missing", + "unsupported_schema", + "unreconciled", + "telemetry_overage", + "duplicate_telemetry", + "mismatched_telemetry", + ): + with self.subTest(case_name=case_name): + + def fake_worker(context: WorkerContext) -> WorkerOutcome: + occurrences = ( + (1, 2) + if case_name + in { + "telemetry_overage", + "duplicate_telemetry", + } + else (1,) + ) + records = tuple( + _telemetry_record( + variant=context.arm.variant, + occurrence=occurrence, + estimated_cost=0.002, + ) + for occurrence in occurrences + ) + journal_invocation_ids = [ + str(record["invocation_id"]) + for record in records + ] + if case_name == "duplicate_telemetry": + records[1]["invocation_id"] = records[0][ + "invocation_id" + ] + elif case_name == "mismatched_telemetry": + records[0]["invocation_id"] = ( + "unmatched-telemetry-invocation" + ) + reserved_count = ( + 2 + if case_name == "duplicate_telemetry" + else 1 + ) + attempts: dict[str, Any] = { + "schema_version": 1, + "journal_available": True, + "journal_schema_version": 2, + "reconciled": True, + "identity_reconciled": True, + "max_invocations": 20, + "reserved_count": reserved_count, + "entry_count": reserved_count, + "telemetry_record_count": len(records), + "unobserved_attempt_count": 0, + "telemetry_overage_count": 0, + "completed_count": reserved_count, + "failed_count": 0, + "timeout_count": 0, + "launch_failed_count": 0, + "terminated_count": 0, + "active_count": 0, + "unknown_state_count": 0, + "malformed_entry_count": 0, + "unaccounted_count": 0, + "overaccounted_count": 0, + "rejected_count": 0, + "remaining_budget": 20 - reserved_count, + "journal_invocation_ids_sha256": ( + invocation_identity_digest( + journal_invocation_ids[ + :reserved_count + ] + ) + ), + "journal_invocation_id_missing_count": 0, + "journal_invocation_id_duplicate_count": 0, + "telemetry_invocation_id_missing_count": 0, + "telemetry_invocation_id_duplicate_count": 0, + "telemetry_invocation_id_unmatched_count": 0, + "journal_invocation_id_unobserved_count": 0, + } + if case_name == "missing": + attempts = {} + elif case_name == "unsupported_schema": + attempts["journal_schema_version"] = 99 + elif case_name == "unreconciled": + attempts["reconciled"] = False + return WorkerOutcome( + status="completed", + telemetry_records=records, + invocation_attempts=attempts, + ) + + benchmark_id = f"untrusted-journal-{case_name}" + result = run_sprint_ab_benchmark( + BenchmarkOptions( + source_root=source_root, + runtime_config_path=runtime_config, + output_dir=output_root, + repetitions=1, + max_invocations=20, + call_timeout_seconds=300, + run_timeout_seconds=1_800, + keep_workspaces="none", + live=False, + benchmark_id=benchmark_id, + ), + worker=fake_worker, + ) + report = json.loads( + result.report_json.read_text(encoding="utf-8") + ) + report_runs = { + str(run["run_id"]): run + for run in report["runs"] + } + + for run in result.runs: + persisted_metrics = json.loads( + ( + result.output_dir + / "runs" + / run.arm.run_id + / "metrics.json" + ).read_text(encoding="utf-8") + ) + for candidate in ( + run.metrics, + persisted_metrics, + report_runs[run.arm.run_id]["metrics"], + ): + totals = candidate["totals"] + self.assertEqual( + totals["coverage_basis"], + "unavailable_untrusted_call_journal", + ) + for field_name in ( + "expected_invocation_count", + "token_coverage_percent", + "tool_call_coverage_percent", + "pricing_coverage_percent", + "estimated_cost_usd", + ): + self.assertIsNone(totals[field_name]) + self.assertTrue( + all( + group["estimated_cost_usd"] is None + for group in candidate["groups"] + ) + ) + self.assertNotIn( + "telemetry_coverage_percent", + run.invocation_attempts, + ) + self.assertNotIn( + "telemetry_coverage_percent", + report_runs[run.arm.run_id][ + "invocation_attempts" + ], + ) + def test_missing_usage_is_inconclusive_but_missing_price_is_explicitly_unpriced(self) -> None: before_records = ( _telemetry_record(variant="before", estimated_cost=0.001), @@ -653,6 +1226,226 @@ def test_reducer_reports_partial_coverage_without_inventing_cost_or_usage(self) {None, 0.002}, ) + def test_reserved_attempt_without_telemetry_keeps_cost_and_usage_incomplete( + self, + ) -> None: + observed = _telemetry_record( + variant="after", + estimated_cost=0.002, + ) + + metrics = reduce_telemetry( + (observed,), + expected_invocation_count=2, + ) + + self.assertEqual(metrics["totals"]["invocation_count"], 1) + self.assertEqual(metrics["totals"]["expected_invocation_count"], 2) + self.assertEqual(metrics["totals"]["unobserved_invocation_count"], 1) + self.assertEqual(metrics["totals"]["token_coverage_percent"], 50.0) + self.assertEqual(metrics["totals"]["pricing_coverage_percent"], 50.0) + self.assertIsNone(metrics["totals"]["estimated_cost_usd"]) + self.assertTrue( + all( + group["estimated_cost_usd"] is None + for group in metrics["groups"] + ) + ) + self.assertEqual( + metrics["compaction"]["unobserved_invocation_count"], + 1, + ) + + def test_missing_call_journal_keeps_coverage_and_cost_unknown(self) -> None: + observed = _telemetry_record( + variant="after", + estimated_cost=0.002, + ) + + metrics = reduce_telemetry( + (observed,), + coverage_available=False, + ) + + totals = metrics["totals"] + self.assertEqual( + totals["coverage_basis"], + "unavailable_untrusted_call_journal", + ) + self.assertIsNone(totals["expected_invocation_count"]) + self.assertIsNone(totals["unobserved_invocation_count"]) + self.assertIsNone(totals["token_coverage_percent"]) + self.assertIsNone(totals["tool_call_coverage_percent"]) + self.assertIsNone(totals["pricing_coverage_percent"]) + self.assertIsNone(totals["estimated_cost_usd"]) + self.assertIsNone( + metrics["compaction"]["unobserved_invocation_count"] + ) + self.assertTrue( + all( + group["estimated_cost_usd"] is None + for group in metrics["groups"] + ) + ) + + def test_coverage_requires_a_supported_reconciled_call_journal(self) -> None: + trusted = { + "schema_version": 1, + "journal_available": True, + "journal_schema_version": 2, + "reconciled": True, + "identity_reconciled": True, + "max_invocations": 20, + "reserved_count": 2, + "entry_count": 2, + "telemetry_record_count": 1, + "unobserved_attempt_count": 1, + "telemetry_overage_count": 0, + "completed_count": 1, + "failed_count": 0, + "timeout_count": 0, + "launch_failed_count": 0, + "terminated_count": 1, + "active_count": 0, + "unknown_state_count": 0, + "malformed_entry_count": 0, + "unaccounted_count": 0, + "overaccounted_count": 0, + "rejected_count": 0, + "remaining_budget": 18, + "journal_invocation_ids_sha256": ( + invocation_identity_digest(("invocation-1", "invocation-2")) + ), + "journal_invocation_id_missing_count": 0, + "journal_invocation_id_duplicate_count": 0, + "telemetry_invocation_id_missing_count": 0, + "telemetry_invocation_id_duplicate_count": 0, + "telemetry_invocation_id_unmatched_count": 0, + "journal_invocation_id_unobserved_count": 1, + } + + self.assertTrue( + _journal_coverage_available( + trusted, + expected_max_invocations=20, + ) + ) + invalid_cases = { + "missing": {}, + "unsupported_summary_schema": { + **trusted, + "schema_version": 2, + }, + "unsupported_journal_schema": { + **trusted, + "journal_schema_version": 3, + }, + "unreconciled": { + **trusted, + "reconciled": False, + }, + "identity_unreconciled": { + **trusted, + "identity_reconciled": False, + }, + "missing_count": { + key: value + for key, value in trusted.items() + if key != "remaining_budget" + }, + "maximum_mismatch": { + **trusted, + "max_invocations": 19, + "remaining_budget": 17, + }, + "state_count_mismatch": { + **trusted, + "completed_count": 0, + }, + "telemetry_overage": { + **trusted, + "telemetry_record_count": 3, + "unobserved_attempt_count": 0, + "telemetry_overage_count": 1, + }, + "duplicate_telemetry_identity": { + **trusted, + "telemetry_invocation_id_duplicate_count": 1, + }, + "unmatched_telemetry_identity": { + **trusted, + "telemetry_invocation_id_unmatched_count": 1, + }, + "unobserved_identity_mismatch": { + **trusted, + "journal_invocation_id_unobserved_count": 0, + }, + } + for label, attempts in invalid_cases.items(): + with self.subTest(label=label): + self.assertFalse( + _journal_coverage_available( + attempts, + expected_max_invocations=20, + ) + ) + + def test_unobserved_terminated_attempt_makes_pair_inconclusive(self) -> None: + before = _arm_result( + "before", + (_telemetry_record(variant="before"),), + ) + after = replace( + _arm_result( + "after", + (_telemetry_record(variant="after"),), + ), + invocation_attempts={ + "journal_available": True, + "journal_schema_version": 2, + "reconciled": True, + "reserved_count": 2, + "entry_count": 2, + "telemetry_record_count": 1, + "unobserved_attempt_count": 1, + "completed_count": 1, + "terminated_count": 1, + "active_count": 0, + "unknown_state_count": 0, + "malformed_entry_count": 0, + "unaccounted_count": 0, + "overaccounted_count": 0, + }, + ) + + report = _report_for_runs((before, after)) + + self.assertEqual(report["status"], "inconclusive") + reasons = report["pairs"][0]["inconclusive_reasons"] + self.assertIn("after_unobserved_attempts", reasons) + self.assertIn("after_terminated_attempts", reasons) + + def test_missing_call_journal_makes_pair_inconclusive(self) -> None: + before = _arm_result( + "before", + (_telemetry_record(variant="before"),), + ) + after = replace( + _arm_result( + "after", + (_telemetry_record(variant="after"),), + ), + invocation_attempts={}, + ) + + report = _report_for_runs((before, after)) + + self.assertEqual(report["status"], "inconclusive") + self.assertIn( + "after_call_journal_missing", + report["pairs"][0]["inconclusive_reasons"], + ) + def test_native_usage_requires_complete_consistent_provider_counts(self) -> None: complete = _telemetry_record(variant="before") missing_output = _telemetry_record(variant="before") From eca4519c34c293e8cd6af3d86958482acf86acf3 Mon Sep 17 00:00:00 2001 From: rica-v3 Date: Mon, 27 Jul 2026 23:53:41 +0900 Subject: [PATCH 6/7] performance_benchmarking.md: document measurement safeguards --- docs/performance_benchmarking.md | 51 ++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/docs/performance_benchmarking.md b/docs/performance_benchmarking.md index 3a0e939..61a9b11 100644 --- a/docs/performance_benchmarking.md +++ b/docs/performance_benchmarking.md @@ -66,14 +66,18 @@ An arm passes its behavior and workflow gates only when all of the following are - the final Git worktree is clean - the isolated repository still has no Git remotes - every persisted invocation has native token usage +- the call journal is present, uses a supported schema, and reconciles every + reservation to exactly one terminal attempt - the After arm records at least one real compacted prompt projection - the Before and After non-feature configuration fingerprints match ## Backfill In this benchmark, **Backfill** means adding a deterministic history prefix to the -request that exercises the sprint TODO. It does not mean importing production -telemetry, replaying customer data, or modifying an existing workspace. +canonical initial sprint-planning request before it is relayed to research or +planner. The same canonical request, including the prefix, is used by later routing +steps. It does not mean importing production telemetry, replaying customer data, or +modifying an existing workspace. The benchmark generates 48 content-safe historical events. They contain neutral checkpoints and one evidence checkpoint for each workflow role: @@ -102,9 +106,11 @@ checkpoints and one evidence checkpoint for each workflow role: ``` The full 48-event sequence and its canonical SHA-256 hash are identical in both arms. -It is prepended only to the internal execution request, before that request's normal -events. This produces a realistic long-history prompt without changing planning -inputs or introducing facts that could change the desired implementation. +It is prepended exactly once, before the initial planning request's normal `created` +and `delegated` events. The benchmark persists and hash-verifies the prefix before +the first provider call. This produces a realistic long-history prompt from the +beginning of the sprint without introducing facts that could change the desired +implementation. Backfill serves three purposes: @@ -138,8 +144,9 @@ The selection policy is 8. Add a projection notice with total, included, and omitted counts plus the path to the complete canonical request. -For example, assume the 48-event backfill is followed by one normal request-created -event. Before compaction, the prompt input contains: +For example, the first relayed planning request normally has the 48-event Backfill +followed by its `created` and `delegated` events. Before compaction, the prompt input +contains: ```json { @@ -147,20 +154,21 @@ event. Before compaction, the prompt input contains: "events": [ {"type": "role_report", "payload": {"role": "research", "status": "completed"}}, "... 47 additional deterministic historical events ...", - {"type": "created", "actor": "orchestrator"} + {"type": "created", "actor": "sprint_runner"}, + {"type": "delegated", "actor": "orchestrator"} ] } ``` -The Before arm embeds all 49 events. The After arm's projection contains the latest +The Before arm embeds all 50 events. The After arm's projection contains the latest 8 events and the most recent older evidence for up to 8 missing roles: ```json { "compacted": true, - "total_events": 49, + "total_events": 50, "included_events": 16, - "omitted_events": 33, + "omitted_events": 34, "recent_events": 8, "max_events": 16, "selection": "recent_tail_plus_latest_role_evidence", @@ -169,7 +177,7 @@ The Before arm embeds all 49 events. The After arm's projection contains the lat ``` The projected `events` array contains 16 complete event objects in chronological -order. The other 33 events still exist in the canonical request. The prompt tells the +order. The other 34 events still exist in the canonical request. The prompt tells the role to open that file only when a decision requires evidence missing from the projection. @@ -241,6 +249,9 @@ The live worker is deliberately narrower than normal runtime execution: - writable paths must resolve inside the isolated arm root - a shared atomic journal reserves a call before launch, so concurrent roles cannot exceed the arm's physical-call budget +- a benchmark-only launcher waits on a private pipe and executes the provider only + after its PID and process group are durably journaled; parent death closes the + pipe and exits the launcher without starting the provider - provider processes run in dedicated process groups and are terminated on call timeout - an arm timeout terminates active provider groups before the worker is stopped @@ -277,7 +288,9 @@ Artifacts are written under: The execution report includes: -- physical invocation and logical-call counts +- journal-reserved, telemetry-observed, completed, failed, timed-out, + launch-failed, parent-terminated, active, and rejected attempt counts +- physical telemetry invocation and logical-call counts - primary, contract-repair, sandbox-retry, failed, and completed counts - tool-call count and coverage - provider and end-to-end wall duration, including p50 and p95 provider latency @@ -291,6 +304,18 @@ The execution report includes: - matched primary calls keyed by role, purpose, workflow step, and occurrence - full-sprint Before/After deltas and reduction percentages +Provider usage is reported only for telemetry-observed calls. Native-token, +tool-call, and pricing coverage use journal-reserved attempts as their denominator. +If the hard arm deadline terminates a provider before telemetry can be finalized, +the call journal records a terminal `terminated` attempt while its token usage +remains unmeasured. Reports show both counts, leave aggregate and per-group costs +unpriced, and never infer tokens for that attempt. If the journal is absent, +unsupported, unreconciled, incomplete, or has more telemetry records than +reservations, all coverage percentages and cost totals remain unknown. Coverage +also requires unique, nonempty telemetry invocation IDs that match a subset of the +journaled invocation IDs. Reports persist only bounded mismatch counts and a +SHA-256 identity digest, not the journal's raw identity list. + Reports are content-safe: they do not contain prompts, model responses, tool output, raw errors, raw session IDs, credentials, or environment values. From 726fc754a93b8cff0435fbee11cb99b6cfbacd87 Mon Sep 17 00:00:00 2001 From: rica-v3 Date: Tue, 11 Aug 2026 23:50:03 +0900 Subject: [PATCH 7/7] codex_runner.py: isolate benchmark provider environment --- benchmarking/worker.py | 12 ++++++++++ docs/performance_benchmarking.md | 10 ++++++++ runtime/codex_runner.py | 6 +++-- runtime/execution_policy.py | 9 +++++-- tests/test_execution_policy.py | 41 ++++++++++++++++++++++++++++++++ tests/test_sprint_benchmark.py | 28 ++++++++++++++++++++++ 6 files changed, 102 insertions(+), 4 deletions(-) diff --git a/benchmarking/worker.py b/benchmarking/worker.py index 8a9aea2..17e0bcf 100644 --- a/benchmarking/worker.py +++ b/benchmarking/worker.py @@ -64,6 +64,10 @@ "launch_failed", "terminated", ) +_PROVIDER_AUTH_ENVIRONMENT_KEYS = ( + "CODEX_API_KEY", + "OPENAI_API_KEY", +) _CHILD_ENVIRONMENT_KEYS = ( "CODEX_API_KEY", "CODEX_HOME", @@ -323,6 +327,14 @@ def _build_execution_policy( *, budget: InvocationBudget, ) -> ModelExecutionPolicy: + if not any( + str(os.environ.get(name) or "").strip() + for name in _PROVIDER_AUTH_ENVIRONMENT_KEYS + ): + raise ModelExecutionPolicyViolation( + "Live sprint benchmark requires provider-only authentication through " + "CODEX_API_KEY or OPENAI_API_KEY; operator Codex home credentials are isolated." + ) return ModelExecutionPolicy.for_benchmark( allowed_workspace_root=context.workspace_root, invocation_budget=budget, diff --git a/docs/performance_benchmarking.md b/docs/performance_benchmarking.md index 61a9b11..94dabb3 100644 --- a/docs/performance_benchmarking.md +++ b/docs/performance_benchmarking.md @@ -204,6 +204,14 @@ python -m teams_runtime benchmark sprint-ab \ Live calls require both the environment variable and `--live`. Omitting either is a preflight failure and makes no model call. +The provider does not inherit the operator's `HOME`, `CODEX_HOME`, or temporary +directories. It uses private paths under the arm's ignored `.teams_runtime` state. +Provide authentication through a supported provider-only environment variable such as +`CODEX_API_KEY` or `OPENAI_API_KEY`; stored credentials from the operator's normal +Codex home are intentionally unavailable. Missing provider-only authentication fails +preflight before a call is reserved. These authentication variables are not injected +into model tool shells. Use a rate card matching the selected credential and backend. + Options: | Option | Default | Meaning | @@ -242,6 +250,8 @@ The live worker is deliberately narrower than normal runtime execution: - dangerous sandbox-bypass requests and automatic bypass retries are rejected - MCP servers, web search, plugins, hooks, computer use, and multi-agent features are disabled for the provider process +- the provider's `HOME`, `CODEX_HOME`, and temporary directories resolve inside private + ignored benchmark state rather than the operator's home or system temporary directory - the provider receives only the authentication and transport variables needed by the outer Codex CLI - tool shells inherit no outer environment and receive an explicit non-secret diff --git a/runtime/codex_runner.py b/runtime/codex_runner.py index 62a4645..6f902cb 100644 --- a/runtime/codex_runner.py +++ b/runtime/codex_runner.py @@ -354,7 +354,9 @@ def _provider_environment(self) -> dict[str, str]: for key in BENCHMARK_PROVIDER_ENVIRONMENT_KEYS if (value := os.environ.get(key)) is not None } - environment["HOME"] = str(Path.home()) + # The explicit policy must override outer HOME, CODEX_HOME, temp, Git, + # and locale values while provider-only auth variables remain available. + environment.update(self.execution_policy.shell_environment) environment.setdefault("PATH", os.defpath) environment["NO_COLOR"] = "1" return environment @@ -652,7 +654,7 @@ def run( "Benchmark execution forbids sandbox bypass requests." ) if self.execution_policy.benchmark_mode: - for directory_key in ("HOME", "TMPDIR", "TMP", "TEMP"): + for directory_key in ("HOME", "CODEX_HOME", "TMPDIR", "TMP", "TEMP"): directory_value = self.execution_policy.shell_environment.get(directory_key) if not directory_value: continue diff --git a/runtime/execution_policy.py b/runtime/execution_policy.py index cdbe933..036ec4b 100644 --- a/runtime/execution_policy.py +++ b/runtime/execution_policy.py @@ -390,13 +390,18 @@ def for_benchmark( shell_environment: Mapping[str, str] | None = None, ) -> "ModelExecutionPolicy": allowed_root = Path(allowed_workspace_root).expanduser().resolve() + provider_state_root = allowed_root / ".teams_runtime" / "benchmark_provider" + provider_tmp = provider_state_root / "tmp" environment = { + "CODEX_HOME": str(provider_state_root / "codex_home"), "GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_NOSYSTEM": "1", "GIT_TERMINAL_PROMPT": "0", - "HOME": str(allowed_root), + "HOME": str(provider_state_root / "home"), "PATH": os.environ.get("PATH") or os.defpath, - "TMPDIR": str(allowed_root / ".tmp"), + "TEMP": str(provider_tmp), + "TMP": str(provider_tmp), + "TMPDIR": str(provider_tmp), } environment.update(shell_environment or {}) return cls( diff --git a/tests/test_execution_policy.py b/tests/test_execution_policy.py index ddabf16..448fdfb 100644 --- a/tests/test_execution_policy.py +++ b/tests/test_execution_policy.py @@ -81,10 +81,15 @@ def test_provider_environment_excludes_github_and_discord_secrets(self) -> None: execution_policy=policy, ) source_environment = { + "CODEX_HOME": "/operator/codex-home", "OPENAI_API_KEY": "provider-secret", "GH_TOKEN": "github-secret", "DISCORD_TOKEN": "discord-secret", + "HOME": "/operator/home", "PATH": os.defpath, + "TEMP": "/operator/temp", + "TMP": "/operator/tmp", + "TMPDIR": "/operator/tmpdir", } with mock.patch.dict(os.environ, source_environment, clear=True): @@ -93,6 +98,42 @@ def test_provider_environment_excludes_github_and_discord_secrets(self) -> None: self.assertEqual(environment["OPENAI_API_KEY"], "provider-secret") self.assertNotIn("GH_TOKEN", environment) self.assertNotIn("DISCORD_TOKEN", environment) + provider_state = root / ".teams_runtime" / "benchmark_provider" + provider_tmp = str(provider_state / "tmp") + self.assertEqual(environment["HOME"], str(provider_state / "home")) + self.assertEqual( + environment["CODEX_HOME"], + str(provider_state / "codex_home"), + ) + self.assertEqual(environment["TMPDIR"], provider_tmp) + self.assertEqual(environment["TMP"], provider_tmp) + self.assertEqual(environment["TEMP"], provider_tmp) + self.assertEqual(environment["GIT_CONFIG_GLOBAL"], os.devnull) + self.assertEqual(environment["GIT_CONFIG_NOSYSTEM"], "1") + self.assertEqual(environment["PYTHONPATH"], str(root)) + + def test_benchmark_rejects_provider_state_directory_escape(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory).resolve() + workspace = root / "role" + workspace.mkdir() + outside = root.parent / "outside-codex-home" + budget = InvocationBudget(1) + policy = ModelExecutionPolicy.for_benchmark( + allowed_workspace_root=root, + invocation_budget=budget, + call_timeout_seconds=1, + shell_environment={"CODEX_HOME": str(outside)}, + ) + runner = CodexRunner( + RoleRuntimeConfig(model="gpt-benchmark"), + execution_policy=policy, + ) + + with self.assertRaises(ModelExecutionPolicyViolation): + runner.run(workspace, "prompt", None) + + self.assertEqual(budget.reserved_count, 0) def test_benchmark_rejects_bypass_and_gemini_before_launch(self) -> None: with tempfile.TemporaryDirectory() as temporary_directory: diff --git a/tests/test_sprint_benchmark.py b/tests/test_sprint_benchmark.py index 0130eee..e3fa4a0 100644 --- a/tests/test_sprint_benchmark.py +++ b/tests/test_sprint_benchmark.py @@ -48,6 +48,7 @@ from teams_runtime.benchmarking.worker import ( _BenchmarkHistorySeedState, _BenchmarkTeamService, + _build_execution_policy, _history_seed_hash, ) from teams_runtime.cli import build_parser, cmd_benchmark_sprint_ab @@ -1627,6 +1628,33 @@ def test_sanitizer_allowlists_nested_rate_card_fields(self) -> None: class SprintBenchmarkExecutionSafetyTests(unittest.TestCase): + def test_live_policy_requires_provider_only_auth_before_reserving_call(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory).resolve() + context = mock.Mock( + workspace_root=root, + call_timeout_seconds=30, + ) + budget = InvocationBudget(1) + + with mock.patch.dict(os.environ, {"PATH": os.defpath}, clear=True): + with self.assertRaisesRegex( + ModelExecutionPolicyViolation, + "CODEX_API_KEY or OPENAI_API_KEY", + ): + _build_execution_policy(context, budget=budget) + + self.assertEqual(budget.reserved_count, 0) + + with mock.patch.dict( + os.environ, + {"CODEX_API_KEY": "provider-only-secret", "PATH": os.defpath}, + clear=True, + ): + policy = _build_execution_policy(context, budget=budget) + + self.assertNotIn("CODEX_API_KEY", policy.shell_environment) + def test_invocation_budget_rejects_the_twenty_first_call_and_journals_no_content(self) -> None: class InvocationContext: invocation_id = "safe-invocation-id"