Bind benchmark resume to runtime and source identity

.github/workflows/ci.yml

@@ -4,7 +4,8 @@
 #   python-syntax      — byte-compiles every Python tree (catches syntax/import-
 #                        time breakage cheaply, no third-party deps needed).
 #   root-pytest        — the suite-level tests/ directory (registry, scoring,
-#                        normalization, acceptance gate). Needs only pytest;
+#                        normalization, acceptance gate) plus source-bound
+#                        orchestrator resume/cohort contracts. Needs only pytest;
 #                        dataset-dependent cases skip gracefully when their
 #                        artifacts are absent.
 #   bun-workspace      — root bun install resolving @elizaos/* from the eliza/
@@ -71,7 +72,11 @@ jobs:
       - name: Run suite-level tests
         env:
           PYTHONPATH: ${{ github.workspace }}
-        run: python -m pytest benchmarks/tests -q
+        run: >-
+          python -m pytest benchmarks/tests
+          benchmarks/suites/orchestrator/tests/test_execution_identity.py
+          benchmarks/suites/orchestrator/tests/test_cohort.py
+          benchmarks/suites/orchestrator/tests/test_cohort_provider_forwarder.py -q
 
   bun-workspace:
     name: bun workspace (plugin + TS suites)
@@ -85,6 +90,8 @@ jobs:
           bun run --cwd plugin-benchmarks test
           bun run --cwd plugin-benchmarks typecheck
           bun run --cwd plugin-benchmarks build
+      - name: Manifest exporter CLI smoke
+        run: bun run lifeops-bench:manifest --help
       - name: framework typecheck
         run: bun run --cwd framework/typescript typecheck
       - name: suite tests (keyless vitest/bun-test lanes)

.github/workflows/lifeops-bench-python.yml

@@ -197,14 +197,12 @@ jobs:
         run: |
           set -e
           mkdir -p lifeops_bench_results
-          # The `wrong` reference oracle runs entirely in-process and
-          # exercises the runner / scorer / executor pipeline without any
-          # LLM. PerfectAgent's CLI entry point currently requires a
-          # per-scenario constructor (see __main__._build_agent_fn) and is
-          # covered instead by the conformance test in unit-tests above.
+          # A deterministic negative control exercises five static domains.
+          # Its results are explicitly non-publishable conformance evidence.
           uv run python -m eliza_lifeops_bench \
             --agent wrong \
-            --mode static \
+            --suite smoke \
+            --offline-conformance \
             --seeds 1 \
             --concurrency 4 \
             --max-cost-usd 0.01 \

suites/lifeops-bench/tests/test_manifest_export.py

@@ -8,25 +8,14 @@
 from pathlib import Path
 
 PACKAGE_ROOT = Path(__file__).resolve().parents[1]
-REPO_ROOT = PACKAGE_ROOT.parents[1]
 MANIFEST_PATH = PACKAGE_ROOT / "manifests" / "actions.manifest.json"
 SUMMARY_PATH = PACKAGE_ROOT / "manifests" / "actions.summary.md"
-ROOT_PACKAGE_JSON = REPO_ROOT / "package.json"
 
 
 def _manifest() -> dict[str, object]:
     return json.loads(MANIFEST_PATH.read_text(encoding="utf-8"))
 
 
-def test_root_manifest_regeneration_script_is_registered() -> None:
-    root_package = json.loads(ROOT_PACKAGE_JSON.read_text(encoding="utf-8"))
-    command = root_package["scripts"]["lifeops-bench:manifest"]
-    assert "suites/lifeops-bench/scripts/export-action-manifest.ts" in command
-    assert "--conditions=eliza-source" in command
-    assert "--conditions=development" in command
-    assert "--import tsx" in command
-
-
 def test_manifest_has_in_tree_generator_metadata() -> None:
     manifest = _manifest()
     assert manifest["schemaVersion"] == 1

suites/orchestrator/README.md

@@ -66,7 +66,7 @@ python3 -m benchmarks.orchestrator run \
 
 Idempotent behavior:
 
-- Existing successful signatures are skipped automatically.
+- Existing successful signatures are skipped only when the request, repository revisions, runtime and benchmark/harness source, corpus inputs, and root lockfiles match. Changed inputs require a new run. Subscription checkpoint namespaces use the same source-sensitive execution contract; old namespaces are not reused. Cross-harness comparison grouping remains separate.
 - `--rerun-failed` reruns only signatures whose latest run failed.
 - `--force` always creates a fresh run.
 

suites/orchestrator/cohort.py

@@ -9,7 +9,6 @@
 
 from __future__ import annotations
 
-import hashlib
 import json
 import shutil
 import threading
@@ -41,6 +40,7 @@
     repair_nonzero_returncode_statuses,
     resume_run_group,
 )
+from .execution_identity import PhaseExecutionIdentity, build_phase_execution_identity
 from .locking import campaign_execution_lock
 from .provider_forwarder import (
     ProviderForwarderProcess,
@@ -88,49 +88,6 @@
 DEFAULT_STORAGE_MIN_FREE_BYTES = 8 * 1024**3
 DEFAULT_STORAGE_EXPECTED_HEADROOM_BYTES = 4 * 1024**3
 DEFAULT_STORAGE_CHECK_INTERVAL_SECONDS = 30.0
-_STORAGE_CONTROL_KEYS = frozenset(
-    {
-        "campaign_storage_min_free_bytes",
-        "campaign_storage_expected_headroom_bytes",
-        "campaign_storage_check_interval_s",
-    }
-)
-_NAMESPACE_RUNTIME_KEYS = _STORAGE_CONTROL_KEYS | frozenset(
-    {
-        "_replace_adapter_defaults",
-        "campaign_silent_timeout_s",
-        "claude_subscription_gateway_url",
-        "eliza_bench_http_timeout_s",
-        "hermes_timeout_s",
-        "hl_bench_command_timeout_s",
-        "openclaw_timeout_s",
-        "timeout_s",
-    }
-)
-_FINGERPRINT_EXCLUDED_PARTS = frozenset(
-    {
-        ".cache",
-        ".git",
-        ".mypy_cache",
-        ".pytest_cache",
-        ".ruff_cache",
-        ".turbo",
-        ".venv",
-        "__pycache__",
-        "artifacts",
-        "benchmark_results",
-        "build",
-        "coverage",
-        "dist",
-        "node_modules",
-        "outputs",
-        "results",
-        "runs",
-        "venv",
-    }
-)
-
-
 class BenchmarkCohortStatus(StrEnum):
     SUCCEEDED = "succeeded"
     FAILED = "failed"
@@ -139,13 +96,6 @@ class BenchmarkCohortStatus(StrEnum):
     UNSUPPORTED = "unsupported"
 
 
-@dataclass(frozen=True)
-class PhaseExecutionIdentity:
-    namespace: str
-    contract: Mapping[str, object]
-    checkpoint_relpath: str
-
-
 @dataclass(frozen=True)
 class StoragePreflight:
     checked_at: str
@@ -213,124 +163,6 @@ def _canonical_json(value: object) -> str:
     return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
 
 
-def build_phase_execution_identity(
-    *,
-    workspace_root: Path,
-    adapter: BenchmarkAdapter,
-    request: RunRequest,
-    harnesses: tuple[str, ...],
-    repo_meta: Mapping[str, str | None],
-) -> PhaseExecutionIdentity:
-    """Build the PID/run-group-independent replay identity for one phase."""
-
-    extra = dict(request.extra_config)
-    dataset_config = {
-        key: value
-        for key, value in extra.items()
-        if key not in _NAMESPACE_RUNTIME_KEYS
-        and key
-        not in {
-            "campaign_corpus_sha256",
-            "campaign_phase",
-            "campaign_profile",
-            "reasoning_effort",
-        }
-    }
-    source_fingerprint = _relevant_source_fingerprint(
-        workspace_root=workspace_root,
-        adapter=adapter,
-    )
-    explicit_corpus = extra.get("campaign_corpus_sha256")
-    if explicit_corpus is not None and (
-        not isinstance(explicit_corpus, str) or not explicit_corpus.strip()
-    ):
-        raise ValueError("campaign_corpus_sha256 must be a non-empty string")
-    corpus_sha256 = (
-        explicit_corpus.strip()
-        if isinstance(explicit_corpus, str)
-        else hashlib.sha256(
-            _canonical_json(
-                {
-                    "adapter_directory": adapter.directory,
-                    "benchmarks_commit": repo_meta.get("benchmarks_commit"),
-                    "dataset_config": dataset_config,
-                    "source_fingerprint_sha256": source_fingerprint,
-                }
-            ).encode("utf-8")
-        ).hexdigest()
-    )
-    contract: dict[str, object] = {
-        "schema_version": 1,
-        "campaign_profile": str(extra.get("campaign_profile") or "ad-hoc"),
-        "benchmark_id": adapter.id,
-        "benchmark_directory": adapter.directory,
-        "phase": str(extra.get("campaign_phase") or "single"),
-        "dataset_config": dataset_config,
-        "corpus_sha256": corpus_sha256,
-        "source_fingerprint_sha256": source_fingerprint,
-        "model": request.model.strip(),
-        "provider": request.provider.strip().lower(),
-        "reasoning_effort": str(extra.get("reasoning_effort") or "").strip().lower(),
-        "harnesses": list(harnesses),
-    }
-    digest = hashlib.sha256(_canonical_json(contract).encode("utf-8")).hexdigest()
-    namespace = f"benchmark-phase-v1-{digest}"
-    return PhaseExecutionIdentity(
-        namespace=namespace,
-        contract=contract,
-        checkpoint_relpath=(
-            f".subscription-checkpoints/{namespace}/responses.jsonl"
-        ),
-    )
-
-
-def _relevant_source_fingerprint(
-    *,
-    workspace_root: Path,
-    adapter: BenchmarkAdapter,
-) -> str:
-    """Hash dirty source/corpus inputs that a Git HEAD cannot represent."""
-
-    benchmarks_root = workspace_root / "suites"
-    candidate_roots = (
-        benchmarks_root / adapter.directory,
-        benchmarks_root / "orchestrator",
-        benchmarks_root / "claude-subscription-gateway",
-        benchmarks_root.parent / "harnesses" / "eliza",
-        benchmarks_root.parent / "harnesses" / "hermes",
-        benchmarks_root.parent / "harnesses" / "openclaw",
-        workspace_root.parent / "package.json",
-        workspace_root.parent / "bun.lock",
-        workspace_root.parent / "bun.lockb",
-    )
-    files: set[Path] = set()
-    for root in candidate_roots:
-        if root.is_file():
-            files.add(root)
-            continue
-        if not root.is_dir():
-            continue
-        for path in root.rglob("*"):
-            try:
-                relative_parts = path.relative_to(root).parts
-            except ValueError:
-                continue
-            if any(part in _FINGERPRINT_EXCLUDED_PARTS for part in relative_parts):
-                continue
-            if path.is_file() and not path.is_symlink():
-                files.add(path)
-    digest = hashlib.sha256()
-    fingerprint_base = workspace_root.parent
-    for path in sorted(files, key=lambda value: value.as_posix()):
-        relative = path.relative_to(fingerprint_base).as_posix().encode("utf-8")
-        digest.update(len(relative).to_bytes(8, "big"))
-        digest.update(relative)
-        with path.open("rb") as handle:
-            while chunk := handle.read(1024 * 1024):
-                digest.update(chunk)
-    return digest.hexdigest()
-
-
 def _storage_integer(extra: Mapping[str, object], key: str, default: int) -> int:
     value = extra.get(key, default)
     if isinstance(value, bool) or not isinstance(value, int) or value < 0:
@@ -777,6 +609,8 @@ def _find_reusable_subscription_cohort(
             harness: _signature_for(
                 adapter,
                 _effective_request(adapter, replace(request, agent=harness)),
+                workspace_root=workspace_root,
+                repo_meta=_repo_meta(workspace_root),
             )
             for harness in harnesses
         }
@@ -875,7 +709,7 @@ def _load_resumable_execution(
 ) -> dict[str, object] | None:
     db_path = (
         workspace_root
-
+        / "suites"
         / "benchmark_results"
         / "orchestrator.sqlite"
     )