Add full-sprint performance and cost benchmark pipeline

adapters/cli/commands.py

@@ -209,6 +209,84 @@ def build_parser(
             help=workspace_root_help_text,
         )
 
+    benchmark_parser = subparsers.add_parser(
+        "benchmark",
+        help="Run an explicitly opted-in performance benchmark.",
+    )
+    benchmark_subparsers = benchmark_parser.add_subparsers(
+        dest="benchmark_command",
+        required=True,
+    )
+    sprint_ab_parser = benchmark_subparsers.add_parser(
+        "sprint-ab",
+        help="Compare full sprints with prompt event compaction disabled and enabled.",
+    )
+    sprint_ab_parser.add_argument(
+        "--live",
+        action="store_true",
+        help="Permit live provider calls; also requires TEAMS_RUNTIME_LIVE_BENCHMARK=1.",
+    )
+    sprint_ab_parser.add_argument(
+        "--runtime-config",
+        required=True,
+        help="Path to the deployed team_runtime.yaml or its workspace directory.",
+    )
+    sprint_ab_parser.add_argument(
+        "--repetitions",
+        type=int,
+        default=1,
+        help="Number of paired runs; execution order alternates AB then BA.",
+    )
+    sprint_ab_parser.add_argument(
+        "--max-invocations",
+        type=int,
+        default=20,
+        help="Hard physical model-invocation cap per arm.",
+    )
+    sprint_ab_parser.add_argument(
+        "--call-timeout-seconds",
+        type=float,
+        default=300.0,
+        help="Hard provider-call timeout.",
+    )
+    sprint_ab_parser.add_argument(
+        "--run-timeout-seconds",
+        type=float,
+        default=1800.0,
+        help="Hard full-arm timeout.",
+    )
+    sprint_ab_parser.add_argument(
+        "--keep-workspaces",
+        choices=("none", "failures", "all"),
+        default="failures",
+        help="Retain no workspaces, failed workspaces, or every workspace.",
+    )
+    sprint_ab_parser.add_argument(
+        "--rate-card-file",
+        default="",
+        help="Optional YAML rate card used only for estimated-cost reporting.",
+    )
+    sprint_ab_parser.add_argument(
+        "--output-dir",
+        default="",
+        help="Optional parent directory for benchmark artifacts.",
+    )
+    sprint_ab_parser.add_argument(
+        "--benchmark-id",
+        default="",
+        help="Optional stable artifact directory name.",
+    )
+    sprint_ab_parser.add_argument(
+        "--allow-dirty-source",
+        action="store_true",
+        help="Allow a dirty source checkout and record its state hash in provenance.",
+    )
+    sprint_ab_parser.add_argument(
+        "--json",
+        action="store_true",
+        help="Print a machine-readable result summary.",
+    )
+
     return parser
 
 
@@ -237,6 +315,7 @@ def dispatch_main(
     cmd_goal_cancel: DispatchSyncCallback,
     default_relay_transport: str,
     cmd_metrics: DispatchSyncCallback | None = None,
+    cmd_benchmark_sprint_ab: DispatchSyncCallback | None = None,
 ) -> int:
     if args.command == "init":
         return cmd_init(
@@ -341,6 +420,24 @@ def dispatch_main(
             return cmd_goal_resume(workspace_root)
         if args.goal_command in {"cancel", "terminate"}:
             return cmd_goal_cancel(workspace_root)
+    if args.command == "benchmark" and args.benchmark_command == "sprint-ab":
+        if cmd_benchmark_sprint_ab is None:
+            parser.error("Sprint benchmark command is unavailable.")
+            return 2
+        return cmd_benchmark_sprint_ab(
+            live=bool(getattr(args, "live", False)),
+            runtime_config=str(getattr(args, "runtime_config", "") or ""),
+            repetitions=int(getattr(args, "repetitions", 1)),
+            max_invocations=int(getattr(args, "max_invocations", 20)),
+            call_timeout_seconds=float(getattr(args, "call_timeout_seconds", 300.0)),
+            run_timeout_seconds=float(getattr(args, "run_timeout_seconds", 1800.0)),
+            keep_workspaces=str(getattr(args, "keep_workspaces", "failures") or "failures"),
+            rate_card_file=str(getattr(args, "rate_card_file", "") or ""),
+            output_dir=str(getattr(args, "output_dir", "") or ""),
+            benchmark_id=str(getattr(args, "benchmark_id", "") or ""),
+            allow_dirty_source=bool(getattr(args, "allow_dirty_source", False)),
+            as_json=bool(getattr(args, "json", False)),
+        )
     parser.error(f"Unsupported command: {args.command}")
     return 2
 

benchmarking/__init__.py

@@ -0,0 +1,25 @@
+"""Repeatable before/after benchmarks for teams_runtime."""
+
+from teams_runtime.benchmarking.models import (
+    ArmPlan,
+    BenchmarkOptions,
+    BenchmarkResult,
+    BenchmarkWorker,
+    QualityEvidence,
+    SprintEvidence,
+    WorkerContext,
+    WorkerOutcome,
+)
+from teams_runtime.benchmarking.runner import run_sprint_ab_benchmark
+
+__all__ = [
+    "ArmPlan",
+    "BenchmarkOptions",
+    "BenchmarkResult",
+    "BenchmarkWorker",
+    "QualityEvidence",
+    "SprintEvidence",
+    "WorkerContext",
+    "WorkerOutcome",
+    "run_sprint_ab_benchmark",
+]