diff --git a/.env.example b/.env.example index c74eeaa..3bb8f54 100644 --- a/.env.example +++ b/.env.example @@ -48,3 +48,7 @@ OPENROUTER_API_KEY=your_openrouter_api_key_here # Valid values: low | medium | high. Leave blank to let Codex decide (with a # clamp applied for models that reject its default). AGENT_REASONING_EFFORT= + +# Optional positive output-token cap for Codex Responses requests routed to a +# non-OpenAI provider. Leave blank to preserve the agent's requested budget. +AGENT_RESPONSES_MAX_OUTPUT_TOKENS= diff --git a/docker-compose.yml b/docker-compose.yml index e476b6c..e137585 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -214,6 +214,7 @@ services: - GROQ_API_KEY=${GROQ_API_KEY} - OPENROUTER_API_KEY=${OPENROUTER_API_KEY} - AGENT_REASONING_EFFORT=${AGENT_REASONING_EFFORT} + - AGENT_RESPONSES_MAX_OUTPUT_TOKENS=${AGENT_RESPONSES_MAX_OUTPUT_TOKENS:-} # Lets AGENT_UPSTREAM_BASE_URL=http://host.docker.internal:11434/v1 reach an # Ollama instance running on the host from inside this container (Docker # Desktop maps this automatically; Linux needs this explicit mapping). diff --git a/scripts/dev/seed_local_byoa_telemetry_study.py b/scripts/dev/seed_local_byoa_telemetry_study.py new file mode 100644 index 0000000..16ad2cd --- /dev/null +++ b/scripts/dev/seed_local_byoa_telemetry_study.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +"""Create one isolated, local-only Goose or Codex telemetry study. + +This development helper runs the installed agent's version and ACP initialize +checks before recording a BYOA release. It deliberately does not enroll or +consent a participant; the test account must use the normal join flow. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import secrets +import selectors +import subprocess +import sys +import time +import uuid +from datetime import datetime, timezone +from pathlib import Path + + +def require_local_dev() -> None: + if os.environ.get("CODE4ME_DEV_SEED") != "1": + raise SystemExit("Set CODE4ME_DEV_SEED=1 for local development seeding") + if os.environ.get("DB_HOST") not in (None, "localhost", "127.0.0.1"): + raise SystemExit("Refusing to seed a non-local database") + if os.environ.get("DB_NAME") != "code4meResearchDev": + raise SystemExit("Refusing to seed a database other than code4meResearchDev") + + +def check_agent(command: list[str], version_command: list[str], environment: dict[str, str]) -> str: + version = subprocess.run(version_command, check=True, capture_output=True, text=True, timeout=20, env=environment).stdout.strip() + if not version: + raise RuntimeError("agent version check produced no version") + process = subprocess.Popen( + command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + text=True, bufsize=1, env=environment, + ) + try: + process.stdin.write(json.dumps({ + "jsonrpc": "2.0", "id": 1, "method": "initialize", + "params": {"protocolVersion": 1, "clientCapabilities": {}, + "clientInfo": {"name": "code4me-local-release-check", "version": "1"}}, + }) + "\n") + process.stdin.flush() + selector = selectors.DefaultSelector() + selector.register(process.stdout, selectors.EVENT_READ) + end = time.monotonic() + 30 + while time.monotonic() < end: + if not selector.select(timeout=1): + continue + line = process.stdout.readline() + if not line: + break + response = json.loads(line) + if response.get("id") == 1: + if response.get("result", {}).get("protocolVersion") != 1: + raise RuntimeError("ACP initialize did not negotiate protocol 1") + return version + raise RuntimeError("ACP initialize did not complete") + finally: + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("framework", choices=("goose", "codex")) + parser.add_argument( + "--interaction-test", + action="store_true", + help="Create a separate local study with multi-step, per-step-approved agent actions", + ) + parser.add_argument( + "--tool-use-test", + action="store_true", + help="Create a separate Goose interaction study pinned to a tool-capable OpenRouter model", + ) + parser.add_argument( + "--free-tool-use-test", + action="store_true", + help="Create a separate Goose interaction study pinned to a tested free tool-capable model", + ) + args = parser.parse_args() + if args.tool_use_test and args.free_tool_use_test: + parser.error("Choose only one tool-use test model") + if (args.tool_use_test or args.free_tool_use_test) and (args.framework != "goose" or not args.interaction_test): + parser.error("Tool-use tests require goose --interaction-test") + model = ( + "inclusionai/ling-3.0-flash-sante:free" if args.free_tool_use_test else + "openai/gpt-4.1-mini" if args.tool_use_test else + "cohere/north-mini-code:free" + ) + gateway_tool_use = args.tool_use_test or args.free_tool_use_test + + from dotenv import load_dotenv + load_dotenv() + require_local_dev() + repo = Path(__file__).resolve().parents[2] + sys.path.insert(0, str(repo)) + from App import App + from database import crud, db_schemas + from research.study.agents import store as agent_store + from research.study.agents.enums import DistributionMode, QualificationStatus + from research.study.agents.models import AdapterRef, AgentConfigBinding, AgentReleaseV1, ReleaseTests + from research.study import lifecycle as study_lifecycle + from research.study.protocol import store as study_store + from scripts.dev.seed_research_study import FIXED_SCHEDULE_START, FIXED_SCHEDULE_END, ensure_dev_owner + from sqlalchemy import select + + client = repo.parent / "code4me2" + source = client / "dev/codex-acp-proxy/codex-acp" + environment = os.environ.copy() + if args.framework == "goose": + command = ["/opt/homebrew/bin/goose", "acp"] + version_command = [command[0], "--version"] + environment.update(GOOSE_PROVIDER="openai", GOOSE_MODEL=model, + OPENAI_BASE_URL="https://openrouter.ai/api/v1", + OPENAI_API_KEY="local-release-probe") + release_command, release_args, package = "goose", ["acp"], "goose" + bindings = [ + AgentConfigBinding(field="model", transport="env", key="GOOSE_MODEL"), + AgentConfigBinding(field="max_steps", transport="env", key="GOOSE_MAX_TURNS"), + AgentConfigBinding(field="approval_policy", transport="env", key="GOOSE_MODE", + value_map={"suggestion_only": "chat", "per_step": "approve", "auto": "auto"}), + ] + if gateway_tool_use: + bindings.append(AgentConfigBinding( + field="tools", transport="gateway", key="tool_allowlist", format="json" + )) + else: + command = [str(source / "node_modules/.bin/tsx"), str(source / "src/index.ts")] + version_command = command + ["--version"] + environment.update(CODEX_PROXY_URL="http://127.0.0.1:1/v1", + OPENAI_API_KEY="local-release-probe", CODEX_MODEL=model) + release_command, release_args, package = "codex-acp", [], "codex" + bindings = [ + AgentConfigBinding(field="model", transport="env", key="CODEX_MODEL"), + AgentConfigBinding(field="max_steps", transport="env", key="CODEX_MAX_TURNS"), + AgentConfigBinding(field="approval_policy", transport="env", key="INITIAL_AGENT_MODE", + value_map={"suggestion_only": "read-only", "per_step": "agent", "auto": "agent-full-access"}), + ] + version = check_agent(command, version_command, environment) + if args.framework == "codex": + version = version.rsplit(" ", 1)[-1] + print(f"{args.framework} self-check and ACP initialize passed (version {version})") + + app = App.get_instance() + session = app.get_db_session() + try: + owner = ensure_dev_owner( + session, f"telemetry-{args.framework}-owner@example.com", + password=secrets.token_urlsafe(32), name=f"Local {args.framework.title()} Telemetry Owner", + ) + connection = crud.get_provider_connection_by_label(session, "dev-openrouter") + if connection is None or not connection.is_active: + raise RuntimeError("dev-openrouter provider connection is missing or inactive") + release_id = f"local-telemetry-{args.framework}-{version}" + if gateway_tool_use: + release_id += "-gateway-tools-v1" + release = agent_store.get_release(session, release_id) + if release is None: + tested_at = datetime.now(timezone.utc) + evidence = { + "framework": args.framework, "version": version, + "command": release_command, "args": release_args, + "bindings": [item.model_dump(mode="json") for item in bindings], + "tested_at": tested_at.isoformat(), + } + digest = "sha256:" + hashlib.sha256(json.dumps(evidence, sort_keys=True).encode()).hexdigest() + release = agent_store.upsert_release(session, AgentReleaseV1( + agent_id=args.framework, release_id=release_id, version=version, + source_manifest_digest=digest, distribution_mode=DistributionMode.BYOA_EXTERNAL, + agent_command=release_command, agent_command_args=release_args, + agent_package=package, byoa_config=bindings, + adapter=AdapterRef(adapter_id="generic-acp", version="1.0.0"), + tests=[ReleaseTests(os="macos", arch="arm64", self_check="PASS", + acp_initialize="PASS", ran_at=tested_at)], + )) + if release.status != QualificationStatus.QUALIFIED.value: + raise RuntimeError(f"release is not qualified: {release.status}") + + study_kind = ( + "interaction-free-native-gateway-tools" if args.free_tool_use_test else + "interaction-native-gateway-tools" if args.tool_use_test else + "interaction" if args.interaction_test else "telemetry" + ) + profile_name = f"local-{study_kind}-{args.framework}" + profile = session.execute(select(db_schemas.AgentProfile).where( + db_schemas.AgentProfile.owner_user_id == owner.user_id, + db_schemas.AgentProfile.name == profile_name, + )).scalars().first() + if profile is None: + profile = db_schemas.AgentProfile( + profile_id=uuid.uuid4(), owner_user_id=owner.user_id, + name=profile_name, model=model, + framework_version=args.framework, release_id=release_id, + connection_id=connection.connection_id, + tools_json=json.dumps(["read", "edit"]) if gateway_tool_use else "[]", + approval_policy="per_step" if args.interaction_test else "suggestion_only", + max_steps=8 if args.interaction_test else 1, is_active=True, + ) + session.add(profile) + session.commit() + session.refresh(profile) + + study_id = uuid.uuid5(uuid.NAMESPACE_URL, f"code4me2://local-{study_kind}/{args.framework}") + study = study_store.get_study(session, study_id) + if study is None: + study = study_store.create_study( + session, study_id=study_id, + name=f"Local {args.framework.title()} {study_kind.title()} Verification", + description="Development-only agent interaction verification; not a research study." + if args.interaction_test else + "Development-only agent telemetry verification; not a research study.", + created_by=owner.user_id, starts_at=FIXED_SCHEDULE_START, + ends_at=FIXED_SCHEDULE_END, is_research=True, + research_config_json={ + "telemetry_policy": {"metadata_only": True}, + "session_policy": {"idle_timeout_seconds": 900, + "resume_grace_seconds": 300, + "heartbeat_seconds": 30}, + "profile_ids": [str(profile.profile_id)], + }, + join_code=study_lifecycle.allocate_join_code(session), + profile_ids=[profile.profile_id], + ) + study_store.set_study_active(session, study_id, True) + study = study_store.get_study(session, study_id) + print(f"study_id={study_id} release_id={release_id} profile_id={profile.profile_id}") + print(f"join_code={study.join_code}") + print("No participant was enrolled or consented by this script.") + finally: + session.close() + + +if __name__ == "__main__": + main() diff --git a/src/agents/inference.py b/src/agents/inference.py index 3ad4fda..393aa23 100644 --- a/src/agents/inference.py +++ b/src/agents/inference.py @@ -25,6 +25,7 @@ import json import logging import math +import os import re import time import uuid @@ -65,6 +66,78 @@ _UPSTREAM_TIMEOUT_SECONDS = 120 +def _cap_responses_output_tokens(body: dict, configured_limit: str) -> Optional[int]: + """Apply an operator limit to non-OpenAI Responses requests, if configured. + + Codex may advertise a very large default output budget. Some compatible + providers reject the request before generating anything when that budget + exceeds the account's available tokens. Never raise a caller's lower cap. + """ + try: + limit = int(configured_limit) + except (TypeError, ValueError): + return None + if limit <= 0: + return None + requested = body.get("max_output_tokens") + if isinstance(requested, int) and not isinstance(requested, bool) and requested > 0: + body["max_output_tokens"] = min(requested, limit) + else: + body["max_output_tokens"] = limit + return body["max_output_tokens"] + + +def _filter_chat_completion_tools( + body: dict, profile_tools_json: Optional[str] +) -> tuple[list[str], int, int]: + """Enforce the frozen tool selection at the managed inference gateway. + + An explicit empty profile is a no-tools arm. A missing/invalid selection + retains the legacy catalogue fallback for old profiles. + """ + requested = body.get("tools", []) or [] + names = [ + tool.get("function", {}).get("name") + for tool in requested + if isinstance(tool, dict) and tool.get("function", {}).get("name") + ] + if not requested: + return names, 0, 0 + + profile_tools: Optional[set[str]] = None + if profile_tools_json is not None: + try: + parsed = json.loads(profile_tools_json) + if isinstance(parsed, list): + profile_tools = {str(item) for item in parsed} + except (json.JSONDecodeError, TypeError): + pass + allowlist = profile_tools if profile_tools is not None else KNOWN_AGENT_TOOLS + kept = [] + for tool in requested: + if not isinstance(tool, dict): + continue + name = tool.get("function", {}).get("name") + if name in allowlist or ( + "mcp__*" in allowlist and isinstance(name, str) and name.startswith("mcp__") + ): + kept.append(tool) + sanitize_schema(tool.get("function", {}).get("parameters", {})) + if kept: + body["tools"] = kept + else: + body.pop("tools", None) + body.pop("tool_choice", None) + logging.info( + "[Agent/inference] tools — allowlist=%s requested=%s kept=%s stripped=%s", + "profile" if profile_tools is not None else "catalogue", + names, + len(kept), + len(requested) - len(kept), + ) + return names, len(kept), len(requested) - len(kept) + + def _retry_after_seconds(headers: httpx.Headers, body: bytes) -> Optional[int]: """Extract a provider rate-limit delay so agents can back off correctly.""" header_value = headers.get("retry-after") @@ -152,6 +225,11 @@ async def run_inference( "[Agent/inference] Responses API → non-OpenAI upstream, normalizing" ) normalize_responses_api_body(openai_body) + capped = _cap_responses_output_tokens( + openai_body, os.getenv("AGENT_RESPONSES_MAX_OUTPUT_TOKENS", "") + ) + if capped is not None: + logging.info("[Agent/inference] non-OpenAI Responses output cap=%s", capped) # Override the model with the task's assigned one. The agent sends whatever # its local config holds; the server decides authoritatively, from the @@ -177,7 +255,7 @@ async def run_inference( streaming = bool(openai_body.get("stream", False)) messages = openai_body.get("messages", []) or [] - max_tokens = openai_body.get("max_tokens") + max_tokens = openai_body.get("max_output_tokens" if is_responses_api else "max_tokens") # The active file path is structural metadata, stored unconditionally. active_file: Optional[str] = enrichment.get("active_file") observed_framework: Optional[str] = enrichment.get("framework_version") @@ -313,9 +391,9 @@ async def run_inference( # deliberately tool-free arm from one with a non-empty allowlist. experiment_tool_access_enabled = bool(original_tools) tool_names_requested = [ - t.get("function", {}).get("name") - for t in original_tools - if isinstance(t, dict) and t.get("function", {}).get("name") + tool.get("function", {}).get("name") + for tool in original_tools + if isinstance(tool, dict) and tool.get("function", {}).get("name") ] tools_kept = 0 tools_stripped = 0 @@ -325,44 +403,8 @@ async def run_inference( # Codex manages its own tool schemas for the Responses API — those are # passed through as-is (already normalized above where needed). # - # Distinguish "profile explicitly selected no tools" (tools_json="[]" → - # empty set, must yield zero tools) from "profile has no tool selection - # at all" (null/unset → None, falls back to the global catalogue). This - # distinction is what makes a no-tools A/B arm possible. - profile_tools: Optional[set[str]] = None - if profile_tools_json is not None: - try: - parsed = json.loads(profile_tools_json) - if isinstance(parsed, list): - profile_tools = {str(t) for t in parsed} - except (json.JSONDecodeError, TypeError): - profile_tools = None - effective_allowlist = ( - profile_tools if profile_tools is not None else KNOWN_AGENT_TOOLS - ) - kept = [] - for tool in original_tools: - tool_name = tool.get("function", {}).get("name") - if tool_name in effective_allowlist or ( - "mcp__*" in effective_allowlist - and isinstance(tool_name, str) - and tool_name.startswith("mcp__") - ): - kept.append(tool) - tools_stripped = len(original_tools) - len(kept) - tools_kept = len(kept) - for tool in kept: - sanitize_schema(tool.get("function", {}).get("parameters", {})) - if kept: - openai_body["tools"] = kept - else: - openai_body.pop("tools", None) - openai_body.pop("tool_choice", None) - logging.info( - f"[Agent/inference] tools — allowlist=" - f"{'profile' if profile_tools is not None else 'catalogue'} " - f"requested={tool_names_requested} kept={tools_kept} " - f"stripped={tools_stripped}" + tool_names_requested, tools_kept, tools_stripped = _filter_chat_completion_tools( + openai_body, profile_tools_json ) # stream_options.include_usage is a Chat Completions extension and the only diff --git a/src/agents/tools.py b/src/agents/tools.py index 4e7506f..5b255ee 100644 --- a/src/agents/tools.py +++ b/src/agents/tools.py @@ -53,6 +53,9 @@ "tree", "edit", "analyze", + # Current Goose builds also expose namespaced developer MCP tools. + "developer__shell", + "developer__text_editor", # extension manager "extensionmanager__list_resources", "extensionmanager__manage_extensions", diff --git a/src/backend/routers/research/bootstrap.py b/src/backend/routers/research/bootstrap.py index 882fa61..1158d75 100644 --- a/src/backend/routers/research/bootstrap.py +++ b/src/backend/routers/research/bootstrap.py @@ -24,7 +24,8 @@ get_current_user, ) from database import crud -from database.db_schemas import ResearchStudyStatus, Study as StudyRow +from database.db_schemas import ResearchStudyStatus +from database.db_schemas import Study as StudyRow from research.analysis.operations import store as operations_store from research.compatibility.enums import CapabilityId, CapabilityState from research.compatibility.evaluate import evaluate_compatibility @@ -38,24 +39,25 @@ from research.participants import identity as identity_store from research.runtime.assignment import store as assignment_store from research.runtime.assignment.enums import AllocationOutcome +from research.runtime.assignment.models import StudyProfileSelection from research.runtime.assignment.service import allocate +from research.runtime.bootstrap.capability import verify_capability from research.runtime.bootstrap.models import ( BootstrapAgentProfile, BootstrapManifestV1, ResearchSessionRef, ) -from research.runtime.bootstrap.capability import verify_capability -from research.runtime.bootstrap.signer import verify_manifest from research.runtime.bootstrap.service import ( BootstrapSigningContext, compose_bootstrap, ) +from research.runtime.bootstrap.signer import verify_manifest from research.runtime.sessions import store as session_store from research.runtime.sessions.enums import SessionState from research.runtime.sessions.models import ResearchSessionV1 +from research.runtime.sessions.service import expire_if_idle, session_policy_from_study from research.study.agents import store as registry_store from research.study.protocol import store as protocol_store -from research.runtime.assignment.models import StudyProfileSelection router = APIRouter() @@ -313,6 +315,24 @@ def create_for_enrollment( existing = session_store.get_active_session_for_context( self._db, enrollment.enrollment_id, context_id ) + if existing is not None: + policy = session_policy_from_study(study) + if policy is not None: + expired = expire_if_idle( + session_store.row_to_session(existing), now, policy=policy + ) + if expired.session.state.is_terminal: + # Bootstrap must not reissue a capability for a session + # whose idle deadline has passed. Retire it in the same + # transaction, then mint a fresh context session below. + if expired.transition is not None: + session_store.insert_transition( + self._db, expired.transition, commit=False + ) + session_store.update_session( + self._db, expired.session, commit=False + ) + existing = None if existing is not None: return ResearchSessionRef( research_session_id=existing.session_id, @@ -365,6 +385,7 @@ def create_research_session( allocation = allocate(enrollment, [], existing=existing, now=_now()) else: from sqlalchemy import select + from database.research_schemas import StudyAgentProfile profile_rows = db.execute( diff --git a/src/research/runtime/bootstrap/models.py b/src/research/runtime/bootstrap/models.py index d7445b7..9f49f50 100644 --- a/src/research/runtime/bootstrap/models.py +++ b/src/research/runtime/bootstrap/models.py @@ -135,8 +135,9 @@ class BootstrapAgentConfigBinding(BaseModel): """One declared BYOA profile→agent configuration translation (ISSUE-03). The release owns how a frozen profile field reaches a participant-installed - agent: an environment variable (``env``) or an argv pair (``arg``), with an - optional server-vocabulary ``value_map`` and list ``format``. Projected so + agent: an environment variable (``env``), an argv pair (``arg``), or the + managed model gateway's tool allowlist (``gateway``), with an optional + server-vocabulary ``value_map`` and list ``format``. Projected so the plugin can apply the exact contract that profile/study validation enforced. """ diff --git a/src/research/runtime/sessions/store.py b/src/research/runtime/sessions/store.py index 977b0d7..c30f49d 100644 --- a/src/research/runtime/sessions/store.py +++ b/src/research/runtime/sessions/store.py @@ -162,7 +162,7 @@ def get_active_session_for_context( def update_session( - session: Session, research_session: ResearchSessionV1 + session: Session, research_session: ResearchSessionV1, *, commit: bool = True ) -> Optional[ResearchSessionRow]: """Persist a state/timestamp/close change onto an existing session row.""" row = session.get(ResearchSessionRow, research_session.research_session_id) @@ -181,13 +181,16 @@ def update_session( row.resume_generation = research_session.resume_generation row.manifest_digest = research_session.manifest_digest row.environment_json = {"environment_ref": research_session.environment_ref} - session.commit() + if commit: + session.commit() + else: + session.flush() session.refresh(row) return row def insert_transition( - session: Session, transition: SessionTransition + session: Session, transition: SessionTransition, *, commit: bool = True ) -> SessionTransition: """Append one recorded state transition to its session's log. @@ -202,7 +205,10 @@ def insert_transition( serialized.append(transition.model_dump(mode="json")) row.transitions_json = serialized session.add(row) - session.commit() + if commit: + session.commit() + else: + session.flush() session.refresh(row) return transition diff --git a/src/research/study/agents/models.py b/src/research/study/agents/models.py index 54cd99d..2542a55 100644 --- a/src/research/study/agents/models.py +++ b/src/research/study/agents/models.py @@ -42,7 +42,7 @@ "tools", "approval_policy", ) -BYOA_CONFIG_TRANSPORTS = ("env", "arg") +BYOA_CONFIG_TRANSPORTS = ("env", "arg", "gateway") BYOA_CONFIG_FORMATS = ("string", "json", "csv") _BASE_CONFIG = ConfigDict(extra="forbid") @@ -159,7 +159,9 @@ class AgentConfigBinding(BaseModel): declares how each frozen profile field reaches the process at launch: * ``transport="env"`` sets the environment variable ``key``; - * ``transport="arg"`` appends ``[key, value]`` to the agent argv. + * ``transport="arg"`` appends ``[key, value]`` to the agent argv; + * ``transport="gateway"`` applies the tool allowlist at the managed model + gateway rather than inventing an environment variable the agent ignores. ``format`` renders list values (``tools``) as ``csv`` or ``json``; ``value_map`` translates the server-side vocabulary to the agent's (for @@ -171,7 +173,7 @@ class AgentConfigBinding(BaseModel): model_config = _BASE_CONFIG field: str - transport: Literal["env", "arg"] + transport: Literal["env", "arg", "gateway"] key: str format: Literal["string", "json", "csv"] = "string" value_map: dict[str, str] = Field(default_factory=dict) @@ -201,6 +203,14 @@ def _non_blank_key(cls, value: str) -> str: @model_validator(mode="after") def _transport_key_shape(self) -> AgentConfigBinding: + if self.transport == "gateway" and ( + self.field != "tools" + or self.key != "tool_allowlist" + or self.format != "json" + ): + raise ValueError( + "a gateway binding requires tools/tool_allowlist with JSON format" + ) if self.transport == "env" and not re.fullmatch( r"[A-Za-z_][A-Za-z0-9_]*", self.key ): diff --git a/src/research/telemetry/normalization/generic_acp.py b/src/research/telemetry/normalization/generic_acp.py index 1d0ad1d..00f1e08 100644 --- a/src/research/telemetry/normalization/generic_acp.py +++ b/src/research/telemetry/normalization/generic_acp.py @@ -47,6 +47,13 @@ #: Canonical ACP permission option kinds that mean "reject". _REJECT_OPTION_KINDS = frozenset({"reject_once", "reject_always"}) +# ACP tool titles are arbitrary display text. They can contain a shell command, +# source snippet, or absolute path, so they must not be copied into the +# metadata-only ``tool_name`` field. Only protocol-level categories are safe. +_SAFE_TOOL_KINDS = frozenset( + {"read", "edit", "delete", "move", "search", "execute", "think", "fetch", "other", "add"} +) + _METHOD_RULES: dict[str, tuple[str, CanonicalEventType]] = { "initialize": ("acp.initialize", CanonicalEventType.INTERACTION_STARTED), "session/new": ("acp.session.new", CanonicalEventType.INTERACTION_STARTED), @@ -163,13 +170,15 @@ def _tool_payload(location: Mapping[str, Any]) -> dict[str, Any]: payload: dict[str, Any] = {} for source_key, target_key in ( ("toolCallId", "tool_call_id"), - ("title", "tool_name"), - ("kind", "tool_kind"), ("status", "status"), ): value = location.get(source_key) if value is not None: payload[target_key] = value + kind = location.get("kind") + if isinstance(kind, str) and kind in _SAFE_TOOL_KINDS: + payload["tool_kind"] = kind + payload["tool_name"] = kind raw_input = location.get("rawInput") if raw_input is not None: payload["arguments"] = raw_input @@ -184,6 +193,10 @@ def _tool_call_id(update: Mapping[str, Any]) -> Optional[str]: return value if isinstance(value, str) and value else None +def _tool_correlations(update: Mapping[str, Any]) -> Correlations: + return Correlations(tool_call_id=_tool_call_id(update)) + + def _status(update: Mapping[str, Any]) -> str: return str(update.get("status") or "").lower() @@ -198,6 +211,7 @@ def _terminal_tool_candidate( CanonicalEventType.TOOL_COMPLETED, "acp.update.tool_call_update.completed", payload=_tool_payload(update), + correlations=_tool_correlations(update), lifecycle_state="completed", ) if status in {"failed", "error"}: @@ -205,6 +219,7 @@ def _terminal_tool_candidate( CanonicalEventType.TOOL_FAILED, "acp.update.tool_call_update.failed", payload=_tool_payload(update), + correlations=_tool_correlations(update), lifecycle_state="failed", ) return None @@ -524,37 +539,12 @@ def _update_candidates( ) -> list[CanonicalCandidateV1]: kind = update.get("sessionUpdate") - if kind == "agent_message_chunk": - payload: dict[str, Any] = {"message_kind": "assistant"} - if isinstance(update.get("messageId"), str): - payload["message_id"] = update["messageId"] - if update.get("content") is not None: - payload["content"] = update["content"] - return [ - _candidate( - CanonicalEventType.AGENT_MESSAGE_STARTED, - "acp.update.agent_message_chunk", - payload=payload, - lifecycle_state="started", - ) - ] - - if kind == "agent_thought_chunk": - payload = {"message_kind": "thought"} - if isinstance(update.get("messageId"), str): - payload["message_id"] = update["messageId"] - if update.get("content") is not None: - # Reasoning is CONTENT and prohibited by default; it is only kept - # when content capture is explicitly allowed AND consented. - payload["reasoning"] = update["content"] - return [ - _candidate( - CanonicalEventType.AGENT_MESSAGE_STARTED, - "acp.update.agent_thought_chunk", - payload=payload, - lifecycle_state="started", - ) - ] + if kind in {"agent_message_chunk", "agent_thought_chunk"}: + # These are streaming deltas, not separate messages. session/prompt + # already starts the turn and its response completes it. Emitting a + # start for every delta inflated one Goose reply into hundreds of + # apparent messages (and duplicated content-bearing observations). + return [] if kind == "tool_call": return self._tool_call_candidates(update) @@ -568,6 +558,7 @@ def _update_candidates( CanonicalEventType.TOOL_STARTED, "acp.update.tool_call_update", payload=_tool_payload(update), + correlations=_tool_correlations(update), lifecycle_state="started", ) ] @@ -603,6 +594,7 @@ def _tool_call_candidates( CanonicalEventType.TOOL_STARTED, "acp.update.tool_call", payload=_tool_payload(update), + correlations=_tool_correlations(update), lifecycle_state="started", ) ] @@ -614,6 +606,7 @@ def _tool_call_candidates( CanonicalEventType.TOOL_CREATED, "acp.update.tool_call.created", payload=_tool_payload(update), + correlations=_tool_correlations(update), lifecycle_state="pending", ) ] diff --git a/tests/backend_tests/research/test_bootstrap_idle_session_recovery.py b/tests/backend_tests/research/test_bootstrap_idle_session_recovery.py new file mode 100644 index 0000000..4be625e --- /dev/null +++ b/tests/backend_tests/research/test_bootstrap_idle_session_recovery.py @@ -0,0 +1,93 @@ +"""Bootstrap must not reuse a context session past its idle deadline.""" + +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace +from uuid import uuid4 + +from backend.routers.research.bootstrap import _PersistentSessionFactory +from research.runtime.sessions.enums import CloseReason, SessionState +from research.runtime.sessions.models import ResearchSessionV1 + + +def test_bootstrap_retires_idle_context_before_issuing_new_session(monkeypatch): + from backend.routers.research import bootstrap + + now = datetime(2026, 9, 23, 14, 0, tzinfo=timezone.utc) + enrollment = SimpleNamespace(enrollment_id=uuid4()) + study = SimpleNamespace( + study_id=uuid4(), + research_config_json={"session_policy": { + "idle_timeout_seconds": 900, + "resume_grace_seconds": 300, + "heartbeat_seconds": 30, + }}, + ) + previous = ResearchSessionV1( + research_session_id=uuid4(), + enrollment_id=enrollment.enrollment_id, + study_id=study.study_id, + context_id="window-a", + state=SessionState.RUNNING, + opened_at=now - timedelta(minutes=30), + last_activity_at=now - timedelta(minutes=20), + ) + old_row = SimpleNamespace(session_id=previous.research_session_id, + opened_at=previous.opened_at) + saved = [] + monkeypatch.setattr(bootstrap.session_store, "get_active_session_for_context", + lambda *_: old_row) + monkeypatch.setattr(bootstrap.session_store, "row_to_session", lambda *_: previous) + monkeypatch.setattr(bootstrap.session_store, "insert_transition", + lambda *args, **kwargs: saved.append(("transition", args[1], kwargs))) + monkeypatch.setattr(bootstrap.session_store, "update_session", + lambda *args, **kwargs: saved.append(("session", args[1], kwargs))) + monkeypatch.setattr(bootstrap.session_store, "create_session", + lambda _db, session, **_kw: SimpleNamespace( + session_id=session.research_session_id, opened_at=session.opened_at)) + + result = _PersistentSessionFactory(object(), commit=False).create_for_enrollment( + enrollment, study, now, "window-a" + ) + + assert result.research_session_id != previous.research_session_id + assert saved[0][0] == "transition" + assert saved[0][2] == {"commit": False} + assert saved[1][1].state is SessionState.ENDED + assert saved[1][1].close_reason is CloseReason.IDLE_TIMEOUT + assert saved[1][2] == {"commit": False} + + +def test_bootstrap_reuses_context_before_idle_deadline(monkeypatch): + from backend.routers.research import bootstrap + + now = datetime(2026, 9, 23, 14, 0, tzinfo=timezone.utc) + enrollment = SimpleNamespace(enrollment_id=uuid4()) + study = SimpleNamespace( + study_id=uuid4(), + research_config_json={"session_policy": { + "idle_timeout_seconds": 900, + "resume_grace_seconds": 300, + }}, + ) + previous = ResearchSessionV1( + research_session_id=uuid4(), + enrollment_id=enrollment.enrollment_id, + study_id=study.study_id, + context_id="window-a", + state=SessionState.RUNNING, + opened_at=now - timedelta(minutes=4), + last_activity_at=now - timedelta(minutes=1), + ) + old_row = SimpleNamespace(session_id=previous.research_session_id, + opened_at=previous.opened_at) + monkeypatch.setattr(bootstrap.session_store, "get_active_session_for_context", + lambda *_: old_row) + monkeypatch.setattr(bootstrap.session_store, "row_to_session", lambda *_: previous) + monkeypatch.setattr(bootstrap.session_store, "create_session", + lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("new session"))) + + result = _PersistentSessionFactory(object()).create_for_enrollment( + enrollment, study, now, "window-a" + ) + + assert result.research_session_id == previous.research_session_id diff --git a/tests/backend_tests/research/test_distributions.py b/tests/backend_tests/research/test_distributions.py index 7eec1e0..1bbe1ff 100644 --- a/tests/backend_tests/research/test_distributions.py +++ b/tests/backend_tests/research/test_distributions.py @@ -306,6 +306,41 @@ def test_bootstrap_projects_the_byoa_configuration_contract(): assert manifest.agent_profile.temperature == 0.2 +def test_goose_tools_may_be_enforced_by_the_managed_inference_gateway(): + release = _release(qualified=True, byoa=True) + release = release.model_copy(update={ + "byoa_config": [ + binding for binding in release.byoa_config if binding.field != "tools" + ] + [AgentConfigBinding( + field="tools", transport="gateway", key="tool_allowlist", format="json" + )] + }) + profile = _profile( + release_id="rel-1", framework_version="goose", + tools_json='["developer__text_editor"]', approval_policy="per_step", + ) + + validate_profile_configuration(profile, release) + manifest = _bootstrap_manifest(profile, release) + binding = next(item for item in manifest.agent_release.config_bindings if item.field == "tools") + assert binding.transport == "gateway" + assert binding.key == "tool_allowlist" + + +@pytest.mark.parametrize( + ("field", "key"), + [("model", "tool_allowlist"), ("tools", "GOOSE_TOOLS")], +) +def test_gateway_binding_rejects_unsupported_targets(field, key): + with pytest.raises(PydanticValidationError): + AgentConfigBinding(field=field, transport="gateway", key=key) + + with pytest.raises(PydanticValidationError): + AgentConfigBinding( + field="tools", transport="gateway", key="tool_allowlist", format="csv" + ) + + def test_parse_command_args_accepts_json_text_and_lists(): assert parse_command_args('["a", "b"]') == ["a", "b"] assert parse_command_args(["a"]) == ["a"] diff --git a/tests/backend_tests/research/test_generic_acp_tool_correlations.py b/tests/backend_tests/research/test_generic_acp_tool_correlations.py new file mode 100644 index 0000000..a64355c --- /dev/null +++ b/tests/backend_tests/research/test_generic_acp_tool_correlations.py @@ -0,0 +1,63 @@ +"""ACP tool lifecycle events retain their native tool-call correlation.""" + +from research.telemetry.enums import CanonicalEventType +from research.telemetry.normalization.generic_acp import GenericAcpNormalizer + + +def _update(kind: str, status: str) -> dict: + return { + "jsonrpc": "2.0", + "method": "session/update", + "params": { + "sessionId": "session-1", + "update": { + "sessionUpdate": kind, + "toolCallId": "tool-1", + "kind": "read", + "status": status, + }, + }, + } + + +def test_tool_lifecycle_carries_tool_call_id_in_canonical_correlations() -> None: + normalizer = GenericAcpNormalizer() + cases = ( + ("tool_call", "pending", CanonicalEventType.TOOL_CREATED), + ("tool_call_update", "in_progress", CanonicalEventType.TOOL_STARTED), + ("tool_call_update", "completed", CanonicalEventType.TOOL_COMPLETED), + ) + for kind, status, expected_type in cases: + candidates = normalizer.normalize(_update(kind, status)).candidates + assert len(candidates) == 1 + assert candidates[0].event_type == expected_type + assert candidates[0].correlations.tool_call_id == "tool-1" + + +def test_failed_tool_retains_tool_call_id() -> None: + candidate = GenericAcpNormalizer().normalize(_update("tool_call_update", "failed")).candidates[0] + assert candidate.event_type == CanonicalEventType.TOOL_FAILED + assert candidate.correlations.tool_call_id == "tool-1" + + +def test_free_form_tool_title_is_not_saved_as_metadata() -> None: + message = _update("tool_call", "pending") + message["params"]["update"]["title"] = "echo 'private code' > /workspace/CANARY.txt" + candidate = GenericAcpNormalizer().normalize(message).candidates[0] + assert candidate.payload["tool_name"] == "read" + assert "private code" not in str(candidate.payload) + + +def test_streaming_text_and_thought_deltas_do_not_start_new_messages() -> None: + normalizer = GenericAcpNormalizer() + for kind in ("agent_message_chunk", "agent_thought_chunk"): + for _ in range(3): + message = { + "jsonrpc": "2.0", + "method": "session/update", + "params": { + "sessionId": "session-1", + "update": {"sessionUpdate": kind, "content": {"type": "text", "text": "fragment"}}, + }, + } + assert normalizer.normalize(message).candidates == [] diff --git a/tests/backend_tests/test_agent_gateway_tools.py b/tests/backend_tests/test_agent_gateway_tools.py new file mode 100644 index 0000000..9ec0b86 --- /dev/null +++ b/tests/backend_tests/test_agent_gateway_tools.py @@ -0,0 +1,46 @@ +"""The gateway enforces BYOA tool selections without a fake agent setting.""" + +from agents.inference import _filter_chat_completion_tools + + +def _request(): + return { + "tools": [ + {"type": "function", "function": {"name": "read", "parameters": {"type": "object"}}}, + {"type": "function", "function": {"name": "edit", "parameters": {"type": "object"}}}, + {"type": "function", "function": {"name": "shell", "parameters": {"type": "object"}}}, + ], + "tool_choice": "auto", + } + + +def test_selected_goose_tool_is_forwarded_and_other_tools_are_removed(): + body = _request() + + requested, kept, stripped = _filter_chat_completion_tools( + body, '["read", "edit"]' + ) + + assert requested == ["read", "edit", "shell"] + assert (kept, stripped) == (2, 1) + assert [item["function"]["name"] for item in body["tools"]] == [ + "read", "edit" + ] + + +def test_explicit_empty_profile_removes_all_tools_and_tool_choice(): + body = _request() + + _, kept, stripped = _filter_chat_completion_tools(body, "[]") + + assert (kept, stripped) == (0, 3) + assert "tools" not in body + assert "tool_choice" not in body + + +def test_legacy_profile_uses_catalogue_fallback(): + body = _request() + + _, kept, stripped = _filter_chat_completion_tools(body, None) + + assert (kept, stripped) == (3, 0) diff --git a/tests/backend_tests/test_agent_responses_cap.py b/tests/backend_tests/test_agent_responses_cap.py new file mode 100644 index 0000000..cba4093 --- /dev/null +++ b/tests/backend_tests/test_agent_responses_cap.py @@ -0,0 +1,25 @@ +from agents.inference import _cap_responses_output_tokens + + +def test_caps_codex_default_without_raising_a_smaller_request(): + large = {"max_output_tokens": 65536} + assert _cap_responses_output_tokens(large, "512") == 512 + assert large["max_output_tokens"] == 512 + + small = {"max_output_tokens": 128} + assert _cap_responses_output_tokens(small, "512") == 128 + assert small["max_output_tokens"] == 128 + + +def test_missing_or_invalid_limit_leaves_request_unchanged(): + for configured in ("", "0", "-1", "not-a-number"): + body = {"max_output_tokens": 65536} + assert _cap_responses_output_tokens(body, configured) is None + assert body["max_output_tokens"] == 65536 + + +def test_cap_fills_missing_or_invalid_request_budget(): + for original in ({}, {"max_output_tokens": None}, {"max_output_tokens": True}): + body = dict(original) + assert _cap_responses_output_tokens(body, "512") == 512 + assert body["max_output_tokens"] == 512 diff --git a/tests/fixtures/research/telemetry/golden_acp_candidates.json b/tests/fixtures/research/telemetry/golden_acp_candidates.json index 9150794..5cccc78 100644 --- a/tests/fixtures/research/telemetry/golden_acp_candidates.json +++ b/tests/fixtures/research/telemetry/golden_acp_candidates.json @@ -31,22 +31,8 @@ "coverage_state": "AVAILABLE" } ], - [ - { - "event_type": "agent.message.started", - "fidelity": "normalized", - "mapping_rule_id": "acp.update.agent_message_chunk", - "coverage_state": "AVAILABLE" - } - ], - [ - { - "event_type": "agent.message.started", - "fidelity": "normalized", - "mapping_rule_id": "acp.update.agent_thought_chunk", - "coverage_state": "AVAILABLE" - } - ], + [], + [], [ { "event_type": "unknown_source_event", diff --git a/tests/fixtures/research/telemetry/golden_acp_completeness.json b/tests/fixtures/research/telemetry/golden_acp_completeness.json index f26b446..7547f54 100644 --- a/tests/fixtures/research/telemetry/golden_acp_completeness.json +++ b/tests/fixtures/research/telemetry/golden_acp_completeness.json @@ -50,10 +50,10 @@ "usage_tokens": null, "usage_state": "UNKNOWN", "permission_id": null, - "tool_call_id": null, + "tool_call_id": "call-5", "payload": { "tool_call_id": "call-5", - "tool_name": "Read file", + "tool_name": "read", "tool_kind": "read", "status": "pending" } @@ -67,7 +67,7 @@ "usage_tokens": null, "usage_state": "UNKNOWN", "permission_id": null, - "tool_call_id": null, + "tool_call_id": "call-5", "payload": { "tool_call_id": "call-5", "status": "in_progress" @@ -82,7 +82,7 @@ "usage_tokens": null, "usage_state": "UNKNOWN", "permission_id": null, - "tool_call_id": null, + "tool_call_id": "call-5", "payload": { "tool_call_id": "call-5", "status": "completed" @@ -97,7 +97,7 @@ "usage_tokens": null, "usage_state": "UNKNOWN", "permission_id": null, - "tool_call_id": null, + "tool_call_id": "call-6", "payload": { "tool_call_id": "call-6", "status": "failed"