From b0bf0bfe9f043b68d8912517e9cda116e8b8e73a Mon Sep 17 00:00:00 2001 From: Abhyudaya Baiju Tharakan Date: Fri, 9 Oct 2026 07:23:38 +0530 Subject: [PATCH] feat(routing): add opt-in canary and A/B auto-route cohorts Behind CANARY_TRAFFIC_ENABLED, an auto route can carry a reviewed canary policy with fixed baseline and candidate weights. Sessions are assigned to a sticky cohort by an HMAC of the authenticated principal, session, route and salt revision; requests without a session stay on the baseline. Shadow mode records the proposed order without changing dispatch, and live mode puts approved candidates first for the candidate cohort while every existing filter still applies. Responses on opted-in routes carry X-MultiLLM-Canary-Cohort. Canary policies are stored in a new canary_traffic table (migration 0025). --- docs/canary-traffic.md | 91 ++++ .../0025_canary_traffic.sql | 6 + package.json | 2 +- route_helpers.py | 2 +- routes/auto_routes.py | 25 +- services/auto_route_service.py | 50 ++- services/canary_traffic.py | 236 +++++++++++ tests/fixtures/secret_firewall_coverage.json | 1 + tests/test_canary_traffic.py | 390 ++++++++++++++++++ tests/test_canary_traffic_worker.mjs | 168 ++++++++ worker/canary-assignment.mjs | 149 +++++++ worker/container-env.mjs | 1 + worker/cors-policy.mjs | 2 +- worker/d1-schema.mjs | 2 +- 14 files changed, 1114 insertions(+), 11 deletions(-) create mode 100644 docs/canary-traffic.md create mode 100644 intelligence-migrations/0025_canary_traffic.sql create mode 100644 services/canary_traffic.py create mode 100644 tests/test_canary_traffic.py create mode 100644 tests/test_canary_traffic_worker.mjs create mode 100644 worker/canary-assignment.mjs diff --git a/docs/canary-traffic.md b/docs/canary-traffic.md new file mode 100644 index 00000000..cd2ca640 --- /dev/null +++ b/docs/canary-traffic.md @@ -0,0 +1,91 @@ +# Canary traffic for auto routes + +`CANARY_TRAFFIC_ENABLED` defaults to `false`. An empty value also disables it. +Malformed values disable it and produce one warning without logging the value. +Each route defaults to `{"enabled": false}`. Explicit provider models and routes +without an enabled canary policy keep their existing dispatch and response headers. + +An administrator saves the candidate chain and its canary policy together through +the existing authenticated `PUT /admin/auto-routes` endpoint. For example: + +```json +{ + "route_id": "auto:experiment", + "candidates": ["openai:baseline", "openai:candidate"], + "canary": { + "enabled": true, + "mode": "shadow", + "weights": {"baseline": 90, "candidate": 10}, + "salt_revision": "r1", + "approved_candidates": ["openai:candidate"] + } +} +``` + +With `CANARY_TRAFFIC_ENABLED=true`, shadow mode assigns a proposed cohort and +records its proposed order. It sends exactly the baseline request; it makes no +comparison or additional provider call. Change `mode` to `live` in a reviewed save +to use the proposed order for candidate sessions. Approved candidates move ahead +of the other candidates, preserving the configured order within each group. +Health ordering and per-candidate permission, budget, credential, cooldown and +deadline checks run afterwards and retain precedence. The existing refusal and +stream failover rules still apply. Assignment does not grant retry permission. + +Weights must be non-negative integers summing to 100. Approved candidates must +be distinct models already in this route. A nonzero enabled candidate weight +requires an approved candidate. The salt revision is a nonempty identifier of +at most 128 letters, digits, dots, underscores or hyphens, starting with a letter +or digit. Missing optional fields use shadow mode, 100/0 weights, revision `1` +and no approved candidates. Invalid configuration returns HTTP 400 before saving. +Weights never change automatically. + +For local storage, include `expected_updated_at` from the route read to reject +a concurrent edit with HTTP 409. Durable saves require `current_revision` when +configuration snapshots or revision sync are enabled; stale revisions return +409. Candidate order, policy and the durable revision commit atomically. Any +durable save failure returns 503 rather than retaining an unconfirmed local copy. +A save omitting `canary` resets the route to its disabled default. To disable an +experiment explicitly, save its chain with `"canary": {"enabled": false}`. +Clearing the global flag disables routing changes without changing saved policy. + +Cohorts use HMAC-SHA256 with the existing server-side `JWT_SECRET`. The input is +compact UTF-8 JSON containing the format label `multillm-canary-v1`, the verified +principal, session identifier, route ID and salt revision. The principal is the +compact JSON pair `[tenant_id, id-or-username]` from authenticated request state. +The first four digest bytes, interpreted as an unsigned big-endian integer modulo +100, select baseline below its weight and candidate otherwise. Changing weights, +salt revision, identity or the server secret can reassign sessions. + +Session lookup follows prompt cache affinity: `X-OpenCode-Session`, `Session-Id`, +`Thread-Id`, body `session_id`, body `conversation_id`, metadata `session_id`, +metadata `conversation_id`, then the signed authenticated session ID. Without a +session or verified principal the cohort is baseline. A caller-supplied principal +or cohort is ignored. An unavailable assignment secret gives baseline routing and +one warning. A session identifier is an affinity hint, not an authorization grant. + +Responses from enabled routes carry the browser-readable +`X-MultiLLM-Canary-Cohort: candidate; mode=shadow` header (or `baseline` and `live` +as appropriate). Cohort identifies an assignment, not which provider won after +the existing filters and fallback. The auto-selected-model header remains authoritative. + +The additive `0025_canary_traffic.sql` migration stores only reviewed route policy +and the matching route update timestamp in `canary_traffic`. Apply it before +enabling durable routing. A missing table, failed read or invalid stored policy +falls back to baseline with one warning and does not cause a generation 500. +An older policy never applies to a newly saved candidate chain. Local storage +creates this table only during an explicit canary save; ordinary saves do not. + +Request observations contain route, cohort, mode, proposed order and health-ordered +candidate order; candidates rejected during validation are removed from the latter. +Unattempted candidates still need their normal validation if reached. Aggregate +cohort counts retain only route/cohort/mode/count, with at most 800 counters per +process. Counts reset on process restart and measure requests, not unique sessions. +Native Worker dispatch supplies verified identity through `authenticatedPrincipal` +and `sessionIdentifier`, applies `prepareCanary` before eligibility, and observes +the resulting order through its request lifecycle. Forwarded requests use the +Container assignment. Neither runtime stores session plaintext, principal, +HMAC output, prompts, responses or credentials for this feature. + +Shadow mode has the baseline request's cost. Live mode can select a more expensive +approved model; existing budgets still apply. Cohort counts do not establish +quality, statistical significance, provider capabilities or exact billing. diff --git a/intelligence-migrations/0025_canary_traffic.sql b/intelligence-migrations/0025_canary_traffic.sql new file mode 100644 index 00000000..8323820a --- /dev/null +++ b/intelligence-migrations/0025_canary_traffic.sql @@ -0,0 +1,6 @@ +-- Reviewed route metadata only. Assignments retain no session or principal. +CREATE TABLE IF NOT EXISTS canary_traffic ( + route_id TEXT PRIMARY KEY, + route_updated_at TEXT NOT NULL, + configuration TEXT NOT NULL +); diff --git a/package.json b/package.json index e90d16c7..5c0e96db 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "lint": "ruff check . --select E9,F63,F7,F82", "typecheck": "mypy --ignore-missing-imports --follow-imports=skip --disable-error-code import-untyped --disable-error-code var-annotated --disable-error-code arg-type --disable-error-code assignment --disable-error-code operator app.py routes services providers streaming", "test": "python -m pytest -q", - "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_managed_idempotency_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_gateway_alerts_worker.mjs tests/test_session_tiers_worker.mjs tests/test_usage_reservations_worker.mjs tests/test_shared_generation_cache_worker.mjs tests/test_deferred_tools_worker.mjs tests/test_generation_deadline_worker.mjs", + "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_managed_idempotency_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_gateway_alerts_worker.mjs tests/test_session_tiers_worker.mjs tests/test_usage_reservations_worker.mjs tests/test_shared_generation_cache_worker.mjs tests/test_deferred_tools_worker.mjs tests/test_generation_deadline_worker.mjs tests/test_canary_traffic_worker.mjs", "db:migrate": "python scripts/validate_sqlite_schema.py --apply", "db:validate": "python scripts/validate_sqlite_schema.py", "security:secrets": "python scripts/check_static_secrets.py", diff --git a/route_helpers.py b/route_helpers.py index b5f9fb02..65fe4248 100644 --- a/route_helpers.py +++ b/route_helpers.py @@ -50,7 +50,7 @@ "X-MultiLLM-Auto-Route, X-MultiLLM-Auto-Selected-Model, " "X-MultiLLM-Auto-Attempts, X-MultiLLM-Auto-Selected-Priority, " "X-MultiLLM-Auto-Ordering, X-MultiLLM-Auto-Failover-Reasons, " - "X-MultiLLM-Cache, X-MultiLLM-Cache-Backend, X-MultiLLM-Usage-Basis, X-MultiLLM-Provider-Calls, Age, X-MultiLLM-Retention, X-MultiLLM-Roleplay-Recovery, X-MultiLLM-Transport-Failure, X-MultiLLM-Token-Count, " + "X-MultiLLM-Cache, X-MultiLLM-Cache-Backend, X-MultiLLM-Usage-Basis, X-MultiLLM-Provider-Calls, Age, X-MultiLLM-Retention, X-MultiLLM-Canary-Cohort, X-MultiLLM-Roleplay-Recovery, X-MultiLLM-Transport-Failure, X-MultiLLM-Token-Count, " "X-MultiLLM-Latency-Ms, X-MultiLLM-Estimated-Cost-USD, " "X-MultiLLM-Cost-Basis, WWW-Authenticate, X-PAYMENT-RESPONSE, X-Poll-After, " "X-NanoGPT-Advisor-ID, X-NanoGPT-Data-Endpoint, " diff --git a/routes/auto_routes.py b/routes/auto_routes.py index 34a29457..7e423dad 100644 --- a/routes/auto_routes.py +++ b/routes/auto_routes.py @@ -9,6 +9,7 @@ from services.model_cooldown import ModelCooldownCapacity, ModelCooldownExhausted from providers.registry import get_registry from services import cloudflare_ai +from services import canary_traffic from services.auto_route_service import AutoRoute, AutoRouteService from services.media_catalog import ( TRANSPORT_FAILURE_HEADER, @@ -129,6 +130,7 @@ def _decorate_response( route_decision: str | None = None, ordering: str = "priority", failures: list[tuple[str, str]] | None = None, + canary: canary_traffic.CanaryAssignment | None = None, ) -> Response: selected_provider, _ = ModelRegistry.parse_model_id(selected_model) if route_decision is None: @@ -147,6 +149,8 @@ def _decorate_response( g.multillm_provider = selected_provider g.multillm_model = selected_model g.multillm_route_decision = route_decision + if canary is not None: + canary.decorate(response) return response @@ -172,7 +176,11 @@ def dispatch_auto_route( status_code=404, ) - order = RouteHealth.order(route.id, route.candidates, circuit_state=_circuit_state) + canary = canary_traffic.prepare_request(route, payload) + candidates = canary.dispatch_order if canary is not None else route.candidates + order = RouteHealth.order(route.id, candidates, circuit_state=_circuit_state) + if canary is not None: + canary.observe(order.candidates) priorities = {candidate: index for index, candidate in enumerate(route.candidates)} attempts = 0 failures: list[tuple[str, str]] = [] @@ -195,6 +203,9 @@ def dispatch_auto_route( type(error).__name__, ) failures.append((candidate, "skipped")) + if canary is not None: + g.multillm_canary["eligible_order"] = [model for model in g.multillm_canary["eligible_order"] + if model != candidate] continue attempts += 1 @@ -260,6 +271,7 @@ def dispatch_auto_route( route_decision=route_decision, ordering=order.mode, failures=failures, + canary=canary, ) logger.info("Auto route %s moves past %s (%s)", route.id, candidate, reason) @@ -281,6 +293,7 @@ def dispatch_auto_route( route_decision=last_decision, ordering=order.mode, failures=[failure for failure in failures if failure[0] != last_model], + canary=canary, ) raise APIError( f"No configured provider is available for auto route: {route.id}", @@ -428,6 +441,7 @@ def _admin_payload(app, auth_service_cls) -> dict: "candidates": candidates, "updated_at": route.updated_at, "ordering": settings.mode_for(route.id), + **({"canary": route.canary.as_dict()} if route.canary.enabled else {}), } ) return { @@ -454,6 +468,9 @@ def register_auto_route_admin_routes( auth_service_cls, proxy_service_cls, ) -> None: + # Preserve cohort visibility when existing policy guards raise an API error. + app.after_request(canary_traffic.decorate_current_response) + @app.route("/admin/auto-routes", methods=["GET", "PUT"]) @login_required def admin_auto_routes(): @@ -467,10 +484,16 @@ def admin_auto_routes(): status_code=400, ) try: + canary_options = {"canary": payload["canary"]} if "canary" in payload else {} + if "expected_updated_at" in payload: + canary_options["expected_updated_at"] = payload["expected_updated_at"] + if "current_revision" in payload: + canary_options["current_revision"] = payload["current_revision"] AutoRouteService.save_route( payload.get("route_id"), payload.get("candidates"), app.config["API_BASE_URLS"], + **canary_options, ) except ValueError as error: raise APIError(str(error), status_code=400) from error diff --git a/services/auto_route_service.py b/services/auto_route_service.py index ac9fdfea..97efa7f3 100644 --- a/services/auto_route_service.py +++ b/services/auto_route_service.py @@ -1,18 +1,20 @@ import re import sqlite3 from contextlib import closing -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import datetime, timezone from typing import Iterable, Mapping from providers.registry import get_registry -from services import auto_route_d1, config_revision_sync +from error_handlers import APIError +from services import auto_route_d1, canary_traffic, config_revision_sync from services.model_registry import ModelRegistry from services.sqlite_store import connect, storage_path AUTO_ROUTE_PREFIX = "auto:" MAX_AUTO_ROUTE_CANDIDATES = 16 +_CANARY_UNSET = object() # Media routes follow the Artificial Analysis image arena (Sep 2026): GPT Image 2.5 # Sunburst and Flare lead, then GPT Image 2 and Grok Imagine Image 2.0. GGUU serves # them first at a flat ¥0.04 per image (1K to 4K); OpenAI charges about $0.21 at max. @@ -232,6 +234,7 @@ class AutoRoute: id: str candidates: tuple[str, ...] updated_at: str + canary: canary_traffic.CanaryConfig = field(default_factory=canary_traffic.CanaryConfig) class AutoRouteService: @@ -383,7 +386,7 @@ def normalize_candidates( return tuple(normalized) @classmethod - def _rows_to_routes(cls, rows: Iterable[sqlite3.Row]) -> list[AutoRoute]: + def _rows_to_routes(cls, rows: Iterable[sqlite3.Row], configurations=None) -> list[AutoRoute]: candidates_by_route: dict[str, list[str]] = {} updated_at_by_route: dict[str, str] = {} for row in rows: @@ -397,6 +400,8 @@ def _rows_to_routes(cls, rows: Iterable[sqlite3.Row]) -> list[AutoRoute]: id=route_id, candidates=tuple(candidates), updated_at=updated_at_by_route[route_id], + canary=canary_traffic.stored_config(configurations or {}, route_id, + updated_at_by_route[route_id], candidates), ) for route_id, candidates in candidates_by_route.items() ] @@ -408,11 +413,13 @@ def _durable_routes(cls) -> list[AutoRoute]: if stored is None: stored = auto_route_d1.stored_routes() routes = {route_id: AutoRoute(route_id, candidates, "") for route_id, candidates in DEFAULT_AUTO_ROUTES.items()} + configurations = canary_traffic.durable_configurations() for route_id, (candidates, updated_at) in stored.items(): # A route still holding a retired default follows the current default. if candidates in LEGACY_DEFAULT_AUTO_ROUTES.get(route_id, ()): candidates = DEFAULT_AUTO_ROUTES[route_id] - routes[route_id] = AutoRoute(route_id, candidates, updated_at) + routes[route_id] = AutoRoute(route_id, candidates, updated_at, + canary_traffic.stored_config(configurations, route_id, updated_at, candidates)) return [routes[route_id] for route_id in sorted(routes)] @classmethod @@ -431,7 +438,8 @@ def list_routes(cls) -> list[AutoRoute]: ORDER BY routes.route_id, candidates.priority """ ).fetchall() - return cls._rows_to_routes(rows) + configurations = canary_traffic.local_configurations(connection) + return cls._rows_to_routes(rows, configurations) @classmethod def get_route(cls, route_id: object) -> AutoRoute | None: @@ -452,7 +460,8 @@ def get_route(cls, route_id: object) -> AutoRoute | None: """, (normalized,), ).fetchall() - routes = cls._rows_to_routes(rows) + configurations = canary_traffic.local_configurations(connection) + routes = cls._rows_to_routes(rows, configurations) return routes[0] if routes else None @classmethod @@ -461,18 +470,44 @@ def save_route( route_id: object, candidates: object, base_urls: Mapping[str, str], + *, + canary: object = _CANARY_UNSET, + expected_updated_at: str | None = None, + current_revision: int | None = None, ) -> AutoRoute: normalized_route_id = cls.normalize_route_id(route_id) normalized_candidates = cls.normalize_candidates(candidates, base_urls) + policy = canary_traffic.normalize_config({} if canary is _CANARY_UNSET else canary, normalized_candidates) + if expected_updated_at is not None and not isinstance(expected_updated_at, str): + raise ValueError("Expected route timestamp must be a string") + if current_revision is not None and (type(current_revision) is not int or not 0 <= current_revision < 9007199254740991): + raise ValueError("Current route revision must be a non-negative safe integer") updated_at = _utcnow_iso() if auto_route_d1.using_d1(): + if expected_updated_at is not None: + raise ValueError("Durable route saves require current_revision instead of a local timestamp") + if canary is not _CANARY_UNSET or current_revision is not None: + result = canary_traffic.durable_request("canary_put", route_id=normalized_route_id, + candidates=list(normalized_candidates), updated_at=updated_at, canary=policy.as_dict(), + current_revision=current_revision) + if result.get("stored") is not True: + raise APIError("Canary route storage did not confirm the save", 503) + auto_route_d1.reset_cache() + return AutoRoute(normalized_route_id, normalized_candidates, updated_at, policy) auto_route_d1.save_route(normalized_route_id, normalized_candidates, updated_at) return AutoRoute(id=normalized_route_id, candidates=normalized_candidates, updated_at=updated_at) + if current_revision is not None: + raise ValueError("Local route saves require expected_updated_at instead of a durable revision") with closing(cls._connect()) as connection: cls._ensure_storage(connection) connection.commit() connection.execute("BEGIN IMMEDIATE") + if expected_updated_at is not None: + previous = connection.execute("SELECT updated_at FROM auto_routes WHERE route_id = ?", + (normalized_route_id,)).fetchone() + if previous is None or previous["updated_at"] != expected_updated_at: + raise APIError("Auto route revision changed; reload before saving", 409) connection.execute( """ INSERT INTO auto_routes (route_id, updated_at) @@ -495,10 +530,13 @@ def save_route( for priority, model_id in enumerate(normalized_candidates) ], ) + if canary is not _CANARY_UNSET: + canary_traffic.save_local_configuration(connection, normalized_route_id, updated_at, policy) connection.commit() return AutoRoute( id=normalized_route_id, candidates=normalized_candidates, updated_at=updated_at, + canary=policy, ) diff --git a/services/canary_traffic.py b/services/canary_traffic.py new file mode 100644 index 00000000..e55bc2f8 --- /dev/null +++ b/services/canary_traffic.py @@ -0,0 +1,236 @@ +"""Reviewed route cohorts; identity and content never enter storage or telemetry.""" +import hashlib +import hmac +import json +import logging +import os +import re +import sqlite3 +from dataclasses import dataclass +from threading import Lock + +from flask import current_app, g, has_request_context, request + +from error_handlers import APIError +from services import intelligence_d1_store + +HEADER = "X-MultiLLM-Canary-Cohort" +logger = logging.getLogger(__name__) +_warned: set[str] = set() +_lock = Lock() +_counts: dict[tuple[str, str, str], int] = {} +_MAX_COUNT_KEYS = 800 +_REVISION = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}\Z") +_SCHEMA = """CREATE TABLE IF NOT EXISTS canary_traffic ( + route_id TEXT PRIMARY KEY, + route_updated_at TEXT NOT NULL, + configuration TEXT NOT NULL +)""" + + +def warn_once(reason: str) -> None: + with _lock: + if reason in _warned: + return + _warned.add(reason) + logger.warning("Canary traffic %s; using baseline routing", reason) + + +def enabled(environ=None) -> bool: + source = os.environ if environ is None else environ + flag = str(source.get("CANARY_TRAFFIC_ENABLED") or "").strip().lower() + if flag not in {"", "0", "false", "no", "off", "1", "true", "yes", "on"}: + warn_once("flag is invalid") + return False + return flag in {"1", "true", "yes", "on"} + + +@dataclass(frozen=True) +class CanaryConfig: + enabled: bool = False + mode: str = "shadow" + baseline: int = 100 + candidate: int = 0 + salt_revision: str = "1" + approved_candidates: tuple[str, ...] = () + + @property + def weights(self) -> dict[str, int]: + return {"baseline": self.baseline, "candidate": self.candidate} + + def as_dict(self) -> dict: + return {"enabled": self.enabled, "mode": self.mode, "weights": self.weights, + "salt_revision": self.salt_revision, "approved_candidates": list(self.approved_candidates)} + + +def normalize_config(raw: object, candidates) -> CanaryConfig: + fields = {"enabled", "mode", "weights", "salt_revision", "approved_candidates"} + if not isinstance(raw, dict) or set(raw) - fields: + raise ValueError("Canary configuration must be an object with supported fields") + flag = raw.get("enabled", False) + mode = raw.get("mode", "shadow") + weights = raw.get("weights", {"baseline": 100, "candidate": 0}) + revision = raw.get("salt_revision", "1") + approved = raw.get("approved_candidates", []) + if type(flag) is not bool or not isinstance(mode, str) or mode not in {"shadow", "live"}: + raise ValueError("Canary enabled must be a boolean and mode must be shadow or live") + if (not isinstance(weights, dict) or set(weights) != {"baseline", "candidate"} + or any(type(value) is not int or value < 0 for value in weights.values()) + or sum(weights.values()) != 100): + raise ValueError("Canary weights must be non-negative integers summing to 100") + if not isinstance(revision, str) or not _REVISION.fullmatch(revision): + raise ValueError("Canary salt revision must be a bounded identifier") + if (not isinstance(approved, list) or len(approved) > 16 + or any(not isinstance(model, str) or model not in candidates for model in approved) + or len(set(approved)) != len(approved)): + raise ValueError("Approved canary candidates must be distinct models in this route") + if flag and weights["candidate"] and not approved: + raise ValueError("Candidate traffic requires an approved candidate") + return CanaryConfig(flag, mode, weights["baseline"], weights["candidate"], revision, tuple(approved)) + + +def assign_cohort(policy: CanaryConfig, *, principal, session, route_id, key) -> str: + if not policy.enabled: + return "baseline" + if not isinstance(key, (str, bytes)) or not key: + warn_once("assignment secret is unavailable") + return "baseline" + if (not isinstance(principal, str) or not principal.strip() + or not isinstance(session, str) or not session.strip()): + return "baseline" + # JSON framing distinguishes separators in session and principal values. + try: + message = json.dumps(["multillm-canary-v1", principal, session, route_id, policy.salt_revision], + ensure_ascii=False, separators=(",", ":")).encode("utf-8") + digest = hmac.new(key.encode("utf-8") if isinstance(key, str) else key, message, hashlib.sha256).digest() + except UnicodeError: + return "baseline" + bucket = int.from_bytes(digest[:4], "big") % 100 + return "baseline" if bucket < policy.baseline else "candidate" + + +def session_identifier(payload: dict, user: dict) -> str | None: + """Match prompt cache affinity's session sources, without its prefix fallback.""" + headers = {name.lower(): value for name, value in request.headers.items()} + metadata = payload.get("metadata") + metadata = metadata if isinstance(metadata, dict) else {} + values = (headers.get("x-opencode-session"), headers.get("session-id"), headers.get("thread-id"), + payload.get("session_id"), payload.get("conversation_id"), metadata.get("session_id"), + metadata.get("conversation_id"), user.get("session_id")) + return next((value for value in values if isinstance(value, str) and value.strip()), None) + + +def authenticated_principal(user: dict) -> str | None: + identity = user.get("id") or user.get("username") + return json.dumps([user.get("tenant_id"), identity], ensure_ascii=False, separators=(",", ":")) if identity else None + + +@dataclass +class CanaryAssignment: + route_id: str + cohort: str + mode: str + proposed_order: tuple[str, ...] + dispatch_order: tuple[str, ...] + + def observe(self, eligible_order) -> None: + event = {"route_id": self.route_id, "cohort": self.cohort, "mode": self.mode, + "proposed_order": list(self.proposed_order), "eligible_order": list(eligible_order)} + g.multillm_canary = event + with _lock: + key = (self.route_id, self.cohort, self.mode) + if key in _counts or len(_counts) < _MAX_COUNT_KEYS: + _counts[key] = _counts.get(key, 0) + 1 + + def decorate(self, response): + response.headers[HEADER] = f"{self.cohort}; mode={self.mode}" + return response + + +def prepare_request(route, payload: dict) -> CanaryAssignment | None: + if not enabled() or not route.canary.enabled or not has_request_context(): + return None + user = getattr(g, "authenticated_user", None) + user = user if isinstance(user, dict) else {} + # Use the same authenticated tenant/user pair as prompt cache affinity. + principal = authenticated_principal(user) + cohort = assign_cohort(route.canary, principal=principal, session=session_identifier(payload, user), + route_id=route.id, key=current_app.config.get("JWT_SECRET")) + approved = set(route.canary.approved_candidates) + candidate_order = tuple(model for model in route.candidates if model in approved) + tuple( + model for model in route.candidates if model not in approved) + proposed = candidate_order if cohort == "candidate" else route.candidates + dispatch = proposed if cohort == "candidate" and route.canary.mode == "live" else route.candidates + return CanaryAssignment(route.id, cohort, route.canary.mode, proposed, dispatch) + + +def cohort_counts() -> list[dict]: + with _lock: + return [{"route_id": route, "cohort": cohort, "mode": mode, "count": count} + for (route, cohort, mode), count in sorted(_counts.items())] + + +def decorate_current_response(response): + event = getattr(g, "multillm_canary", None) + if event is not None: + response.headers[HEADER] = f"{event['cohort']}; mode={event['mode']}" + return response + + +def local_configurations(connection) -> dict: + if not enabled(): + return {} + if connection.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='canary_traffic'").fetchone() is None: + warn_once("configuration storage is unavailable") + return {} + return {row["route_id"]: (row["route_updated_at"], row["configuration"]) + for row in connection.execute("SELECT route_id, route_updated_at, configuration FROM canary_traffic")} + + +def stored_config(configurations, route_id, updated_at, candidates) -> CanaryConfig: + row = configurations.get(route_id) + if row is None or row[0] != updated_at: + return CanaryConfig() + try: + raw = json.loads(row[1]) if isinstance(row[1], str) else row[1] + return normalize_config(raw, candidates) + except (ValueError, TypeError): + warn_once("stored configuration is invalid") + return CanaryConfig() + + +def save_local_configuration(connection, route_id, updated_at, policy: CanaryConfig) -> None: + connection.execute(_SCHEMA) + connection.execute("""INSERT INTO canary_traffic (route_id, route_updated_at, configuration) + VALUES (?, ?, ?) ON CONFLICT(route_id) DO UPDATE SET + route_updated_at=excluded.route_updated_at, configuration=excluded.configuration""", + (route_id, updated_at, json.dumps(policy.as_dict(), separators=(",", ":")))) + + +def durable_request(operation, **values): + try: + result = intelligence_d1_store.request_private_intelligence( + {"version": 1, "operation": operation, **values}, endpoint="auto_routes") + if not isinstance(result, dict) or result.get("version") != 1 or "error" in result: + raise ValueError("Unconfirmed canary storage operation") + return result + except intelligence_d1_store.PrivateIntelligenceError as error: + if error.status in {400, 409}: + raise APIError("Canary route save was rejected", error.status) from None + except Exception: + pass + raise APIError("Canary route storage is unavailable; the route was not saved", 503) from None + + +def durable_configurations() -> dict: + if not enabled(): + return {} + try: + response = durable_request("canary_list") + rows = response.get("routes") + if not isinstance(rows, list) or len(rows) > 200: + raise ValueError("Invalid configuration list") + return {row["route_id"]: (row["updated_at"], row["canary"]) for row in rows} + except (APIError, KeyError, TypeError, ValueError, sqlite3.Error): + warn_once("configuration storage is unavailable") + return {} diff --git a/tests/fixtures/secret_firewall_coverage.json b/tests/fixtures/secret_firewall_coverage.json index 2c079f26..f64dee35 100644 --- a/tests/fixtures/secret_firewall_coverage.json +++ b/tests/fixtures/secret_firewall_coverage.json @@ -225,6 +225,7 @@ "worker/api-paths.mjs", "worker/auto-routes-d1.mjs", "worker/build-id.mjs", + "worker/canary-assignment.mjs", "worker/cascades-d1.mjs", "worker/client-headers.mjs", "worker/config-revision.mjs", diff --git a/tests/test_canary_traffic.py b/tests/test_canary_traffic.py new file mode 100644 index 00000000..a1fe2756 --- /dev/null +++ b/tests/test_canary_traffic.py @@ -0,0 +1,390 @@ +"""Reviewed canary routing with synthetic identity, storage and transports.""" +import importlib +import json +import sqlite3 +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from unittest.mock import patch + +import pytest +from flask import Flask, Response, g + +MODELS = ("openai:baseline", "openai:candidate", "openai:other") +BASE_URLS = {"openai": "https://synthetic.invalid/v1"} +KEY = "synthetic-canary-key" +VECTOR = ("tenant:user", "session-42", "auto:experiment", "r1") + + +@pytest.fixture(autouse=True) +def isolated(monkeypatch, tmp_path): + global canary, routes, service + canary = importlib.import_module("services.canary_traffic") + routes = importlib.import_module("routes.auto_routes") + service = importlib.import_module("services.auto_route_service") + for name, value in {"CANARY_TRAFFIC_ENABLED": "true", "AUTO_ROUTE_ORDERING": "priority", + "INTELLIGENCE_STORAGE_BACKEND": "", "CONTROL_PLANE_DATABASE_URL": "", + "GENERATION_DEADLINE_ENABLED": "false", "MULTILLM_STREAM_PREFLIGHT": "off", + "MODEL_REGISTRY_DB_PATH": str(tmp_path / "models.sqlite3")}.items(): + monkeypatch.setenv(name, value) + monkeypatch.setattr(canary, "_warned", set()) + monkeypatch.setattr(canary, "_counts", {}) + routes.RouteHealth.reset() + yield + routes.RouteHealth.reset() + + +def config(**changes): + raw = {"enabled": True, "mode": "live", "weights": {"baseline": 0, "candidate": 100}, + "salt_revision": "r1", "approved_candidates": [MODELS[1]]} + raw.update(changes) + return raw + + +def app(): + application = Flask(__name__) + application.config.update(TESTING=True, JWT_SECRET=KEY, API_BASE_URLS=BASE_URLS) + return application + + +def save(**changes): + return service.AutoRouteService.save_route("auto:experiment", list(MODELS), BASE_URLS, + canary=config(**changes)) + + +def dispatch(application, *, payload=None, principal=True, headers=None, rejected=(), status=200): + payload = payload or {"model": "auto:experiment", "session_id": "session-42"} + calls = [] + validated = [] + def validate(model): + validated.append(model) + if model in rejected: + raise routes.APIError("Denied by existing policy", 403) + def send(body, model, decision): + calls.append(model) + return Response("provider body", status=status) + with application.test_request_context("/v1/chat/completions", method="POST", json=payload, headers=headers): + if principal: + g.authenticated_user = {"tenant_id": "tenant", "id": "user"} + response = routes.dispatch_auto_route_chat_completion(payload, validate_candidate=validate, + dispatch_candidate=send) + telemetry = getattr(g, "multillm_canary", None) + return response, calls, validated, telemetry + + +def test_shared_assignment_vector_and_concurrency(): + policy = canary.normalize_config(config(weights={"baseline": 50, "candidate": 50}), MODELS) + def assign(_): + return canary.assign_cohort(policy, principal=VECTOR[0], session=VECTOR[1], + route_id=VECTOR[2], key=KEY) + assert set(ThreadPoolExecutor(max_workers=8).map(assign, range(64))) == {"baseline"} + + +def test_weights_revision_identity_and_unambiguous_framing(): + policy = canary.normalize_config(config(weights={"baseline": 50, "candidate": 50}), MODELS) + def cohort(principal=VECTOR[0], session=VECTOR[1], revision="r1"): + policy2 = canary.normalize_config(config(weights=policy.weights, salt_revision=revision), MODELS) + return canary.assign_cohort(policy2, principal=principal, session=session, route_id=VECTOR[2], key=KEY) + assert {cohort(session=f"s{i}") for i in range(200)} == {"baseline", "candidate"} + assert any(cohort(session=f"s{i}") != cohort(session=f"s{i}", revision="r2") for i in range(40)) + assert any(cohort(principal="tenant:other", session=f"s{i}") != cohort(session=f"s{i}") for i in range(40)) + zero = canary.normalize_config(config(weights={"baseline": 100, "candidate": 0}), MODELS) + assert canary.assign_cohort(zero, principal="a", session="b", route_id="auto:x", key=KEY) == "baseline" + + +@pytest.mark.parametrize("change", [dict(enabled="true"), dict(mode="adaptive"), dict(salt_revision=""), + dict(weights={"baseline": -1, "candidate": 101}), dict(weights={"baseline": 50.0, "candidate": 50}), + dict(weights={"baseline": True, "candidate": 99}), dict(weights={"baseline": 40, "candidate": 40}), + dict(approved_candidates=["openai:outside"]), dict(approved_candidates=[MODELS[1], MODELS[1]]), + dict(approved_candidates=[]), dict(extra="ignored"), dict(salt_revision=5)]) +def test_invalid_config_rejected_before_storage(change): + with pytest.raises(ValueError): + service.AutoRouteService.save_route("auto:experiment", list(MODELS), BASE_URLS, + canary=config(**change)) + assert not Path(service.storage_path("MODEL_REGISTRY_DB_PATH", "unused")).exists() + + +@pytest.mark.parametrize("raw", [None, [], "canary", 0]) +def test_malformed_object_rejected(raw): + with pytest.raises(ValueError): + canary.normalize_config(raw, MODELS) + + +@pytest.mark.parametrize("flag", ["", "false", "off", "0", "malformed-private-value"]) +def test_disabled_exact_dispatch_headers_telemetry_and_warning(flag, monkeypatch, caplog): + save() + monkeypatch.setenv("CANARY_TRAFFIC_ENABLED", flag) + application = app() + response, calls, _, telemetry = dispatch(application) + assert calls == [MODELS[0]] and response.data == b"provider body" + assert canary.HEADER not in response.headers and telemetry is None + dispatch(application) + assert "malformed-private-value" not in caplog.text + assert len(caplog.records) == (1 if flag == "malformed-private-value" else 0) + assert canary.cohort_counts() == [] + + +def test_legacy_route_keeps_storage_headers_and_telemetry_unchanged(): + service.AutoRouteService.save_route("auto:experiment", list(MODELS), BASE_URLS) + response, calls, _, telemetry = dispatch(app()) + assert calls == [MODELS[0]] and telemetry is None and canary.HEADER not in response.headers + with service.AutoRouteService._connect() as conn: + assert conn.execute("SELECT name FROM sqlite_master WHERE name='canary_traffic'").fetchone() is None + + +def test_global_off_does_not_read_or_warn_about_stored_config(monkeypatch, caplog): + save() + monkeypatch.setenv("CANARY_TRAFFIC_ENABLED", "false") + def refuse_read(*args): + raise AssertionError("Disabled traffic must not load canary policy") + monkeypatch.setattr(canary, "json", type("Unreadable", (), {"loads": refuse_read})) + response, calls, _, telemetry = dispatch(app()) + assert calls == [MODELS[0]] and telemetry is None and canary.HEADER not in response.headers + assert caplog.records == [] + + +def test_live_reorders_shadow_dispatches_once_and_records_proposal(): + save(mode="shadow") + response, calls, _, telemetry = dispatch(app()) + assert calls == [MODELS[0]] and response.headers[canary.HEADER] == "candidate; mode=shadow" + assert telemetry["cohort"] == "candidate" and telemetry["mode"] == "shadow" + assert telemetry["proposed_order"] == [MODELS[1], MODELS[0], MODELS[2]] + save() + response, calls, _, telemetry = dispatch(app()) + assert calls == [MODELS[1]] and response.headers[canary.HEADER] == "candidate; mode=live" + assert telemetry["eligible_order"] == [MODELS[1], MODELS[0], MODELS[2]] + counts = canary.cohort_counts() + assert sum(row["count"] for row in counts) == 2 + assert "session-42" not in json.dumps(counts) and "tenant:user" not in json.dumps(counts) + + +def test_baseline_cohort_proposes_the_baseline_chain(): + save(mode="shadow", weights={"baseline": 100, "candidate": 0}) + response, calls, _, event = dispatch(app()) + assert calls == [MODELS[0]] and event["proposed_order"] == list(MODELS) + assert response.headers[canary.HEADER] == "baseline; mode=shadow" + + +def test_permission_and_health_ordering_win(monkeypatch): + save() + response, calls, validated, _ = dispatch(app(), rejected=(MODELS[1],)) + assert calls == [MODELS[0]] and validated == [MODELS[1], MODELS[0]] + seen = [] + from services.route_health import RouteOrder + def ordered(route, candidates, **kw): + seen.append(tuple(candidates)) + return RouteOrder((MODELS[0], MODELS[1], MODELS[2]), "health") + monkeypatch.setattr(routes.RouteHealth, "order", ordered) + _, calls, _, _ = dispatch(app()) + assert seen == [(MODELS[1], MODELS[0], MODELS[2])] and calls == [MODELS[0]] + + +@pytest.mark.parametrize("payload,principal", [({"model": "auto:experiment"}, True), + ({"model": "auto:experiment", "session_id": "session-42", "principal": "spoof", "cohort": "candidate"}, False)]) +def test_missing_identity_or_session_is_baseline(payload, principal): + save() + response, calls, _, _ = dispatch(app(), payload=payload, principal=principal, + headers={"X-MultiLLM-Canary-Cohort": "candidate", "X-Principal": "spoof"}) + assert calls == [MODELS[0]] and response.headers[canary.HEADER] == "baseline; mode=live" + + +def test_missing_key_warns_once_and_stays_baseline(caplog): + save() + application = app() + application.config.pop("JWT_SECRET") + for _ in range(3): + response, calls, _, _ = dispatch(application) + assert calls == [MODELS[0]] and response.headers[canary.HEADER] == "baseline; mode=live" + assert len(caplog.records) == 1 and KEY not in caplog.text + + +def test_session_sources_match_affinity_precedence(): + application = app() + with application.test_request_context(headers={"X-OpenCode-Session": "header", "Session-Id": "second"}): + assert canary.session_identifier({"session_id": "body"}, {"session_id": "signed"}) == "header" + with application.test_request_context(): + assert canary.session_identifier({"metadata": {"conversation_id": "metadata"}}, {}) == "metadata" + assert canary.session_identifier({}, {"session_id": "signed"}) == "signed" + assert canary.session_identifier({}, {}) is None + assert canary.authenticated_principal({"tenant_id": "tenant", "id": "user"}) == '["tenant","user"]' + assert canary.authenticated_principal({"username": "user"}) == '[null,"user"]' + assert canary.authenticated_principal({}) is None + + +@pytest.mark.parametrize("status", [400, 408, 504]) +def test_no_new_retry_for_ambiguous_or_request_failures(status): + save() + response, calls, _, _ = dispatch(app(), status=status) + assert response.status_code == status and calls == [MODELS[1]] + + +def test_refusals_keep_fallback_and_canary_header(): + save() + response, calls, _, _ = dispatch(app(), status=429) + assert calls == [MODELS[1], MODELS[0], MODELS[2]] + assert response.status_code == 429 and response.headers[canary.HEADER] == "candidate; mode=live" + + +def test_save_roundtrip_atomic_revision_and_old_rows(tmp_path): + service.AutoRouteService.save_route("auto:old", [MODELS[0]], BASE_URLS) + stored = save() + loaded = service.AutoRouteService.get_route(stored.id) + assert loaded.canary.as_dict() == config() + assert service.AutoRouteService.get_route("auto:old").canary.enabled is False + with pytest.raises(routes.APIError) as error: + service.AutoRouteService.save_route(stored.id, list(reversed(MODELS)), BASE_URLS, + canary=config(), expected_updated_at="stale") + assert error.value.status_code == 409 + assert service.AutoRouteService.get_route(stored.id) == loaded + conn = sqlite3.connect(tmp_path / "migration.sqlite3") + conn.execute("CREATE TABLE old (value TEXT)") + conn.execute("INSERT INTO old VALUES ('old-row')") + migration = (Path(__file__).resolve().parents[1] / "intelligence-migrations/0025_canary_traffic.sql").read_text() + conn.executescript(migration) + conn.executescript(migration) + assert conn.execute("SELECT * FROM old").fetchall() == [("old-row",)] + assert "session" not in {r[1] for r in conn.execute("PRAGMA table_info(canary_traffic)")} + conn.close() + + +def test_missing_durable_table_and_unwired_operation_fail_closed(monkeypatch, caplog): + monkeypatch.setattr(service.auto_route_d1, "using_d1", lambda: True) + monkeypatch.setattr(service.config_revision_sync, "route_copy", lambda: None) + monkeypatch.setattr(service.auto_route_d1, "stored_routes", lambda: {"auto:experiment": (MODELS, "r1")}) + def unavailable(*args, **kw): + raise routes.APIError("Storage unavailable", 503) + monkeypatch.setattr(canary, "durable_request", unavailable) + loaded = service.AutoRouteService.get_route("auto:experiment") + assert loaded.canary.enabled is False + service.AutoRouteService.get_route("auto:experiment") + assert len(caplog.records) == 1 + with pytest.raises(routes.APIError) as error: + save() + assert error.value.status_code == 503 + + +def test_durable_save_and_read_use_fixed_operation_and_preserve_revision(monkeypatch): + monkeypatch.setattr(service.auto_route_d1, "using_d1", lambda: True) + monkeypatch.setattr(service.config_revision_sync, "route_copy", lambda: None) + calls = [] + saved = {} + def storage(operation, **values): + calls.append((operation, values)) + if operation == "canary_put": + saved.update(values) + return {"version": 1, "stored": True} + return {"version": 1, "routes": [{"route_id": saved["route_id"], + "updated_at": saved["updated_at"], "canary": saved["canary"]}]} + monkeypatch.setattr(canary, "durable_request", storage) + monkeypatch.setattr(service.auto_route_d1, "stored_routes", lambda: { + "auto:experiment": (tuple(saved["candidates"]), saved["updated_at"])}) + stored = service.AutoRouteService.save_route("auto:experiment", list(MODELS), BASE_URLS, + canary=config(), current_revision=4) + assert calls[0][0] == "canary_put" and calls[0][1]["current_revision"] == 4 + assert service.AutoRouteService.get_route(stored.id) == stored + monkeypatch.setenv("CANARY_TRAFFIC_ENABLED", "false") + previous = len(calls) + assert not service.AutoRouteService.get_route(stored.id).canary.enabled + assert len(calls) == previous + + +def test_registered_admin_save_refuses_invalid_config_with_real_400(monkeypatch): + application = app() + from error_handlers import init_error_handlers + init_error_handlers(application) + class Auth: + @staticmethod + def get_current_user(): + return {"is_admin": True} + monkeypatch.setattr(routes, "refresh_model_catalogs", lambda *a: None) + monkeypatch.setattr(routes, "_admin_payload", lambda *a: {"routes": []}) + routes.register_auto_route_admin_routes(application, lambda f: f, Auth, object) + client = application.test_client() + assert client.put("/admin/auto-routes", json={"route_id": "auto:experiment", "candidates": list(MODELS), + "canary": config(mode="adaptive")}).status_code == 400 + assert client.put("/admin/auto-routes", json={"route_id": "auto:experiment", "candidates": list(MODELS), + "canary": config()}).status_code == 200 + assert service.AutoRouteService.get_route("auto:experiment").canary.enabled + + +def test_storage_failure_rolls_back_both_candidates_and_configuration(monkeypatch): + previous = save() + def unavailable(*args): + raise sqlite3.OperationalError("synthetic unavailable storage") + monkeypatch.setattr(canary, "save_local_configuration", unavailable) + with pytest.raises(sqlite3.OperationalError): + service.AutoRouteService.save_route(previous.id, list(reversed(MODELS)), BASE_URLS, + canary=config(mode="shadow")) + assert service.AutoRouteService.get_route(previous.id) == previous + + +def test_explicit_route_save_without_canary_disables_previous_review(): + save() + service.AutoRouteService.save_route("auto:experiment", list(MODELS), BASE_URLS) + response, calls, _, telemetry = dispatch(app()) + assert calls == [MODELS[0]] and canary.HEADER not in response.headers and telemetry is None + + +def test_registered_admission_guard_keeps_cohort_visible_without_dispatch(monkeypatch): + save() + application = app() + from error_handlers import init_error_handlers + init_error_handlers(application) + class Auth: + @staticmethod + def get_current_user(): + return {"is_admin": True} + routes.register_auto_route_admin_routes(application, lambda f: f, Auth, object) + calls = [] + @application.post("/synthetic-auto-dispatch") + def generate(): + g.authenticated_user = {"id": "verified"} + def admission(model): + raise routes.ModelCooldownCapacity(1) + return routes.dispatch_auto_route({"model": "auto:experiment", "session_id": "s"}, + validate_candidate=admission, dispatch_candidate=lambda *args: calls.append(args)) + response = application.test_client().post("/synthetic-auto-dispatch", json={}) + assert response.status_code == 503 and calls == [] + assert response.headers[canary.HEADER] == "candidate; mode=live" + + +from tests.unified_api_test_case import UnifiedApiTestCase + + +class RegisteredCanaryTest(UnifiedApiTestCase): + def setUp(self): + with patch("env_loader.load_runtime_env"), patch("config.load_runtime_env"): + super().setUp() + self.flag = patch.dict("os.environ", {"CANARY_TRAFFIC_ENABLED": "true", + "INTELLIGENCE_STORAGE_BACKEND": "", "AUTO_ROUTE_ORDERING": "priority", + "SECRET_SCAN_DEFAULT": "off", "MODEL_COOLDOWN_ENABLED": "false"}) + self.flag.start() + self.addCleanup(self.flag.stop) + + def test_registered_admin_and_authenticated_generation_shadow_and_live(self): + with self.client.session_transaction() as session: + session["authenticated"] = True + session["user"] = {"username": "admin", "is_admin": True, + "api_key_prefix": "mllm_admin-te", "scopes": ["admin"]} + active_routes = importlib.import_module("routes.auto_routes") + policy = config(approved_candidates=["mimo:mimo-v2-flash"]) + saved = {"route_id": "auto:registered-canary", "candidates": ["opencode:glm-5.2", "mimo:mimo-v2-flash"]} + for mode, provider in [("shadow", "opencode"), ("live", "mimo")]: + policy["mode"] = mode + with patch.object(active_routes, "refresh_model_catalogs"), patch("requests.sessions.Session.request", + side_effect=AssertionError("Provider network is forbidden")): + response = self.client.put("/admin/auto-routes", json={**saved, "canary": policy}) + self.assertEqual(response.status_code, 200) + stored = next(row for row in response.json["routes"] if row["id"] == saved["route_id"]) + self.assertEqual(stored["canary"], policy) + with patch.object(self.app_module.AuthService, "get_api_keys", return_value=[]), \ + patch("requests.sessions.Session.request", side_effect=AssertionError("Provider network is forbidden")): + # Track the actual transport collaborator through its object, after module re-imports. + with patch.object(self.app_module.ProxyService, "make_request", return_value=self._chat_response()) as send: + response = self.client.post("/v1/chat/completions", headers={"Authorization": "Bearer admin-test-key", + "X-MultiLLM-Canary-Cohort": "baseline", "X-Principal": "spoof"}, + json={"model": saved["route_id"], "session_id": "session-42", + "messages": [{"role": "user", "content": "synthetic prompt"}]}) + self.assertEqual(response.status_code, 200) + self.assertEqual(send.call_count, 1) + self.assertEqual(send.call_args.kwargs["api_provider"], provider) + self.assertEqual(response.headers["X-MultiLLM-Canary-Cohort"], f"candidate; mode={mode}") diff --git a/tests/test_canary_traffic_worker.mjs b/tests/test_canary_traffic_worker.mjs new file mode 100644 index 00000000..9b729ed1 --- /dev/null +++ b/tests/test_canary_traffic_worker.mjs @@ -0,0 +1,168 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { webcrypto } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { DatabaseSync } from "node:sqlite"; +import { assignCohort, canaryEnabled, normalizeCanary, prepareCanary, + handleCanaryRouteOperation, authenticatedPrincipal, sessionIdentifier, CANARY_HEADER } from "../worker/canary-assignment.mjs"; + +globalThis.crypto ??= webcrypto; +const candidates = ["openai:baseline", "openai:candidate", "openai:other"]; +const key = "synthetic-canary-key"; +const config = (changes = {}) => ({ enabled: true, mode: "live", weights: { baseline: 0, candidate: 100 }, + salt_revision: "r1", approved_candidates: [candidates[1]], ...changes }); + +test("shared Python vector, stable assignment and parallel requests", async () => { + const policy = normalizeCanary(config({ weights: { baseline: 50, candidate: 50 } }), candidates); + const input = { principal: "tenant:user", session: "session-42", routeId: "auto:experiment", key }; + assert.deepEqual(new Set(await Promise.all(Array.from({ length: 64 }, () => assignCohort(policy, input)))), new Set(["baseline"])); + assert.equal(await assignCohort(policy, { ...input, session: null }), "baseline"); + assert.equal(await assignCohort(policy, { ...input, principal: null }), "baseline"); + assert.equal(await assignCohort(policy, { ...input, key: null }), "baseline"); +}); + +test("weights and salt revisions change only reviewed cohorts", async () => { + const first = normalizeCanary(config({ weights: { baseline: 50, candidate: 50 } }), candidates); + const second = normalizeCanary(config({ weights: { baseline: 50, candidate: 50 }, salt_revision: "r2" }), candidates); + const pairs = await Promise.all(Array.from({ length: 100 }, async (_, i) => { + const input = { principal: "tenant:user", session: `s${i}`, routeId: "auto:experiment", key }; + return [await assignCohort(first, input), await assignCohort(second, input)]; + })); + assert.deepEqual(new Set(pairs.flat()), new Set(["baseline", "candidate"])); + assert.ok(pairs.some(([a, b]) => a !== b)); +}); + +test("verified principal and session lookup match Container framing", () => { + assert.equal(authenticatedPrincipal({ tenant_id: "tenant", id: "user" }), '["tenant","user"]'); + assert.equal(authenticatedPrincipal({ username: "user" }), '[null,"user"]'); + assert.equal(authenticatedPrincipal({}), null); + assert.equal(sessionIdentifier(new Headers({ "X-OpenCode-Session": "header", "Session-Id": "second" }), + { session_id: "body" }, { session_id: "signed" }), "header"); + assert.equal(sessionIdentifier(new Headers(), { metadata: { conversation_id: "metadata" } }), "metadata"); + assert.equal(sessionIdentifier(new Headers(), {}, { session_id: "signed" }), "signed"); + assert.equal(sessionIdentifier(new Headers()), null); +}); + +test("observation failures stay observational", async () => { + const warnings = []; + const assignment = await prepareCanary({ id: "auto:experiment", candidates, canary: config() }, { + env: { CANARY_TRAFFIC_ENABLED: "true", JWT_SECRET: key }, principal: "verified", session: "s", + observe: async () => { throw new Error("private telemetry detail"); }, warn: text => warnings.push(text) }); + assert.equal(assignment.candidates[0], candidates[1]); + assert.equal(warnings.length, 1); + assert.ok(!warnings.join().includes("private telemetry detail")); +}); + +test("defaults and invalid flags leave inputs identical and warn without values", async () => { + const warnings = []; + for (const flag of [undefined, "", "false", "off", "malformed-worker-value"]) { + const env = { CANARY_TRAFFIC_ENABLED: flag, JWT_SECRET: key }; + const route = { id: "auto:experiment", candidates, canary: config() }; + assert.equal(await prepareCanary(route, { env, principal: "verified", session: "s", warn: text => warnings.push(text) }), null); + canaryEnabled(env, text => warnings.push(text)); + assert.equal(route.candidates, candidates); + } + assert.equal(warnings.length, 1); + assert.ok(!warnings.join().includes("malformed-worker-value")); + assert.equal(await prepareCanary({ id: "auto:old", candidates }, { env: { CANARY_TRAFFIC_ENABLED: "true" } }), null); +}); + +test("malformed configurations refuse saves", () => { + for (const change of [{ enabled: "true" }, { mode: "adaptive" }, { weights: { baseline: -1, candidate: 101 } }, + { weights: { baseline: true, candidate: 99 } }, { weights: { baseline: 40, candidate: 40 } }, + { approved_candidates: ["openai:outside"] }, { approved_candidates: [] }, + { approved_candidates: [candidates[1], candidates[1]] }, { salt_revision: "" }, { extra: true }]) { + assert.throws(() => normalizeCanary(config(change), candidates)); + } + for (const value of [null, [], "canary", 0]) assert.throws(() => normalizeCanary(value, candidates)); +}); + +test("live proposal, shadow zero dispatch, content-free observation and policy wins", async () => { + for (const mode of ["live", "shadow"]) { + const events = []; + const assignment = await prepareCanary({ id: "auto:experiment", candidates, canary: config({ mode }) }, { + env: { CANARY_TRAFFIC_ENABLED: "true", JWT_SECRET: key }, principal: "verified", session: "private-session", + observe: event => events.push(event) }); + assert.deepEqual(assignment.proposedOrder, [candidates[1], candidates[0], candidates[2]]); + assert.deepEqual(assignment.candidates, mode === "live" ? assignment.proposedOrder : candidates); + const eligible = assignment.candidates.filter(model => model !== candidates[1]); + assert.equal(eligible[0], candidates[0]); + const response = new Response("provider body"); + assignment.decorate(response); + assert.equal(response.headers.get(CANARY_HEADER), `candidate; mode=${mode}`); + assert.equal(events.length, 1); + assert.ok(!JSON.stringify(events).includes("private-session")); + assert.ok(!JSON.stringify(events).includes("verified")); + } +}); + +test("baseline assignment proposes the unchanged route", async () => { + const assignment = await prepareCanary({ id: "auto:experiment", candidates, + canary: config({ mode: "shadow", weights: { baseline: 100, candidate: 0 } }) }, { + env: { CANARY_TRAFFIC_ENABLED: "true", JWT_SECRET: key }, principal: "verified", session: "s" }); + assert.equal(assignment.cohort, "baseline"); + assert.equal(assignment.proposedOrder, candidates); + assert.equal(assignment.candidates, candidates); +}); + +function database() { + const sql = new DatabaseSync(":memory:"); + sql.exec("CREATE TABLE auto_routes (route_id TEXT PRIMARY KEY, candidates TEXT NOT NULL, updated_at TEXT NOT NULL)"); + sql.exec("CREATE TABLE config_snapshot_revisions (domain TEXT PRIMARY KEY, revision INTEGER NOT NULL)"); + const migration = readFileSync(new URL("../intelligence-migrations/0025_canary_traffic.sql", import.meta.url), "utf8"); + sql.exec(migration); sql.exec(migration); + const db = { prepare(query) { + const statement = sql.prepare(query); + let args = []; + return { bind(...values) { args = values; return this; }, + async all() { return { results: statement.all(...args) }; }, + async first() { return statement.get(...args) ?? null; }, + async run() { const result = statement.run(...args); return { success: true, meta: { changes: Number(result.changes) } }; } }; + }, async batch(statements) { + sql.exec("BEGIN"); + try { const results = []; for (const statement of statements) results.push(await statement.run()); sql.exec("COMMIT"); return results; } + catch (error) { sql.exec("ROLLBACK"); throw error; } + } }; + return { db, sql }; +} +const write = (changes = {}) => ({ version: 1, operation: "canary_put", route_id: "auto:experiment", candidates, + updated_at: "2026-10-09T00:00:00Z", canary: config(), current_revision: null, ...changes }); + +test("atomic route/config persistence, revision conflict and stale config baseline", async t => { + const { db, sql } = database(); t.after(() => sql.close()); + const env = { CONFIG_REVISION_SYNC_ENABLED: "true" }; + assert.equal((await handleCanaryRouteOperation(db, write({ current_revision: 0 }), env)).status, 200); + assert.equal((await handleCanaryRouteOperation(db, write({ current_revision: 0, canary: config({ mode: "shadow" }) }), env)).status, 409); + const result = await (await handleCanaryRouteOperation(db, { version: 1, operation: "canary_list" }, env)).json(); + assert.equal(result.routes[0].canary.mode, "live"); + assert.equal(sql.prepare("SELECT revision FROM config_snapshot_revisions").get().revision, 1); + sql.prepare("UPDATE auto_routes SET updated_at='new-review'").run(); + assert.deepEqual((await (await handleCanaryRouteOperation(db, { version: 1, operation: "canary_list" }, env)).json()).routes, []); +}); + +test("invalid settings, missing schema and failed transaction cannot partially save", async t => { + const { db, sql } = database(); t.after(() => sql.close()); + assert.equal((await handleCanaryRouteOperation(db, write({ canary: config({ mode: "adaptive" }) }), {})).status, 400); + assert.equal(sql.prepare("SELECT COUNT(*) AS n FROM auto_routes").get().n, 0); + const warnings = []; + const broken = { prepare() { throw new Error("private database text"); } }; + const response = await handleCanaryRouteOperation(broken, write(), {}, text => warnings.push(text)); + assert.equal(response.status, 503); + assert.ok(!(await response.text()).includes("private")); + assert.equal(warnings.length, 1); + assert.equal((await handleCanaryRouteOperation(db, write(), { CONFIG_SNAPSHOTS_ENABLED: "true" })).status, 409); + assert.equal(await handleCanaryRouteOperation(db, { version: 1, operation: "list" }, {}), null); +}); + +test("a policy write failure rolls back route, policy and revision together", async t => { + const { db, sql } = database(); t.after(() => sql.close()); + const env = { CONFIG_REVISION_SYNC_ENABLED: "true" }; + assert.equal((await handleCanaryRouteOperation(db, write({ current_revision: 0 }), env)).status, 200); + sql.exec("CREATE TRIGGER reject_policy BEFORE INSERT ON canary_traffic BEGIN SELECT RAISE(ABORT, 'synthetic failure'); END"); + const response = await handleCanaryRouteOperation(db, write({ current_revision: 1, updated_at: "2026-10-09T01:00:00Z", + candidates: [...candidates].reverse(), canary: config({ mode: "shadow" }) }), env); + assert.equal(response.status, 503); + assert.equal(sql.prepare("SELECT revision FROM config_snapshot_revisions").get().revision, 1); + assert.equal(sql.prepare("SELECT updated_at FROM auto_routes").get().updated_at, "2026-10-09T00:00:00Z"); + assert.equal(JSON.parse(sql.prepare("SELECT configuration FROM canary_traffic").get().configuration).mode, "live"); +}); diff --git a/worker/canary-assignment.mjs b/worker/canary-assignment.mjs new file mode 100644 index 00000000..f39c4dd3 --- /dev/null +++ b/worker/canary-assignment.mjs @@ -0,0 +1,149 @@ +/** Reviewed cohort policy; trusted identity is supplied by the authentication boundary. */ +export const CANARY_HEADER = "X-MultiLLM-Canary-Cohort"; +const warned = new Set(); +function warnOnce(reason, warn = console.warn, baseline = true) { + if (warned.has(reason)) return; + warned.add(reason); + warn(`Canary traffic ${reason}${baseline ? "; using baseline routing" : ""}`); +} +const flagOn = value => ["1", "true", "yes", "on"].includes(String(value || "").trim().toLowerCase()); +const record = value => value !== null && typeof value === "object" && !Array.isArray(value); +const fields = (value, names) => record(value) && Object.keys(value).length === names.length + && names.every(name => Object.hasOwn(value, name)); + +export function canaryEnabled(env = {}, warn = console.warn) { + const flag = String(env.CANARY_TRAFFIC_ENABLED || "").trim().toLowerCase(); + if (!["", "0", "false", "no", "off", "1", "true", "yes", "on"].includes(flag)) { + warnOnce("flag is invalid", warn); + return false; + } + return flagOn(flag); +} + +export function normalizeCanary(value, candidates) { + const allowed = ["enabled", "mode", "weights", "salt_revision", "approved_candidates"]; + if (!record(value) || Object.keys(value).some(name => !allowed.includes(name))) throw new Error("Invalid canary configuration"); + const enabled = Object.hasOwn(value, "enabled") ? value.enabled : false; + const mode = Object.hasOwn(value, "mode") ? value.mode : "shadow"; + const weights = Object.hasOwn(value, "weights") ? value.weights : { baseline: 100, candidate: 0 }; + const revision = Object.hasOwn(value, "salt_revision") ? value.salt_revision : "1"; + const approved = Object.hasOwn(value, "approved_candidates") ? value.approved_candidates : []; + if (typeof enabled !== "boolean" || !["shadow", "live"].includes(mode) + || !fields(weights, ["baseline", "candidate"]) + || !Object.values(weights).every(n => Number.isSafeInteger(n) && n >= 0) + || weights.baseline + weights.candidate !== 100 + || typeof revision !== "string" || !/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(revision) + || !Array.isArray(approved) || approved.length > 16 + || !approved.every(model => typeof model === "string" && candidates.includes(model)) + || new Set(approved).size !== approved.length || enabled && weights.candidate > 0 && !approved.length) { + throw new Error("Invalid canary configuration"); + } + return Object.freeze({ enabled, mode, weights: Object.freeze({ ...weights }), salt_revision: revision, + approved_candidates: Object.freeze([...approved]) }); +} + +export async function assignCohort(policy, { principal, session, routeId, key, warn = console.warn }) { + if (!policy.enabled) return "baseline"; + if (typeof key !== "string" || !key) { + warnOnce("assignment secret is unavailable", warn); + return "baseline"; + } + if (typeof principal !== "string" || !principal.trim() || typeof session !== "string" || !session.trim()) return "baseline"; + if ([principal, session, routeId, key].some(value => /[\uD800-\uDFFF]/u.test(value))) return "baseline"; + const text = JSON.stringify(["multillm-canary-v1", principal, session, routeId, policy.salt_revision]); + const encoder = new TextEncoder(); + const secret = await crypto.subtle.importKey("raw", encoder.encode(key), { name: "HMAC", hash: "SHA-256" }, false, ["sign"]); + const digest = await crypto.subtle.sign("HMAC", secret, encoder.encode(text)); + const bucket = new DataView(digest).getUint32(0, false) % 100; + return bucket < policy.weights.baseline ? "baseline" : "candidate"; +} + +export function authenticatedPrincipal(user) { + const identity = user?.id || user?.username; + return identity ? JSON.stringify([user.tenant_id ?? null, identity]) : null; +} + +/** Match Container prompt cache affinity, without assigning a missing session. */ +export function sessionIdentifier(headers, payload = {}, user = {}) { + const metadata = record(payload.metadata) ? payload.metadata : {}; + return [headers.get("x-opencode-session"), headers.get("session-id"), headers.get("thread-id"), + payload.session_id, payload.conversation_id, metadata.session_id, metadata.conversation_id, user.session_id] + .find(value => typeof value === "string" && value.trim()) ?? null; +} + +/** Call before existing eligibility and health ordering; never dispatches a provider. */ +export async function prepareCanary(route, { env = {}, principal, session, observe = () => {}, warn = console.warn } = {}) { + if (!canaryEnabled(env, warn) || !route.canary?.enabled) return null; + let policy; + try { policy = normalizeCanary(route.canary, route.candidates); } + catch { warnOnce("stored configuration is invalid", warn); return null; } + const cohort = await assignCohort(policy, { principal, session, routeId: route.id, key: env.JWT_SECRET, warn }); + const approved = new Set(policy.approved_candidates); + const candidateOrder = [...route.candidates.filter(model => approved.has(model)), ...route.candidates.filter(model => !approved.has(model))]; + const proposedOrder = cohort === "candidate" ? candidateOrder : route.candidates; + const candidates = cohort === "candidate" && policy.mode === "live" ? proposedOrder : route.candidates; + try { await observe({ route_id: route.id, cohort, mode: policy.mode, proposed_order: [...proposedOrder] }); } + catch { warnOnce("observation is unavailable", warn, false); } + return Object.freeze({ cohort, mode: policy.mode, proposedOrder, candidates, + decorate(response) { response.headers.set(CANARY_HEADER, `${cohort}; mode=${policy.mode}`); return response; } }); +} + +const reply = (body, status = 200) => Response.json(body, { status, headers: { "cache-control": "no-store" } }); +const failure = (code, status) => reply({ version: 1, error: { code, message: "Canary route storage operation failed" } }, status); +const routeIdValid = value => typeof value === "string" && /^auto:[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(value); +const candidateListValid = value => Array.isArray(value) && value.length > 0 && value.length <= 16 + && new Set(value).size === value.length && value.every(model => typeof model === "string" + && /^[A-Za-z0-9][A-Za-z0-9._:/+@-]{0,255}$/.test(model) && model.includes(":") && !model.startsWith("auto:")); +const revisionValid = value => Number.isSafeInteger(value) && value >= 0 && value < Number.MAX_SAFE_INTEGER; + +/** Parsed operations from handleAutoRoutesRequest's existing private authorization boundary. */ +export async function handleCanaryRouteOperation(db, body, env = {}, warn = console.warn) { + if (!["canary_list", "canary_put"].includes(body?.operation)) return null; + if (body.version !== 1) return failure("invalid_request", 400); + try { + if (body.operation === "canary_list") { + if (!fields(body, ["version", "operation"])) return failure("invalid_request", 400); + const { results } = await db.prepare(`SELECT policy.route_id, policy.route_updated_at, policy.configuration + FROM canary_traffic AS policy JOIN auto_routes AS route ON policy.route_id=route.route_id + AND policy.route_updated_at=route.updated_at ORDER BY policy.route_id LIMIT 200`).all(); + return reply({ version: 1, routes: results.map(row => ({ route_id: row.route_id, + updated_at: row.route_updated_at, canary: JSON.parse(row.configuration) })) }); + } + const names = ["version", "operation", "route_id", "candidates", "updated_at", "canary", "current_revision"]; + if (!fields(body, names) || !routeIdValid(body.route_id) || !candidateListValid(body.candidates) + || typeof body.updated_at !== "string" || !/^[0-9T:.+\-Z]{10,40}$/.test(body.updated_at) + || body.current_revision !== null && !revisionValid(body.current_revision)) return failure("invalid_request", 400); + let policy; + try { policy = normalizeCanary(body.canary, body.candidates); } + catch { return failure("invalid_request", 400); } + const revisioned = flagOn(env.CONFIG_SNAPSHOTS_ENABLED) || flagOn(env.CONFIG_REVISION_SYNC_ENABLED) + || body.current_revision !== null; + if (revisioned && body.current_revision === null) return failure("revision_conflict", 409); + return await saveRoute(db, body, policy, revisioned); + } catch { + warnOnce("configuration storage is unavailable", warn); + return failure("canary_route_storage_unavailable", 503); + } +} + +async function saveRoute(db, body, policy, revisioned) { + // Probe before any route write: a code deployment does not apply the migration. + await db.prepare("SELECT route_id FROM canary_traffic LIMIT 1").all(); + const statements = []; + if (revisioned) statements.push( + db.prepare("INSERT OR IGNORE INTO config_snapshot_revisions (domain, revision) VALUES ('auto_routes', 0)"), + db.prepare(`UPDATE config_snapshot_revisions SET revision=revision+1 + WHERE domain='auto_routes' AND revision=? AND revision < 9007199254740991`).bind(body.current_revision)); + const condition = revisioned ? "WHERE changes()=1" : ""; + statements.push(db.prepare(`INSERT INTO auto_routes (route_id, candidates, updated_at) + SELECT ?, ?, ? ${condition} ON CONFLICT(route_id) DO UPDATE SET + candidates=excluded.candidates, updated_at=excluded.updated_at`) + .bind(body.route_id, JSON.stringify(body.candidates), body.updated_at)); + statements.push(db.prepare(`INSERT INTO canary_traffic (route_id, route_updated_at, configuration) + SELECT ?, ?, ? ${condition} ON CONFLICT(route_id) DO UPDATE SET + route_updated_at=excluded.route_updated_at, configuration=excluded.configuration`) + .bind(body.route_id, body.updated_at, JSON.stringify(policy))); + const results = await db.batch(statements); + const confirmed = results.slice(revisioned ? 1 : 0).every(result => result?.success === true && result.meta?.changes === 1); + return confirmed ? reply({ version: 1, stored: true }) : failure("revision_conflict", 409); +} diff --git a/worker/container-env.mjs b/worker/container-env.mjs index 1a7994db..560082f1 100644 --- a/worker/container-env.mjs +++ b/worker/container-env.mjs @@ -40,6 +40,7 @@ const DIRECT_ENV_KEYS = [ "CONNECTION_PROFILES_DB_PATH", "CONFIG_REVISION_SYNC_ENABLED", "CONFIG_SYNC_TTL_SECONDS", + "CANARY_TRAFFIC_ENABLED", "GATEWAY_ALERTS_ENABLED", "GATEWAY_ALERT_WEBHOOK_ALLOWLIST", "CONFIG_SECURITY_TTL_SECONDS", diff --git a/worker/cors-policy.mjs b/worker/cors-policy.mjs index 4e5b33fb..e2652dfb 100644 --- a/worker/cors-policy.mjs +++ b/worker/cors-policy.mjs @@ -2,5 +2,5 @@ export const CORS_ALLOWED_METHODS = "GET, POST, PUT, DELETE, PATCH, OPTIONS"; export const CORS_DEFAULT_HEADERS = "X-MultiLLM-Tool-Repair, X-MultiLLM-Image-QA, X-MultiLLM-Cascade, X-MultiLLM-Conversion-Report, X-MultiLLM-Retention, X-MultiLLM-Deadline-Ms, Authorization, X-Api-Key, X-Goog-Api-Key, X-MultiLLM-Api-Key, X-Roleplay-Session-ID, Anthropic-Version, Anthropic-Beta, Anthropic-Dangerous-Direct-Browser-Access, Content-Type, Accept, Origin, X-Requested-With, OpenAI-Beta, OpenAI-Organization, OpenAI-Project, Idempotency-Key, Moderation, Moderation-Model, Redaction, X-Client-Request-ID, X-App-Name, X-Billing-Mode, X-BYOK-Provider, X-Encryption-Key, X-Encryption-Passphrase, X-Fal-Object-Lifecycle-Preference, X-PAYMENT, X-Prompt-Caching-Cut-After, X-Provider, X-Team-ID, X-Use-BYOK, x-x402"; export const CORS_EXPOSE_HEADERS = - "X-MultiLLM-Secret-Scan, X-MultiLLM-Retention, X-MultiLLM-Roleplay-Recovery, " + + "X-MultiLLM-Secret-Scan, X-MultiLLM-Canary-Cohort, X-MultiLLM-Retention, X-MultiLLM-Roleplay-Recovery, " + "X-MultiLLM-Tool-Repair, X-MultiLLM-Image-QA, X-MultiLLM-Cascade, X-MultiLLM-Conversion-Fidelity, X-MultiLLM-Conversion-Fields, Retry-After, X-Request-ID, X-MultiLLM-RateLimit-Limit, X-MultiLLM-RateLimit-Remaining, X-MultiLLM-RateLimit-Reset, X-MultiLLM-Idempotency, X-MultiLLM-Optimization, X-MultiLLM-Optimization-Mode, X-MultiLLM-Estimated-Input-Before, X-MultiLLM-Estimated-Input-After, X-MultiLLM-Image-Prompts-Compacted, X-MultiLLM-Messages-Summarized, X-MultiLLM-Optimization-Target-Met, X-MultiLLM-Summary, X-MultiLLM-Optimization-Cache-Hits, X-MultiLLM-Optimization-Cache-Misses, X-MultiLLM-Prompt-Cache, X-MultiLLM-Prompt-Cache-Mode, X-MultiLLM-Prompt-Cache-Estimated-Tokens, X-MultiLLM-Provider, X-MultiLLM-Model, X-MultiLLM-Credential-Attempts, X-MultiLLM-Route-Decision, X-MultiLLM-Circuit-State, X-MultiLLM-Auto-Route, X-MultiLLM-Auto-Selected-Model, X-MultiLLM-Auto-Attempts, X-MultiLLM-Auto-Selected-Priority, X-MultiLLM-Auto-Ordering, X-MultiLLM-Auto-Failover-Reasons, X-MultiLLM-Cache, X-MultiLLM-Cache-Backend, X-MultiLLM-Usage-Basis, X-MultiLLM-Provider-Calls, Age, X-MultiLLM-Transport-Failure, X-MultiLLM-Token-Count, X-MultiLLM-Latency-Ms, X-MultiLLM-Estimated-Cost-USD, X-MultiLLM-Cost-Basis, X-MultiLLM-Reasoning-Normalized, X-Roleplay-Session-ID, X-Roleplay-Trace-ID, X-Roleplay-Session-Source, X-Roleplay-Provider, X-Roleplay-Model, X-Roleplay-Selection, X-Roleplay-Memory, X-Roleplay-Estimated-Input-Tokens, X-Roleplay-Max-Output-Tokens, X-Roleplay-Fallback-Count, X-Roleplay-State-Cache, X-Roleplay-Credential-Check, Server-Timing, WWW-Authenticate, X-PAYMENT-RESPONSE, X-Poll-After, X-NanoGPT-Advisor-ID, X-NanoGPT-Data-Endpoint, X-NanoGPT-Direct-Endpoint, X-NanoGPT-Inline-Moderation-Cost-USD, X-NanoGPT-Inline-Moderation-Flagged, X-NanoGPT-Inline-Moderation-Model"; diff --git a/worker/d1-schema.mjs b/worker/d1-schema.mjs index 366c2e78..ac1c7711 100644 --- a/worker/d1-schema.mjs +++ b/worker/d1-schema.mjs @@ -10,7 +10,7 @@ export const REQUIRED_D1_TABLES = Object.freeze(["intelligence_policy", "intelli "usage_events", "usage_daily", "usage_batches", "media_jobs", "media_job_items", "shadow_eval_samples", "shadow_eval_results", "shadow_eval_config", "shadow_eval_policy_backups", "prompt_templates", - "config_snapshot_revisions", "config_snapshots", "config_snapshot_applications", "control_revisions", "tool_grants", "generation_cache", "usage_reservation_budgets", "usage_reservations", "usage_reservation_transitions", "session_tiers", "gateway_alert_rules", "gateway_alert_events", "managed_idempotency"]); + "config_snapshot_revisions", "config_snapshots", "config_snapshot_applications", "control_revisions", "tool_grants", "generation_cache", "usage_reservation_budgets", "usage_reservations", "usage_reservation_transitions", "session_tiers", "gateway_alert_rules", "gateway_alert_events", "managed_idempotency", "canary_traffic"]); /** * Readiness for the D1 schema. A deploy that skipped `wrangler d1 migrations apply` reports