diff --git a/app.py b/app.py index 58860680..48ab3195 100644 --- a/app.py +++ b/app.py @@ -38,6 +38,7 @@ from services.cache_service import CacheService from services.image_relay_catalog import ImageRelayCatalogRefresh from services.provider_catalog_refresh import ProviderCatalogAutoRefresh +from services.gateway_extensions import register_gateway_extensions from services.metrics_service import MetricsService from services.proxy_service import ProxyService from services import usage_ledger @@ -146,6 +147,7 @@ def create_app() -> Flask: register_gateway_mcp_routes(app, csrf) register_documentation_routes(app, AuthService, ProxyService) register_usage_routes(app, csrf) + register_gateway_extensions(app) # Restore the dashboard's recent requests from the durable ledger in the background. usage_ledger.start(MetricsService.get_instance()) diff --git a/docs/config-revision-sync.md b/docs/config-revision-sync.md new file mode 100644 index 00000000..0b11c08d --- /dev/null +++ b/docs/config-revision-sync.md @@ -0,0 +1,116 @@ +# Revision-aware live configuration sync + +Configuration revisions are committed counters, not a push consistency guarantee. The +feature is off by default. Disabled startup does not register a poller or a freshness +guard, and existing reads, saves and upstream catalog refresh timing stay unchanged. +No disabled operation reads or writes `control_revisions`. + +```text +CONFIG_REVISION_SYNC_ENABLED=false +``` + +Apply migration `0017_control_revisions.sql` through the separately approved migration +process, after the config snapshot and usage bucket migrations (`0015` and `0016`). A +code deploy does not apply D1 migrations. Enable on the Worker and its Containers: + +```text +CONFIG_REVISION_SYNC_ENABLED=true +CONFIG_SYNC_TTL_SECONDS=30 +CONFIG_SECURITY_TTL_SECONDS=5 +INTELLIGENCE_STORAGE_BACKEND=d1 +AUTH_STORAGE_BACKEND=d1 +``` + +Flask sync requires Container D1 control and account storage. Standalone SQL/SQLite +mode and unsupported storage combinations disable the feature and log one warning, +without changing existing authentication. This does not fall back from an unavailable +D1 authority: configured D1 with a missing migration or an outage still fails closed. +Storage selection is a startup setting; restart Containers after changing it. + +Empty values mean defaults. Malformed settings disable sync and log the variable name +once, without its value. Ordinary TTL accepts 1–3600 seconds. Security TTL accepts +1–5 seconds; a longer value cannot extend the security safety bound. + +The existing control-state scheduler polls revisions with an interval between 80% and +100% of the TTL. Each consumer permits one poll at a time and failures keep the same +bounded schedule. A revision check costs one private D1 call per due group. Changed +domains additionally load their stored copy and verify the revision again. Revisions +that decrease, failed loads, and writes racing a load cannot certify freshness. A copy +loads at startup and thereafter only when a newer committed revision is observed. +Scheduler delay and authority latency can cause a temporary fail-closed response. + +`GET /admin/config/revisions` requires a dashboard administrator when enabled and is +404 when disabled. It returns only domain names, revision numbers, age, stale flags, +security classification and the time until the next poll. It contains no keys, model +lists, prompts, responses or provider payloads. Ordinary auto routes and provider +catalogs retain the last installed copy during an outage and report `stale: true`. +Before the first route load, seeded routes remain available. New processes cannot +recover an ordinary last-good copy while its authority is unavailable. + +API and MCP dispatch fails with HTTP 503 and `config_security_stale` while security +metadata has never been verified, a newer security revision cannot be installed, or +the last verification is older than the security TTL. Admin recovery and status remain +accessible; preflight does not dispatch. Enabled revision operations with a missing +table return a content-free JSON 503. There is no automatic policy change, inference +retry, or success response substituted for a failure. + +## Committed domains + +`auto_routes` retains the config snapshots counter in `config_snapshot_revisions`; it +has no second row in `control_revisions`. The private auto-route wrapper reuses atomic +snapshot saves for normal writes without enabling snapshot APIs. Snapshot apply +retains CAS and bumps exactly once. Model overrides and provider catalog writes update +their domain counter in the same D1 transaction as their data. + +Account upserts and deletes atomically increment both `key_controls` and `model_grants` +with the account mutation and its audit record. Upserts replace the full account, +covering revocation, rotation, scopes, model allowlists, budgets, expiry and address +ranges; both counters advance conservatively even for an unchanged replacement or a +missing-account delete. Usage-only touches and refused account writes do not advance +security revisions. With the feature off, existing writes keep their original SQL and +responses, including compatibility reads for older account schemas. + +Flask installs strict account refreshers for both security domains by default. Each +reads all account pages from the private D1 authority using the full control schema, +validates the records, clears the verified-key memo, and installs the account and grant +copy. Invalid records or incomplete schemas raise; there is no fallback to missing +controls or cached grants. Authentication uses this installed copy, serialized with +refreshes, so a revoked, deleted or rotated key is rejected after the successful poll +without waiting for the existing 60-second memo TTL. Model grants and scopes come from +the same installed records. Budgets, model allowlists, expiry and address restrictions +remain enforced by their existing request policies. Environment bootstrap admin and +integration-key authentication retain their existing authorities; integration keys are +verified against their private authority on each request. + +Both domains currently load the shared account table separately. Each refresh costs +one private call per page of up to 200 accounts plus the group's revision confirmation; +new revisions clear all verified-key memos in that process. Account copies remain in +process memory only. The revision table holds one counter and timestamp per domain, +with no event history or request content. Existing account audit, snapshot history and +catalog retention policies remain unchanged. + +## Runtime interfaces + +`register_gateway_extensions` mounts the Flask guard and status route. Other middleware +can be supplied as explicit callbacks in policy order; there are no dynamic imports or +plugin discovery. `configure_sync` supplies the model-override and both account/grant +installers automatically. An explicit `security_refreshers` override must load from +its committed authority or raise; an absent installer remains fail closed. + +Private auto-route dispatch can call +`handleRevisionedAutoRoutes(request, env, {handleAutoRoutesRequest, boundedBody})` using +its static handler and bounded body reader. Revision persistence has no dependency on +the account or auto-route handler. `commitRevision(db, ["key_controls", "model_grants"], +statements)` accepts trusted fixed D1 statements, never SQL from a request. Its optional +expected revision permits one concurrent CAS winner; a failed CAS rolls back all data, +audit and revision statements. + +Native Worker dispatch can supply strict installers to `RevisionConsumer`, invoke +`tick` through its background lifecycle, and call `requireFreshSecurity` before dispatch. +The guard returns a JSON 503 response or null when freshness permits normal dispatch. +Read callbacks return an exact domain-to-integer mapping; installer callbacks throw on +authority failure. The native consumer has no default account/grant installer: a missing +callback is its fail-closed fallback, equivalent to Flask's `missing_security_refresh` +for an explicitly absent installer. Native status can use the content-free `status` +method. Polling does not guarantee immediate cross-region visibility or live deployment +health. diff --git a/intelligence-migrations/0017_control_revisions.sql b/intelligence-migrations/0017_control_revisions.sql new file mode 100644 index 00000000..991167f4 --- /dev/null +++ b/intelligence-migrations/0017_control_revisions.sql @@ -0,0 +1,6 @@ +-- Content-free counters. auto_routes retains its W13 counter for snapshot CAS. +CREATE TABLE IF NOT EXISTS control_revisions ( + domain TEXT PRIMARY KEY CHECK(domain IN ('model_overrides', 'provider_catalog', 'key_controls', 'model_grants')), + revision INTEGER NOT NULL CONSTRAINT revision_cas_confirmed CHECK(revision >= 0 AND revision <= 9007199254740991), + updated_at TEXT NOT NULL +); diff --git a/package.json b/package.json index 4b42830a..951c0843 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "lint": "ruff check . --select E9,F63,F7,F82", "typecheck": "mypy --ignore-missing-imports --follow-imports=skip --disable-error-code import-untyped --disable-error-code var-annotated --disable-error-code arg-type --disable-error-code assignment --disable-error-code operator app.py routes services providers streaming", "test": "python -m pytest -q", - "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_prompt_cache_cost_worker.mjs", + "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_prompt_cache_cost_worker.mjs", "db:migrate": "python scripts/validate_sqlite_schema.py --apply", "db:validate": "python scripts/validate_sqlite_schema.py", "security:secrets": "python scripts/check_static_secrets.py", @@ -18,7 +18,7 @@ "cf:dry-run": "wrangler deploy --dry-run", "cf:tail": "wrangler tail", "test:knowledge": "node --test tests/test_knowledge_*.mjs", - "test:integration": "node --test tests/test_d1_integration.mjs", + "test:integration": "node --test tests/test_d1_integration.mjs tests/test_config_revision_sync_integration.mjs", "cf:knowledge:dry-run": "wrangler deploy --config wrangler.knowledge.jsonc --dry-run", "cf:knowledge:deploy": "wrangler deploy --config wrangler.knowledge.jsonc", "cf:migrations:check": "node scripts/verify_d1_migrations.mjs --check", diff --git a/services/auth_service.py b/services/auth_service.py index 5a76f70f..98bc3f46 100644 --- a/services/auth_service.py +++ b/services/auth_service.py @@ -983,6 +983,10 @@ def verify_api_key(cls, api_key: Optional[str], remote_addr: Optional[str] = Non if api_key.startswith(KEY_NAMESPACE): return verify_integration_key(api_key) + revisioned = key_controls.verify_revisioned_key(cls, api_key, remote_addr) + if revisioned is not key_controls.LEGACY_AUTH: + return revisioned + remembered = cls._remembered_username(api_key) user = cls._users.get(remembered) if remembered else None if remembered and user and not user.get("revoked_at"): diff --git a/services/auto_route_service.py b/services/auto_route_service.py index 09049f6a..ac9fdfea 100644 --- a/services/auto_route_service.py +++ b/services/auto_route_service.py @@ -6,7 +6,7 @@ from typing import Iterable, Mapping from providers.registry import get_registry -from services import auto_route_d1 +from services import auto_route_d1, config_revision_sync from services.model_registry import ModelRegistry from services.sqlite_store import connect, storage_path @@ -404,7 +404,9 @@ def _rows_to_routes(cls, rows: Iterable[sqlite3.Row]) -> list[AutoRoute]: @classmethod def _durable_routes(cls) -> list[AutoRoute]: """Seeded defaults overlaid by the routes operators saved in D1.""" - stored = auto_route_d1.stored_routes() + stored = config_revision_sync.route_copy() + if stored is None: + stored = auto_route_d1.stored_routes() routes = {route_id: AutoRoute(route_id, candidates, "") for route_id, candidates in DEFAULT_AUTO_ROUTES.items()} for route_id, (candidates, updated_at) in stored.items(): # A route still holding a retired default follows the current default. diff --git a/services/config_revision_sync.py b/services/config_revision_sync.py new file mode 100644 index 00000000..d88caed6 --- /dev/null +++ b/services/config_revision_sync.py @@ -0,0 +1,249 @@ +"""Bounded revision polling on the existing control-state background scheduler.""" +from __future__ import annotations + +import logging +import math +import os +import random +import re +import threading +import time +from dataclasses import dataclass +from collections.abc import Callable, Mapping +from functools import partial + +from services import auto_route_d1, control_state_d1, key_controls, model_override_d1 +from services.auth_service import AuthService + +logger = logging.getLogger(__name__) +_warned: set[str] = set() +_active: RevisionSync | None = None +_active_lock = threading.Lock() +SECURITY_DOMAINS = ("model_overrides", "key_controls", "model_grants") +ORDINARY_DOMAINS = ("auto_routes", "provider_catalog") + + +@dataclass(frozen=True) +class SyncSettings: + enabled: bool = False + ttl: float = 30 + security_ttl: float = 5 + + +def _invalid(name): + if name not in _warned: + _warned.add(name) + logger.warning("Invalid %s; configuration revision sync disabled", name) + + +def load_settings(env: Mapping[str, str] | None = None) -> SyncSettings: + env = os.environ if env is None else env + flag = env.get("CONFIG_REVISION_SYNC_ENABLED", "").strip().lower() + if flag not in {"", "0", "false", "no", "off", "1", "true", "yes", "on"}: + _invalid("CONFIG_REVISION_SYNC_ENABLED") + return SyncSettings() + if flag not in {"1", "true", "yes", "on"}: + return SyncSettings() + values = [] + for name, default, maximum in (("CONFIG_SYNC_TTL_SECONDS", 30, 3600), ("CONFIG_SECURITY_TTL_SECONDS", 5, 5)): + text = env.get(name, "").strip() + if not text: + values.append(default) + continue + if not re.fullmatch(r"(?:\d+(?:\.\d*)?|\.\d+)", text): + _invalid(name) + return SyncSettings() + value = float(text) + if not math.isfinite(value) or not 1 <= value <= maximum: + _invalid(name) + return SyncSettings() + values.append(value) + return SyncSettings(True, *values) + + +@dataclass +class DomainState: + refresh: Callable[[int], None] + security: bool + revision: int | None = None + observed: int | None = None + checked: float | None = None + due: float = 0 + + +class RevisionSync: + """Refresh a loaded copy only on newer revisions; never renew failed security reads.""" + + def __init__(self, settings: SyncSettings, *, fetch=None, clock=time.monotonic, jitter=random.random): + self.settings = settings + self.fetch = fetch or fetch_revisions + self.clock, self.jitter = clock, jitter + self._domains: dict[str, DomainState] = {} + self._lock = threading.RLock() + self._poll_lock = threading.Lock() + self.routes: dict = {} + + def register(self, domain: str, refresh: Callable[[int], None], *, security=False): + with self._lock: + self._domains[domain] = DomainState(refresh, security) + + def _ttl(self, state): + return self.settings.security_ttl if state.security else self.settings.ttl + + @staticmethod + def _validate(revisions, domains): + if not isinstance(revisions, dict) or set(revisions) != set(domains) or any( + type(value) is not int or not 0 <= value <= 9007199254740991 for value in revisions.values() + ): + raise ValueError("Invalid revision metadata") + + def tick(self, final=False): + if not self.settings.enabled or final or not self._poll_lock.acquire(blocking=False): + return + try: + for security in (True, False): + self._poll_group(security) + finally: + self._poll_lock.release() + + def _poll_group(self, security): + started = self.clock() + with self._lock: + due = {name: state for name, state in self._domains.items() if state.security == security and started >= state.due} + for state in due.values(): + # Negative jitter keeps freshness within TTL and a failed poll cannot hot-loop. + state.due = started + self._ttl(state) * (0.8 + 0.2 * min(1, max(0, self.jitter()))) + if not due: + return + try: + revisions = self.fetch(list(due)) + self._validate(revisions, due) + except Exception: + logger.warning("Configuration revision metadata unavailable") + return + changed = {} + for name, state in due.items(): + revision = revisions[name] + with self._lock: + if state.observed is not None and revision < state.observed: + state.checked = None + continue + state.observed = revision + if state.revision == revision: + state.checked = started + continue + # Observing a new security revision immediately invalidates the older copy. + state.checked = None + try: + state.refresh(revision) + changed[name] = revision + except Exception: + logger.warning("Configuration domain refresh unavailable domain=%s", name) + if not changed: + return + try: + confirmed = self.fetch(list(changed)) + self._validate(confirmed, changed) + except Exception: + logger.warning("Configuration revision confirmation unavailable") + return + with self._lock: + for name, revision in changed.items(): + due[name].observed = max(due[name].observed or 0, confirmed[name]) + if confirmed[name] == revision: + due[name].revision, due[name].checked = revision, started + # Concurrent writes are retried at the next bounded poll, never in a loop. + + def security_ready(self): + if not self.settings.enabled: + return True + now = self.clock() + with self._lock: + security = [state for state in self._domains.values() if state.security] + return bool(security) and all(state.checked is not None and now - state.checked <= self.settings.security_ttl for state in security) + + def status(self): + now = self.clock() + with self._lock: + return {"enabled": self.settings.enabled, "domains": {name: { + "revision": state.revision, + "age_seconds": None if state.checked is None else max(0, now - state.checked), + "stale": state.checked is None or now - state.checked > self._ttl(state), + "security": state.security, + "next_poll_seconds": max(0, state.due - now), + } for name, state in self._domains.items()}} + + +def fetch_revisions(domains): + if not control_state_d1.using_d1(): + raise ValueError("Revision authority requires D1") + response = control_state_d1.call("model_overrides", "revisions", domains=domains) + if not isinstance(response, dict) or set(response) != {"version", "revisions"} or response["version"] != 1: + raise ValueError("Invalid revision metadata envelope") + return response["revisions"] + + +def _refresh_routes(sync, revision): + routes = auto_route_d1._routes(control_state_d1.call("auto_routes", "list")) + with sync._lock: + sync.routes = routes + + +def _refresh_models(revision): + overrides = model_override_d1._overrides(control_state_d1.call("model_overrides", "list")) + with model_override_d1._lock: + model_override_d1._cache.update(overrides=overrides, expires=float("inf")) + + +def missing_security_refresh(revision): + raise ValueError("Security revision collaborator is not configured") + + +def supported_settings(settings): + """Revision authority and accounts must both live in Container D1 storage.""" + if settings.enabled and (not control_state_d1.using_d1() + or os.environ.get("AUTH_STORAGE_BACKEND", "").strip().lower() != "d1"): + name = "CONFIG_REVISION_SYNC_RUNTIME" + if name not in _warned: + _warned.add(name) + logger.warning("Configuration revision sync requires Container D1 control and account storage; disabled") + return SyncSettings() + return settings + + +def configure_sync(settings, *, security_refreshers=None, catalog_refresh=None): + """Security authorities must install a verified copy or raise, never a fallback copy.""" + global _active + settings = supported_settings(settings) + if not settings.enabled: + return RevisionSync(settings) + with _active_lock: + if _active is None: + _active = RevisionSync(settings) + _active.register("auto_routes", partial(_refresh_routes, _active)) + if catalog_refresh is not None: + _active.register("provider_catalog", catalog_refresh) + callbacks = {"model_overrides": _refresh_models, + "key_controls": partial(key_controls.refresh_security_copy, AuthService), + "model_grants": partial(key_controls.refresh_security_copy, AuthService), + **(security_refreshers or {})} + for name in SECURITY_DOMAINS: + _active.register(name, callbacks.get(name) or missing_security_refresh, security=True) + control_state_d1.register(_active.tick) + sync = _active + control_state_d1.ensure_running() + return sync + + +def route_copy(): + if not load_settings().enabled or _active is None or not _active.settings.enabled: + return None + with _active._lock: + return dict(_active.routes) + + +def request_poll(app): + """Reads wake the existing scheduler; they never poll storage on a request thread.""" + sync = app.extensions.get("config_revision_sync") + if sync is not None and sync.settings.enabled: + control_state_d1.wake() diff --git a/services/gateway_extensions.py b/services/gateway_extensions.py new file mode 100644 index 00000000..6b94ef43 --- /dev/null +++ b/services/gateway_extensions.py @@ -0,0 +1,43 @@ +"""Static Flask extension registration; callbacks are supplied explicitly in policy order.""" +from flask import jsonify, request + +from route_helpers import is_api_request_path, login_required +from services.auth_service import AuthService +from services.config_revision_sync import configure_sync, load_settings, supported_settings +from services.provider_catalog_refresh import refresh_provider_catalog_revision + + +def register_gateway_extensions(app, *, callbacks=(), revision_sync=None, security_refreshers=None): + """Mount revision sync and explicitly supplied middleware without peer imports.""" + settings = revision_sync.settings if revision_sync is not None else supported_settings(load_settings()) + if settings.enabled: + sync = revision_sync or configure_sync(settings, security_refreshers=security_refreshers, + catalog_refresh=refresh_provider_catalog_revision) + app.extensions["config_revision_sync"] = sync + + @app.before_request + def require_config_security_freshness(): + # Cover every API entry, including forwarded/native protocols and MCP. Recovery + # and admin status remain reachable while the authority is unavailable. + protected = (is_api_request_path(request.path) and request.path.rstrip("/") not in {"/health", "/healthz"} + or request.endpoint == "proxy" or request.path in {"/mcp", "/mcp/", "/googleai/chat/completions", + "/api/backends/chat-completions/generate"}) + if request.method != "OPTIONS" and protected and not sync.security_ready(): + return jsonify({"error": {"code": "config_security_stale", + "message": "Security configuration freshness could not be verified"}}), 503, {"Cache-Control": "no-store"} + return None + + @login_required + def admin_status(): + if not (AuthService.get_current_user() or {}).get("is_admin"): + return jsonify({"error": "admin_required"}), 403 + return jsonify(app.extensions["config_revision_sync"].status()), 200, {"Cache-Control": "no-store"} + + @app.get("/admin/config/revisions") + def config_revision_status(): + if not settings.enabled: + return jsonify({"error": "not_found"}), 404 + return admin_status() + + for callback in callbacks: + callback(app) diff --git a/services/key_controls.py b/services/key_controls.py index 70026c22..bd8d3276 100644 --- a/services/key_controls.py +++ b/services/key_controls.py @@ -10,10 +10,16 @@ import ipaddress import os import re +import threading from datetime import datetime, timezone from typing import Any, Mapping, Optional +from flask import current_app, has_app_context +from werkzeug.security import check_password_hash + from error_handlers import APIError +from services import user_store +from services.auth_primitives import build_api_key_prefix, normalized_username CONTROL_FIELDS = ("daily_budget_usd", "monthly_budget_usd", "allowed_models", "allowed_ips", "expires_at", "secret_scan_mode", "shadow_eval_rate") MAX_BUDGET_USD = 1_000_000_000 @@ -268,3 +274,88 @@ def client_ip(request) -> Optional[str]: except ValueError: pass return request.remote_addr + + +LEGACY_AUTH = object() +_security_lock = threading.RLock() +_security_copies: dict[Any, dict[str, dict[str, Any]]] = {} + + +def _security_user(auth, row): + """Reject malformed authority records before permissive legacy decoding.""" + if not isinstance(row, dict) or set(row) != set(user_store.USER_FIELDS): + raise ValueError("Incomplete security account") + user_store._row(row) + if normalized_username(row["username"]) != row["username"]: + raise ValueError("Invalid security account identity") + if not row["api_key_hash"] or not row["api_key_prefix"] or not re.fullmatch(r"[A-Za-z0-9:_.,-]{0,512}", row["scopes"]): + raise ValueError("Invalid security account metadata") + for name in ("created_at", "last_login", "last_used_at", "rotated_at", "revoked_at"): + if row[name] is not None and _parse_time(row[name]) is None: + raise ValueError("Invalid security account timestamp") + if row["created_at"] is None: + raise ValueError("Missing security account timestamp") + models = row["allowed_models"] + if models is not None and (not models or not all(MODEL_PATTERN.fullmatch(item) for item in models.split(","))): + raise ValueError("Invalid security model grants") + controls = validate({name: row[name] for name in CONTROL_FIELDS}) + if any(row[name] is not None and controls[name] is None for name in CONTROL_FIELDS): + raise ValueError("Invalid security account controls") + return auth._row_to_user(row) + + +def refresh_security_copy(auth, revision): + """Install the shared account/grant authority, invalidating all verified-key memos. + + The poller confirms the committed counters after loading every page; a write racing + this copy cannot certify either security domain. No local or legacy-column fallback. + """ + if not user_store.using_d1(): + raise ValueError("Security account authority requires D1") + users, after = {}, None + while True: + response = user_store._call("list", after=after, limit=user_store.PAGE_SIZE) + if not isinstance(response, dict) or set(response) != {"version", "users"} or type(response["version"]) is not int or response["version"] != 1: + raise ValueError("Invalid security account envelope") + page = response["users"] + if not isinstance(page, list) or len(page) > user_store.PAGE_SIZE: + raise ValueError("Invalid security account page") + for row in page: + user = _security_user(auth, row) + name = user["username"] + if name in users or after is not None and name <= after: + raise ValueError("Invalid security account pagination") + users[name] = user + after = name + if len(page) < user_store.PAGE_SIZE: + break + with _security_lock: + auth._forget_verified_keys() + _security_copies[auth] = users + auth._users = dict(users) + auth._rebuild_api_key_prefix_index() + + +def verify_revisioned_key(auth, api_key, remote_addr): + """Verify against the installed account/grant copy, serialized with refreshes.""" + sync = current_app.extensions.get("config_revision_sync") if has_app_context() else None + if sync is None or not sync.settings.enabled: + return LEGACY_AUTH + with _security_lock: + if not sync.security_ready() or auth not in _security_copies: + raise APIError("Security configuration freshness could not be verified", 503, + {"error": {"code": "config_security_stale"}}) + users = _security_copies[auth] + remembered = auth._remembered_username(api_key) + user = users.get(remembered) if remembered else None + if not user or user.get("revoked_at"): + prefix = build_api_key_prefix(api_key) + user = next((row for row in users.values() if row["api_key_prefix"] == prefix + and not row.get("revoked_at") and check_password_hash(row["api_key_hash"], api_key)), None) + if user is None: + return None + auth._remember_key(api_key, user["username"]) + username = user["username"] + auth._users[username] = user + auth._update_key_usage(username, remote_addr) + return auth._public_user(username, user) diff --git a/services/provider_catalog_refresh.py b/services/provider_catalog_refresh.py index 8e3e7512..1ab9581e 100644 --- a/services/provider_catalog_refresh.py +++ b/services/provider_catalog_refresh.py @@ -128,6 +128,9 @@ def _run(self, app, targets, auth_service_cls, proxy_service_cls) -> None: def start_provider_catalog_refresh(app, auth_service_cls, proxy_service_cls): """Start the background provider catalog refresh when it is enabled.""" + from services.config_revision_sync import request_poll + + request_poll(app) refresher = app.extensions.get("provider_catalog_refresh") if refresher is None or not app.config.get("PROVIDER_CATALOG_AUTO_REFRESH", True): return None @@ -141,3 +144,20 @@ def refresh_model_catalogs(app, auth_service_cls, proxy_service_cls) -> None: refresher = app.extensions.get("provider_catalog_refresh") if refresher is not None: refresher.wait_for_first_refresh() + + +def refresh_provider_catalog_revision(revision): + """Load the durable catalog strictly; an outage leaves the last installed copy intact.""" + from services import control_state_d1, provider_catalog_d1 + + fetched = {} + listing = provider_catalog_d1._listing(control_state_d1.call(provider_catalog_d1.ENDPOINT, "list")) + for provider in listing: + snapshot = provider_catalog_d1._snapshot( + control_state_d1.call(provider_catalog_d1.ENDPOINT, "get", provider=provider), provider) + if snapshot is None: + raise ValueError("Catalog snapshot disappeared during refresh") + fetched[provider] = snapshot + with provider_catalog_d1._refresh_lock, provider_catalog_d1._lock: + provider_catalog_d1._state.update(snapshots=fetched, loaded=True, expires=float("inf"), + version=provider_catalog_d1._state["version"] + 1) diff --git a/tests/fixtures/secret_firewall_coverage.json b/tests/fixtures/secret_firewall_coverage.json index 1116edc7..fbf36968 100644 --- a/tests/fixtures/secret_firewall_coverage.json +++ b/tests/fixtures/secret_firewall_coverage.json @@ -222,6 +222,7 @@ "worker/build-id.mjs", "worker/cascades-d1.mjs", "worker/client-headers.mjs", + "worker/config-revision.mjs", "worker/container-env.mjs", "worker/control-audit-d1.mjs", "worker/control-state-d1.mjs", diff --git a/tests/support/config_revision_fixtures.mjs b/tests/support/config_revision_fixtures.mjs new file mode 100644 index 00000000..06c1af8b --- /dev/null +++ b/tests/support/config_revision_fixtures.mjs @@ -0,0 +1,40 @@ +// Shared D1 fixtures for the config revision Worker and integration suites. +import { convertV4MiniflareOptions, Miniflare } from "miniflare"; +import { handleIntelligenceOutbound } from "../../worker/intelligence-outbound.mjs"; +import { handleRevisionedAutoRoutes } from "../../worker/config-revision.mjs"; +import { handleAutoRoutesRequest } from "../../worker/auto-routes-d1.mjs"; +import { boundedBody } from "../../worker/control-users-d1.mjs"; +import { applyMigrations } from "../d1_migrations.mjs"; + +export async function database(t, missing = false) { + const mf = new Miniflare(convertV4MiniflareOptions({ modules: true, + script: "export default {fetch(){return new Response('ok')}}", d1Databases: ["INTELLIGENCE_DB"] })); + t.after(() => mf.dispose()); + const db = await mf.getD1Database("INTELLIGENCE_DB"); + await applyMigrations(db, { skip: missing ? ["0017_control_revisions.sql"] : [] }); + const env = { INTELLIGENCE_DB: db, CONFIG_REVISION_SYNC_ENABLED: "true" }; + const call = async (domain, body, targetEnv = env) => { + const request = new Request(`http://intelligence.internal${domain === "routes" ? "/v1/auto-routes" : `/v1/state/${domain}`}`, { + method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ version: 1, ...body }) }); + const response = domain === "routes" ? await handleRevisionedAutoRoutes(request, targetEnv, { handleAutoRoutesRequest, boundedBody }) + : await handleIntelligenceOutbound(request, targetEnv); + return { status: response.status, body: await response.json() }; + }; + const revisions = async domains => call("models", { operation: "revisions", domains }); + return { db, env, call, revisions }; +} + +export const account = (changes = {}) => ({ username: "alice", api_key_hash: "synthetic-hash", api_key_prefix: "mllm_syntheti", + scopes: "chat,models", is_admin: 0, created_at: "2026-10-09T00:00:00+00:00", last_login: null, + last_used_at: null, last_used_ip: null, created_by: null, rotated_at: null, revoked_at: null, + daily_budget_usd: null, monthly_budget_usd: null, allowed_models: null, allowed_ips: null, + expires_at: null, secret_scan_mode: null, shadow_eval_rate: null, ...changes }); + +const usersRequest = body => new Request("http://intelligence.internal/v1/users", { + method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ version: 1, ...body }), +}); + +export async function usersCall(env, body) { + const response = await handleIntelligenceOutbound(usersRequest(body), env); + return { status: response.status, body: await response.json() }; +} diff --git a/tests/test_config_revision_sync.py b/tests/test_config_revision_sync.py new file mode 100644 index 00000000..0971a4fd --- /dev/null +++ b/tests/test_config_revision_sync.py @@ -0,0 +1,521 @@ +"""Revision polling never certifies configuration that was not loaded.""" +import threading +import time +import sqlite3 +from pathlib import Path +from unittest.mock import Mock, patch + +import pytest +from flask import Flask + +from services.config_revision_sync import RevisionSync, SyncSettings, load_settings +from services.gateway_extensions import register_gateway_extensions + + +def harness(*, security=False): + now = [100.0] + revisions = {"models": 0} + loads = [] + def fetch(domains): + return {name: revisions[name] for name in domains} + sync = RevisionSync(SyncSettings(True, 30, 5), fetch=fetch, + clock=lambda: now[0], jitter=lambda: 0.5) + sync.register("models", lambda revision: loads.append(revision), security=security) + return sync, now, revisions, loads + + +@pytest.mark.parametrize("env", [{}, {"CONFIG_REVISION_SYNC_ENABLED": ""}, + {"CONFIG_REVISION_SYNC_ENABLED": "invalid"}, + {"CONFIG_REVISION_SYNC_ENABLED": "true", "CONFIG_SYNC_TTL_SECONDS": "nan"}, + {"CONFIG_REVISION_SYNC_ENABLED": "true", "CONFIG_SECURITY_TTL_SECONDS": "0"}, + {"CONFIG_REVISION_SYNC_ENABLED": "true", "CONFIG_SECURITY_TTL_SECONDS": "6"}]) +def test_disabled_or_malformed_settings_are_inert(env): + settings = load_settings(env) + assert not settings.enabled + fetch = Mock(side_effect=AssertionError("No storage")) + sync = RevisionSync(settings, fetch=fetch) + sync.register("models", Mock(), security=True) + sync.tick() + assert sync.security_ready() + fetch.assert_not_called() + + +def test_empty_ttls_use_defaults(): + assert load_settings({"CONFIG_REVISION_SYNC_ENABLED": "true", + "CONFIG_SYNC_TTL_SECONDS": "", "CONFIG_SECURITY_TTL_SECONDS": ""}) == SyncSettings(True, 30, 5) + + +def test_two_instances_only_reload_newer_committed_revision(): + first, now, revisions, loads = harness() + second_loads = [] + second = RevisionSync(first.settings, fetch=first.fetch, clock=lambda: now[0], jitter=lambda: 0) + second.register("models", second_loads.append) + first.tick(); second.tick() + assert loads == second_loads == [0] + now[0] += 31 + first.tick(); second.tick() + assert loads == second_loads == [0] + revisions["models"] = 2 + now[0] += 31 + first.tick(); second.tick() + assert loads == second_loads == [0, 2] + assert first.status()["domains"]["models"]["revision"] == 2 + + +def test_outage_marks_ordinary_stale_and_security_fails_closed(): + ordinary, now, _, _ = harness() + security, security_now, _, _ = harness(security=True) + assert not security.security_ready() + ordinary.tick(); security.tick() + assert security.security_ready() + ordinary.fetch = security.fetch = Mock(side_effect=RuntimeError("private detail")) + now[0] += 31; security_now[0] += 5.01 + ordinary.tick(); security.tick() + assert ordinary.status()["domains"]["models"]["stale"] + assert not security.security_ready() + assert "private detail" not in str(security.status()) + + +def test_failed_reload_and_revision_rollback_do_not_renew_security(): + sync, now, revisions, _ = harness(security=True) + sync.tick() + revisions["models"] = 1 + sync.register("models", Mock(side_effect=RuntimeError("offline")), security=True) + now[0] += 5 + sync.tick() + assert not sync.security_ready() + assert sync.status()["domains"]["models"]["revision"] is None + sync.register("models", lambda revision: None, security=True) + sync.tick() + now[0] += 5 + sync.tick() + assert sync.security_ready() + revisions["models"] = 0 + now[0] += 5 + sync.tick() + assert not sync.security_ready() + assert sync.status()["domains"]["models"]["revision"] == 1 + + +def test_write_during_reload_never_certifies_mixed_revision(): + sync, now, revisions, _ = harness(security=True) + sync.register("models", lambda revision: revisions.update(models=revision + 1), security=True) + sync.tick() + assert not sync.security_ready() + + +def test_inflight_poll_and_failures_cannot_create_refresh_storm(): + sync, now, _, _ = harness(security=True) + entered, release = threading.Event(), threading.Event() + def fetch(domains): + entered.set() + assert release.wait(2) + return {name: 0 for name in domains} + sync.fetch = Mock(side_effect=fetch) + thread = threading.Thread(target=sync.tick) + thread.start() + assert entered.wait(2) + for _ in range(100): + sync.tick() + release.set(); thread.join(2) + assert not thread.is_alive() + assert sync.fetch.call_count == 2 # read, load, verification read + sync.fetch = Mock(side_effect=RuntimeError("offline")) + now[0] += 5 + for _ in range(100): + sync.tick() + assert sync.fetch.call_count == 1 + + +@pytest.mark.parametrize("jitter", [0, 1]) +def test_jitter_poll_bound_never_exceeds_security_ttl(jitter): + sync, now, _, _ = harness(security=True) + sync.jitter = lambda: jitter + sync.tick() + due = sync.status()["domains"]["models"]["next_poll_seconds"] + assert 4 <= due <= 5 + + +def test_static_registrar_preserves_order_and_disabled_requests(): + app = Flask(__name__) + app.config["TESTING"] = True + calls = [] + with patch("services.gateway_extensions.load_settings", return_value=SyncSettings()): + register_gateway_extensions(app, callbacks=(lambda app: calls.append("retention"), + lambda app: calls.append("admission"))) + @app.get("/v1/probe") + def probe(): + return "unchanged", 200, {"X-Existing": "yes"} + response = app.test_client().get("/v1/probe") + assert response.data == b"unchanged" + assert response.headers["X-Existing"] == "yes" + assert calls == ["retention", "admission"] + assert app.test_client().get("/admin/config/revisions").status_code == 404 + + +def test_registered_security_guard_and_admin_freshness(): + app = Flask(__name__) + app.config["TESTING"] = True + sync, now, _, _ = harness(security=True) + register_gateway_extensions(app, revision_sync=sync) + @app.post("/v1/chat/completions") + def dispatch(): + return {"dispatched": True} + client = app.test_client() + assert client.post("/v1/chat/completions").status_code == 503 + sync.tick() + assert client.post("/v1/chat/completions").status_code == 200 + now[0] += 5.01 + assert client.post("/v1/chat/completions").status_code == 503 + with patch("route_helpers.AuthService.is_authenticated", return_value=True), \ + patch("services.gateway_extensions.AuthService.get_current_user", return_value={"is_admin": True}): + result = client.get("/admin/config/revisions") + assert result.status_code == 200 + assert result.json["domains"]["models"]["stale"] + with patch("route_helpers.AuthService.is_authenticated", return_value=True), \ + patch("services.gateway_extensions.AuthService.get_current_user", return_value={"is_admin": False}): + assert client.get("/admin/config/revisions").status_code == 403 + + +def test_owned_route_reader_uses_revision_cache_only_when_enabled(monkeypatch): + from services import auto_route_d1, config_revision_sync + from services.auto_route_service import AutoRouteService + monkeypatch.setattr(auto_route_d1, "using_d1", lambda: True) + with patch.object(config_revision_sync, "route_copy", return_value={"auto:test": (("openai:new",), "time")}), \ + patch.object(auto_route_d1, "stored_routes", side_effect=AssertionError("Legacy read")): + assert AutoRouteService.get_route("auto:test").candidates == ("openai:new",) + + +def test_enabled_unverified_authority_stays_closed(monkeypatch): + monkeypatch.setenv("INTELLIGENCE_STORAGE_BACKEND", "d1") + monkeypatch.setenv("AUTH_STORAGE_BACKEND", "d1") + app = Flask(__name__) + with patch("services.gateway_extensions.load_settings", return_value=SyncSettings(True)), \ + patch("services.config_revision_sync.control_state_d1.register"), \ + patch("services.config_revision_sync.control_state_d1.ensure_running"): + register_gateway_extensions(app) + assert not app.extensions["config_revision_sync"].security_ready() + + +def test_factory_mounts_single_static_registrar(monkeypatch, tmp_path): + import importlib + for name in ("FLASK_SECRET_KEY", "JWT_SECRET", "ADMIN_API_KEY"): + monkeypatch.setenv(name, "synthetic-revision-test") + for name in ("AUTH_DB_PATH", "MODEL_REGISTRY_DB_PATH", "RATE_LIMIT_DB_PATH"): + monkeypatch.setenv(name, str(tmp_path / (name + ".sqlite"))) + monkeypatch.delenv("CONFIG_REVISION_SYNC_ENABLED", raising=False) + with patch("config.load_runtime_env"), patch("env_loader.load_runtime_env"), \ + patch("services.usage_ledger.start"), patch("requests.Session.send", side_effect=AssertionError("No network")): + module = importlib.import_module("app") + with patch.object(module, "load_runtime_env"), patch.object(module, "register_gateway_extensions") as registrar: + module.create_app() + assert registrar.call_count == 1 + + +def test_additive_migration_preserves_old_rows_and_rehearses_twice(): + migration = Path(__file__).resolve().parents[1] / "intelligence-migrations/0017_control_revisions.sql" + with sqlite3.connect(":memory:") as db: + db.execute("CREATE TABLE config_snapshot_revisions (domain TEXT PRIMARY KEY, revision INTEGER)") + db.execute("INSERT INTO config_snapshot_revisions VALUES ('auto_routes', 9)") + db.executescript(migration.read_text()) + db.executescript(migration.read_text()) + assert db.execute("SELECT revision FROM config_snapshot_revisions").fetchone() == (9,) + assert db.execute("SELECT COUNT(*) FROM control_revisions").fetchone() == (0,) + + +def test_malformed_metadata_cannot_renew_freshness(): + sync, now, _, _ = harness(security=True) + sync.tick() + now[0] += 5.01 + for invalid in ({"models": True}, {"models": -1}, {"models": 1, "extra": 0}, {"models": 2 ** 60}): + sync.fetch = lambda domains: invalid + sync.tick() + assert not sync.security_ready() + now[0] += 5 + + +def test_strict_owned_installers_keep_copies_on_authority_failure(): + from services import config_revision_sync, model_override_d1, provider_catalog_d1 + from services.provider_catalog_refresh import refresh_provider_catalog_revision + with patch.dict(model_override_d1._cache, {"overrides": {"openai:old": "disabled"}, "expires": 0}), \ + patch.object(config_revision_sync.control_state_d1, "call", side_effect=RuntimeError("offline")): + with pytest.raises(RuntimeError): + config_revision_sync._refresh_models(1) + assert model_override_d1._cache["overrides"] == {"openai:old": "disabled"} + prior = dict(provider_catalog_d1._state) + with patch.object(config_revision_sync.control_state_d1, "call", side_effect=RuntimeError("offline")): + with pytest.raises(RuntimeError): + refresh_provider_catalog_revision(1) + assert provider_catalog_d1._state == prior + + +def test_registrar_reuses_one_process_poller_for_repeated_factories(monkeypatch): + from services import config_revision_sync + monkeypatch.setattr(config_revision_sync, "_active", None) + monkeypatch.setenv("INTELLIGENCE_STORAGE_BACKEND", "d1") + monkeypatch.setenv("AUTH_STORAGE_BACKEND", "d1") + with patch.object(config_revision_sync.control_state_d1, "register") as register, \ + patch.object(config_revision_sync.control_state_d1, "ensure_running"): + first = config_revision_sync.configure_sync(SyncSettings(True)) + second = config_revision_sync.configure_sync(SyncSettings(True)) + assert first is second + assert register.call_count == 1 + + +def test_unloaded_new_revision_cannot_be_hidden_by_metadata_rollback(): + sync, now, revisions, _ = harness(security=True) + sync.tick() + sync._domains["models"].refresh = Mock(side_effect=RuntimeError("offline")) + revisions["models"] = 1; now[0] += 5 + sync.tick() + assert not sync.security_ready() + revisions["models"] = 0; now[0] += 5 + sync.tick() + assert not sync.security_ready() + + +@pytest.mark.parametrize("path", ["/openai/v1/chat/completions", "/googleai/chat/completions", + "/api/backends/chat-completions/generate", "/intelligence/run", "/optimize/run", "/mcp"]) +def test_registered_security_guard_covers_raw_and_special_protocols(path): + app = Flask(__name__) + sync, _, _, _ = harness(security=True) + register_gateway_extensions(app, revision_sync=sync) + app.add_url_rule(path, "synthetic_dispatch", lambda: {"dispatched": True}, methods=["POST", "OPTIONS"]) + client = app.test_client() + assert client.post(path).status_code == 503 + assert client.options(path).status_code == 200 + + +def test_guard_keeps_health_accessible(): + app = Flask(__name__) + sync, _, _, _ = harness(security=True) + register_gateway_extensions(app, revision_sync=sync) + app.add_url_rule("/healthz", "health", lambda: {"healthy": True}) + assert app.test_client().get("/healthz").status_code == 200 + + +class AccountAuthority: + """Private RPC responses captured from an account writer, without network IO.""" + + def __init__(self, states): + self.states, self.index, self.offline = states, 0, False + self.calls = [] + + def __call__(self, payload, *, endpoint): + self.calls.append((endpoint, payload["operation"])) + if self.offline: + raise RuntimeError("Authority unavailable") + state = self.states[self.index] + if endpoint == "model_overrides": + if payload["operation"] == "revisions": + return {"version": 1, "revisions": {name: state["revisions"].get(name, 0) for name in payload["domains"]}} + return {"version": 1, "overrides": []} + if endpoint == "users": + assert payload["operation"] == "list", "Enabled authentication must use the installed copy" + users = [row for row in state["users"] if payload["after"] is None or row["username"] > payload["after"]] + return {"version": 1, "users": users[:payload["limit"]]} + if endpoint == "auto_routes": + return {"version": 1, "routes": []} + if endpoint == "provider_catalog": + return {"version": 1, "snapshots": []} + raise AssertionError(endpoint) + + +def account_states(changes=None): + from werkzeug.security import generate_password_hash + from services import user_store + row = dict.fromkeys(user_store.USER_FIELDS) + row.update(username="alice", api_key_hash=generate_password_hash("synthetic-revision-key"), + api_key_prefix="mllm_syntheti", scopes="chat,models", is_admin=0, + created_at="2026-10-09T00:00:00+00:00") + return [{"users": [row], "revisions": {"key_controls": 1, "model_grants": 1}}, + {"users": [{**row, **(changes or {"revoked_at": "2026-10-09T01:00:00+00:00"})}], + "revisions": {"key_controls": 2, "model_grants": 2}}] + + +def registered_authority(monkeypatch, states, *, enabled=True): + from flask import request + from services import auth_service, config_revision_sync, intelligence_d1_store, user_store + auth = auth_service.AuthService + authority, now = AccountAuthority(states), [100.0] + monkeypatch.setenv("CONFIG_REVISION_SYNC_ENABLED", "true" if enabled else "false") + monkeypatch.setenv("INTELLIGENCE_STORAGE_BACKEND", "d1") + monkeypatch.setenv("AUTH_STORAGE_BACKEND", "d1") + monkeypatch.setenv("ADMIN_API_KEY", "synthetic-other-admin") + monkeypatch.setattr(config_revision_sync, "_active", None) + monkeypatch.setattr(config_revision_sync, "AuthService", auth) + monkeypatch.setattr(config_revision_sync.control_state_d1, "BACKGROUND", False) + monkeypatch.setattr(config_revision_sync.control_state_d1, "_tasks", []) + monkeypatch.setattr(intelligence_d1_store, "request_private_intelligence", authority) + monkeypatch.setattr(user_store, "request_private_intelligence", authority) + monkeypatch.setattr(auth, "_users", {}) + monkeypatch.setattr(auth, "_verified_keys", {}) + monkeypatch.setattr(auth, "_api_key_prefix_index", {}) + monkeypatch.setattr(auth, "_update_key_usage", lambda *args: None) + monkeypatch.setattr("requests.Session.send", Mock(side_effect=AssertionError("No network"))) + app = Flask(__name__) + app.config["TESTING"] = True + register_gateway_extensions(app) + sync = app.extensions.get("config_revision_sync") + if sync: + sync.clock, sync.jitter = lambda: now[0], lambda: 0 + + @app.get("/v1/probe") + def probe(): + user = auth.verify_api_key(request.headers.get("X-Test-Key"), "203.0.113.9") + return ({"user": user}, 200) if user else ({"error": "invalid_api_key"}, 401) + + return app.test_client(), sync, authority, now, auth + + +def exercise_registered_writer_states(states): + """Also run by the Worker suite with snapshots read from its real D1 writer.""" + with pytest.MonkeyPatch.context() as monkeypatch: + client, sync, authority, now, auth = registered_authority(monkeypatch, states) + headers = {"X-Test-Key": "synthetic-revision-key"} + assert client.get("/v1/probe", headers=headers).status_code == 503 + sync.tick() + assert sync.security_ready() + assert client.get("/v1/probe", headers=headers).status_code == 200 + memo = dict(auth._verified_keys) + assert memo + authority.index = 1 + now[0] += 4.1 + sync.tick() + assert sync.security_ready() + assert not auth._verified_keys + assert client.get("/v1/probe", headers=headers).status_code == 401 + assert all(expiry > time.monotonic() for expiry, _ in memo.values()) + authority.offline = True + now[0] += 5.01 + sync.tick() + response = client.get("/v1/probe", headers=headers) + assert response.status_code == 503 + assert response.json["error"]["code"] == "config_security_stale" + assert response.headers["Cache-Control"] == "no-store" + + +def test_registered_default_installers_refresh_revoked_keys_and_expired_authority(): + exercise_registered_writer_states(account_states()) + + +@pytest.mark.parametrize("changes", [ + {"scopes": "models", "allowed_models": "openai:new"}, + {"expires_at": "2020-01-01T00:00:00+00:00", "daily_budget_usd": 2}, + {"allowed_ips": "192.0.2.0/24", "monthly_budget_usd": 5}, +]) +def test_registered_installer_replaces_cached_grants_and_controls(monkeypatch, changes): + client, sync, authority, now, auth = registered_authority(monkeypatch, account_states(changes)) + sync.tick() + headers = {"X-Test-Key": "synthetic-revision-key"} + assert client.get("/v1/probe", headers=headers).status_code == 200 + authority.index = 1 + now[0] += 4.1 + sync.tick() + assert not auth._verified_keys + user = client.get("/v1/probe", headers=headers).json["user"] + for name, value in changes.items(): + assert user[name] == (value.split(",") if name in {"scopes", "allowed_models", "allowed_ips"} else value) + + +@pytest.mark.parametrize("changes", [{"revoked_at": "invalid"}, {"daily_budget_usd": -1}, + {"allowed_models": "openai:invalid model"}, {"allowed_ips": "garbage"}, {"expires_at": "tomorrow"}]) +def test_invalid_account_copy_cannot_certify_security(monkeypatch, changes): + client, sync, authority, now, auth = registered_authority(monkeypatch, account_states(changes)) + sync.tick() + authority.index = 1 + now[0] += 4.1 + sync.tick() + assert not sync.security_ready() + assert client.get("/v1/probe").status_code == 503 + assert auth._users["alice"]["revoked_at"] is None + + +@pytest.mark.parametrize("backend,auth_backend", [("", ""), ("sql", "sql"), ("invalid", "d1"), ("d1", "sql"), ("d1", "invalid")]) +def test_unsupported_runtime_disables_sync_once(monkeypatch, caplog, backend, auth_backend): + from services import config_revision_sync + monkeypatch.setenv("CONFIG_REVISION_SYNC_ENABLED", "true") + monkeypatch.setenv("INTELLIGENCE_STORAGE_BACKEND", backend) + monkeypatch.setenv("AUTH_STORAGE_BACKEND", auth_backend) + monkeypatch.setattr(config_revision_sync, "_warned", set()) + with patch.object(config_revision_sync.control_state_d1, "register") as register: + for _ in range(2): + app = Flask(__name__) + register_gateway_extensions(app) + assert "config_revision_sync" not in app.extensions + app.add_url_rule("/v1/probe", "probe", lambda: "unchanged") + assert app.test_client().get("/v1/probe").data == b"unchanged" + register.assert_not_called() + assert sum("requires Container D1" in record.message for record in caplog.records) == 1 + + +def test_flag_off_retains_verified_key_memo_and_legacy_auth(monkeypatch): + from services import user_store + client, sync, authority, now, auth = registered_authority(monkeypatch, account_states(), enabled=False) + assert sync is None + # The disabled path retains its existing prefix lookup and 60-second memo. + with patch.object(user_store, "users_by_prefix", return_value=authority.states[0]["users"]) as lookup: + headers = {"X-Test-Key": "synthetic-revision-key"} + assert client.get("/v1/probe", headers=headers).status_code == 200 + authority.index = 1 + assert client.get("/v1/probe", headers=headers).status_code == 200 + assert lookup.call_count == 1 + assert authority.calls == [] + + +@pytest.mark.parametrize("mutation", ["delete", "rotate"]) +def test_installed_copy_rejects_deleted_or_rotated_memoized_keys(monkeypatch, mutation): + states = account_states({"api_key_hash": "scrypt:32768:8:1$other$" + "0" * 128, + "api_key_prefix": "mllm_rotated"}) + if mutation == "delete": + states[1]["users"] = [] + exercise_registered_writer_states(states) + + +def test_explicit_missing_installer_remains_closed_in_d1(monkeypatch): + client, sync, authority, now, auth = registered_authority(monkeypatch, account_states()) + from services.config_revision_sync import missing_security_refresh + sync._domains["key_controls"].refresh = missing_security_refresh + sync.tick() + assert not sync.security_ready() + assert client.get("/v1/probe").status_code == 503 + + +def test_security_pages_install_together_and_bad_pagination_preserves_old_copy(monkeypatch): + from services import key_controls, user_store + client, sync, authority, now, auth = registered_authority(monkeypatch, account_states()) + monkeypatch.setattr(user_store, "PAGE_SIZE", 1) + sync.tick() + assert sync.security_ready() + assert authority.calls.count(("users", "list")) == 4 # two pages for each security domain + prior = dict(auth._users) + row = account_states()[0]["users"][0] + with patch.object(user_store, "_call", return_value={"version": 1, "users": [row]}): + with pytest.raises(ValueError, match="pagination"): + key_controls.refresh_security_copy(auth, 2) + assert auth._users == prior + + +@pytest.mark.parametrize("malformed", ["missing_control", "extra_field", "boolean_version", "missing_users", "not_a_list"]) +def test_security_copy_rejects_incomplete_private_responses(monkeypatch, malformed): + from services import key_controls, user_store + client, sync, authority, now, auth = registered_authority(monkeypatch, account_states()) + sync.tick() + assert client.get("/v1/probe", headers={"X-Test-Key": "synthetic-revision-key"}).status_code == 200 + prior, memo = dict(auth._users), dict(auth._verified_keys) + response = {"version": 1, "users": [dict(authority.states[0]["users"][0])]} + if malformed == "missing_control": + response["users"][0].pop("shadow_eval_rate") + elif malformed == "extra_field": + response["unexpected"] = True + elif malformed == "boolean_version": + response["version"] = True + elif malformed == "missing_users": + response.pop("users") + else: + response["users"] = None + with patch.object(user_store, "_call", return_value=response): + with pytest.raises(ValueError): + key_controls.refresh_security_copy(auth, 2) + assert auth._users == prior + assert auth._verified_keys == memo diff --git a/tests/test_config_revision_sync_integration.mjs b/tests/test_config_revision_sync_integration.mjs new file mode 100644 index 00000000..48bcfdc8 --- /dev/null +++ b/tests/test_config_revision_sync_integration.mjs @@ -0,0 +1,35 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { execFile } from "node:child_process"; +import { scryptSync } from "node:crypto"; +import { promisify } from "node:util"; +import { account, database, usersCall } from "./support/config_revision_fixtures.mjs"; + +// Drives the registered Flask auth path, so it runs with the Python dependencies installed. +test("a key revoked by the real writer is rejected by registered Flask auth before memo expiry", async t => { + const { env, revisions } = await database(t); + const digest = scryptSync("synthetic-revision-key", "synthetic-salt", 64, { N: 32768, r: 8, p: 1, maxmem: 64 * 1024 * 1024 }).toString("hex"); + const user = account({ api_key_hash: `scrypt:32768:8:1$synthetic-salt$${digest}` }); + const states = []; + for (const revoked_at of [null, "2026-10-09T01:00:00+00:00"]) { + assert.equal((await usersCall(env, { operation: "upsert", user: { ...user, revoked_at } })).status, 200); + states.push({ users: (await usersCall(env, { operation: "list", after: null, limit: 200 })).body.users, + revisions: (await revisions(["key_controls", "model_grants"])).body.revisions }); + } + // Pass only synthetic committed snapshots across the process boundary, never a + // private HTTP endpoint or the host environment. The Python harness rejects IO. + const script = `import os, sys, json, types +r = os.getcwd() +sys.path.insert(0, r) +sys.path.insert(0, os.path.join(r, "tests")) +sys.modules["env_loader"] = types.SimpleNamespace(load_runtime_env=lambda *a, **kw: None) +import test_knowledge_onboarding +from test_config_revision_sync import exercise_registered_writer_states +exercise_registered_writer_states(json.loads(sys.argv[1])) +print("registered-revocation-ok")`; + const run = promisify(execFile); + const python = process.env.PYTHON || "python3"; + const result = await run(python, ["-I", "-c", script, JSON.stringify(states)], + { cwd: process.cwd(), env: { PATH: process.env.PATH }, timeout: 30000 }); + assert.match(result.stdout, /registered-revocation-ok/); +}); diff --git a/tests/test_config_revision_sync_worker.mjs b/tests/test_config_revision_sync_worker.mjs new file mode 100644 index 00000000..5a32faee --- /dev/null +++ b/tests/test_config_revision_sync_worker.mjs @@ -0,0 +1,237 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { convertV4MiniflareOptions, Miniflare } from "miniflare"; +import { handleIntelligenceOutbound } from "../worker/intelligence-outbound.mjs"; +import { handleRevisionedAutoRoutes, commitRevision, revisionSyncSettings, RevisionConsumer } from "../worker/config-revision.mjs"; +import { handleAutoRoutesRequest } from "../worker/auto-routes-d1.mjs"; +import { boundedBody } from "../worker/control-users-d1.mjs"; +import { applyMigrations } from "./d1_migrations.mjs"; +import { account, database, usersCall } from "./support/config_revision_fixtures.mjs"; + +const putModel = { operation: "put", model_id: "openai:test", status: "disabled", updated_at: "2026-10-09T00:00:00Z" }; + +test("disabled and malformed flags make no revision statements", async t => { + const { db, call } = await database(t, true); + for (const flag of [undefined, "", "false", "invalid"]) { + const env = { INTELLIGENCE_DB: db, CONFIG_REVISION_SYNC_ENABLED: flag }; + assert.equal((await call("models", { operation: "revisions", domains: ["model_overrides"] }, env)).status, 404); + assert.deepEqual((await call("models", putModel, env)).body, { version: 1, stored: true }); + } + assert.equal(revisionSyncSettings({ CONFIG_REVISION_SYNC_ENABLED: "true", CONFIG_SYNC_TTL_SECONDS: "NaN" }).enabled, false); + assert.equal(revisionSyncSettings({ CONFIG_REVISION_SYNC_ENABLED: "true", CONFIG_SYNC_TTL_SECONDS: "" }).ttl, 30); +}); + +test("missing migration fails closed before existing data changes", async t => { + const { call, revisions, db } = await database(t, true); + assert.equal((await revisions(["model_overrides"])).status, 503); + assert.equal((await call("models", putModel)).status, 503); + assert.equal((await db.prepare("SELECT COUNT(*) AS n FROM control_model_overrides").first()).n, 0); + assert.equal((await call("routes", { operation: "put", route_id: "auto:test", candidates: ["openai:test"], updated_at: putModel.updated_at })).status, 503); +}); + +test("registered model writes atomically advance revision without altering response", async t => { + const { call, revisions, db } = await database(t); + assert.equal((await revisions(["model_overrides"])).body.revisions.model_overrides, 0); + await Promise.all([call("models", putModel), call("models", { ...putModel, model_id: "openai:other" })]); + assert.equal((await revisions(["model_overrides"])).body.revisions.model_overrides, 2); + const metadata = await db.prepare("SELECT * FROM control_revisions").all(); + assert.equal(metadata.results[0].domain, "model_overrides"); + assert.equal(Object.hasOwn(metadata.results[0], "model_id"), false); +}); + +test("CAS admits one concurrent writer and preserves rejected configuration", async t => { + const { db } = await database(t); + const write = model => commitRevision(db, "model_overrides", [db.prepare("INSERT INTO control_model_overrides VALUES (?, 'disabled', 'time')").bind(model)], 0); + const results = await Promise.all([write("openai:a"), write("openai:b")]); + assert.deepEqual(results.sort(), [false, true]); + assert.equal((await db.prepare("SELECT COUNT(*) AS n FROM control_model_overrides").first()).n, 1); + assert.equal((await db.prepare("SELECT revision FROM control_revisions WHERE domain='model_overrides'").first()).revision, 1); +}); + +test("failed write rolls back revision and cannot leak error details", async t => { + const { db } = await database(t); + await assert.rejects(commitRevision(db, "model_overrides", [db.prepare("INSERT INTO missing_table VALUES ('private detail')")])); + assert.equal((await db.prepare("SELECT COUNT(*) AS n FROM control_revisions").first()).n, 0); +}); + +test("account mutations can increment key and grant domains in the same transaction", async t => { + const { db, revisions } = await database(t); + assert.equal(await commitRevision(db, ["key_controls", "model_grants"], [db.prepare("INSERT INTO control_model_overrides VALUES ('openai:account', 'disabled', 'time')")]), true); + assert.deepEqual((await revisions(["key_controls", "model_grants"])).body.revisions, { key_controls: 1, model_grants: 1 }); + await assert.rejects(commitRevision(db, ["key_controls", "model_grants"], [db.prepare("INSERT INTO missing_table VALUES ('failure')")])); + assert.deepEqual((await revisions(["key_controls", "model_grants"])).body.revisions, { key_controls: 1, model_grants: 1 }); +}); + +test("auto routes bridge W13 and snapshot apply advances exactly once", async t => { + const { env, call, revisions, db } = await database(t); + assert.equal((await call("routes", { operation: "put", route_id: "auto:test", candidates: ["openai:old"], updated_at: putModel.updated_at })).status, 200); + assert.equal((await revisions(["auto_routes"])).body.revisions.auto_routes, 1); + assert.equal((await db.prepare("SELECT COUNT(*) AS n FROM control_revisions WHERE domain='auto_routes'").first()).n, 0); + env.CONFIG_SNAPSHOTS_ENABLED = "true"; + const created = await call("routes", { operation: "snapshot_create", domain: "auto_routes", base_revision: 1, + configuration: { routes: [{ route_id: "auto:test", candidates: ["openai:new"] }] }, actor: "a".repeat(64) }); + assert.equal(created.status, 200); + const apply = () => call("routes", { operation: "snapshot_apply", id: created.body.snapshot.id, current_revision: 1, + confirm: true, actor: "a".repeat(64) }); + const attempts = await Promise.all([apply(), apply()]); + assert.deepEqual(attempts.map(item => item.status).sort(), [200, 409]); + assert.equal((await revisions(["auto_routes"])).body.revisions.auto_routes, 2); + assert.equal((await db.prepare("SELECT COUNT(*) AS n FROM config_snapshot_applications").first()).n, 1); +}); + +test("sync does not enable public snapshot operations", async t => { + const { call } = await database(t); + assert.equal((await call("routes", { operation: "snapshot_list" })).status, 404); +}); + +test("catalog writes and revision commit in one transaction; W11 remains reachable", async t => { + const { call, revisions, env } = await database(t); + assert.equal((await call("catalog", { operation: "put", provider: "openai", updated_at: putModel.updated_at, data: "YWJj" })).status, 200); + assert.equal((await revisions(["provider_catalog"])).body.revisions.provider_catalog, 1); + assert.equal((await call("prompt-templates", { operation: "list", principal: "a".repeat(64), after: null }, + { ...env, PROMPT_TEMPLATES_ENABLED: "true" })).status, 200); +}); + +test("native consumers load only newer revisions and fail closed after security TTL", async () => { + let now = 100, revision = 0, offline = false; + const loads = []; + const consumer = new RevisionConsumer({ CONFIG_REVISION_SYNC_ENABLED: "true" }, { + clock: () => now, jitter: () => 0, + read: async domains => { + if (offline) throw new Error("offline"); + return Object.fromEntries(domains.map(domain => [domain, revision])); + }, + refreshers: Object.fromEntries(["auto_routes", "provider_catalog", "key_controls", "model_grants", "model_overrides"].map( + domain => [domain, async value => loads.push([domain, value])])), + }); + assert.equal(consumer.requireFreshSecurity().status, 503); + await consumer.tick(); + assert.equal(consumer.requireFreshSecurity(), null); + assert.equal(loads.length, 5); + now += 5; + await consumer.tick(); + assert.equal(loads.length, 5); + revision = 1; now += 30; + await consumer.tick(); + assert.equal(loads.length, 10); + offline = true; now += 31; + await consumer.tick(); + assert.equal(consumer.requireFreshSecurity().status, 503); + assert.equal(consumer.status().domains.auto_routes.stale, true); + assert.equal(JSON.stringify(consumer.status()).includes("offline"), false); +}); + +test("native disabled mode is inert and missing collaborators cannot certify keys", async () => { + const inert = new RevisionConsumer({}, { read: () => assert.fail("No storage") }); + await inert.tick(); + assert.equal(inert.requireFreshSecurity(), null); + const missing = new RevisionConsumer({ CONFIG_REVISION_SYNC_ENABLED: "true" }, { + read: async domains => Object.fromEntries(domains.map(domain => [domain, 0])), + }); + await missing.tick(); + assert.equal(missing.requireFreshSecurity().status, 503); +}); + +test("native polling is single flight and concurrent writes never renew security", async () => { + let now = 100, calls = 0, revision = 0; + const consumer = new RevisionConsumer({ CONFIG_REVISION_SYNC_ENABLED: "true" }, { + clock: () => now, jitter: () => 1, + read: async domains => { calls++; return Object.fromEntries(domains.map(domain => [domain, revision])); }, + refreshers: { model_overrides: async () => { revision++; } }, + }); + await Promise.all(Array.from({ length: 100 }, () => consumer.tick())); + assert.equal(calls, 3); // security read+confirmation, ordinary read + assert.equal(consumer.status().domains.model_overrides.revision, null); + assert.equal(consumer.requireFreshSecurity().status, 503); + now += 1; + await consumer.tick(); + assert.equal(calls, 3); +}); + +test("native rollback cannot hide an observed revision whose load failed", async () => { + let now = 100, revision = 0, failure = false; + const consumer = new RevisionConsumer({ CONFIG_REVISION_SYNC_ENABLED: "true" }, { + clock: () => now, + read: async domains => Object.fromEntries(domains.map(domain => [domain, revision])), + refreshers: Object.fromEntries(["model_overrides", "key_controls", "model_grants"].map(domain => [domain, async () => { + if (failure) throw new Error("offline"); + }])), + }); + await consumer.tick(); + assert.equal(consumer.requireFreshSecurity(), null); + failure = true; revision = 1; now += 5; + await consumer.tick(); + assert.equal(consumer.requireFreshSecurity().status, 503); + revision = 0; now += 5; + await consumer.tick(); + assert.equal(consumer.requireFreshSecurity().status, 503); +}); + +test("revision request validation excludes contents, arbitrary domains and extra fields", async t => { + const { call } = await database(t); + for (const domains of [[], ["unknown"], ["model_overrides", "model_overrides"], ["model_overrides; DROP TABLE users"]]) { + assert.equal((await call("models", { operation: "revisions", domains })).status, 400); + } + assert.equal((await call("models", { operation: "revisions", domains: ["model_overrides"], content: "private" })).status, 400); +}); + + +test("real account writes advance both security domains; touches and refused writes do not", async t => { + const { db, env, revisions } = await database(t); + const current = async () => (await revisions(["key_controls", "model_grants"])).body.revisions; + assert.deepEqual((await usersCall(env, { operation: "upsert", user: account() })).body, { version: 1, stored: true }); + assert.deepEqual(await current(), { key_controls: 1, model_grants: 1 }); + for (const changes of [{ revoked_at: putModel.updated_at }, { scopes: "models" }, { allowed_models: "openai:new" }, + { daily_budget_usd: 2, monthly_budget_usd: 3, expires_at: "2099-01-01T00:00:00Z" }, + { api_key_hash: "rotated-hash", api_key_prefix: "rotated", rotated_at: putModel.updated_at }, + { allowed_ips: "192.0.2.0/24" }]) { + const before = await current(); + assert.equal((await usersCall(env, { operation: "upsert", user: account(changes) })).status, 200); + assert.deepEqual(await current(), { key_controls: before.key_controls + 1, model_grants: before.model_grants + 1 }); + } + const before = await current(); + assert.equal((await usersCall(env, { operation: "touch", username: "alice", last_used_at: putModel.updated_at, last_used_ip: null })).status, 200); + assert.equal((await usersCall(env, { operation: "upsert", user: account({ is_admin: 1 }) })).status, 403); + assert.deepEqual(await current(), before); + assert.deepEqual((await usersCall(env, { operation: "delete", username: "alice" })).body, { version: 1, deleted: true }); + assert.deepEqual(await current(), { key_controls: before.key_controls + 1, model_grants: before.model_grants + 1 }); + const audit = (await db.prepare("SELECT outcome FROM control_user_audit WHERE operation='delete'").all()).results; + assert.deepEqual(audit, [{ outcome: "deleted" }]); + assert.deepEqual((await usersCall(env, { operation: "delete", username: "alice" })).body, { version: 1, deleted: false }); + assert.deepEqual((await db.prepare("SELECT outcome FROM control_user_audit WHERE operation='delete' ORDER BY id").all()).results, + [{ outcome: "deleted" }, { outcome: "missing" }]); +}); + +test("enabled missing revision table refuses account mutations and disabled writers stay unchanged", async t => { + const { db, env } = await database(t, true); + assert.equal((await usersCall(env, { operation: "upsert", user: account() })).status, 503); + assert.equal((await db.prepare("SELECT COUNT(*) AS n FROM control_users").first()).n, 0); + const disabled = { ...env, CONFIG_REVISION_SYNC_ENABLED: "false" }; + assert.deepEqual((await usersCall(disabled, { operation: "upsert", user: account() })).body, { version: 1, stored: true }); + assert.equal((await usersCall(env, { operation: "delete", username: "alice" })).status, 503); + assert.equal((await db.prepare("SELECT COUNT(*) AS n FROM control_users").first()).n, 1); + assert.deepEqual((await usersCall(disabled, { operation: "delete", username: "alice" })).body, { version: 1, deleted: true }); +}); + +test("enabled account transaction failure rolls back both revisions and the account", async t => { + const { db, env, revisions } = await database(t); + // An injected audit failure aborts the same batch as the account and both counters. + const failing = { ...env, INTELLIGENCE_DB: { prepare: sql => db.prepare(sql.includes("INSERT INTO control_user_audit") + ? "INSERT INTO missing_audit_table VALUES (?, ?, ?, ?, ?, ?, ?, ?)" : sql), batch: statements => db.batch(statements) } }; + assert.equal((await usersCall(failing, { operation: "upsert", user: account() })).status, 503); + assert.equal((await db.prepare("SELECT COUNT(*) AS n FROM control_users").first()).n, 0); + assert.deepEqual((await revisions(["key_controls", "model_grants"])).body.revisions, { key_controls: 0, model_grants: 0 }); +}); + + +test("enabled security listing never silently substitutes missing account controls", async t => { + const mf = new Miniflare(convertV4MiniflareOptions({ modules: true, + script: "export default {fetch(){return new Response('ok')}}", d1Databases: ["INTELLIGENCE_DB"] })); + t.after(() => mf.dispose()); + const db = await mf.getD1Database("INTELLIGENCE_DB"); + await applyMigrations(db, { skip: ["0013_shadow_eval.sql"] }); + const env = { INTELLIGENCE_DB: db, CONFIG_REVISION_SYNC_ENABLED: "true" }; + assert.equal((await usersCall(env, { operation: "list", after: null, limit: 200 })).status, 503); + assert.deepEqual((await usersCall({ ...env, CONFIG_REVISION_SYNC_ENABLED: "false" }, + { operation: "list", after: null, limit: 200 })).body, { version: 1, users: [] }); +}); diff --git a/worker/config-revision.mjs b/worker/config-revision.mjs new file mode 100644 index 00000000..9e3adf23 --- /dev/null +++ b/worker/config-revision.mjs @@ -0,0 +1,194 @@ +/** Revision metadata and atomic writes behind the fixed private control-state endpoint. */ + +const DOMAINS = Object.freeze(["auto_routes", "model_overrides", "provider_catalog", "key_controls", "model_grants"]); +const warned = new Set(); +function malformed(name) { + if (!warned.has(name)) { + warned.add(name); + console.warn(`Invalid ${name}; configuration revision sync disabled`); + } +} + +export function revisionSyncSettings(env) { + const flag = String(env.CONFIG_REVISION_SYNC_ENABLED || "").trim().toLowerCase(); + if (!["", "0", "false", "no", "off", "1", "true", "yes", "on"].includes(flag)) { + malformed("CONFIG_REVISION_SYNC_ENABLED"); + return { enabled: false, ttl: 30, securityTtl: 5 }; + } + const result = { enabled: ["1", "true", "yes", "on"].includes(flag), ttl: 30, securityTtl: 5 }; + if (!result.enabled) return result; + for (const [name, field, maximum] of [["CONFIG_SYNC_TTL_SECONDS", "ttl", 3600], ["CONFIG_SECURITY_TTL_SECONDS", "securityTtl", 5]]) { + const value = String(env[name] || "").trim(); + if (!value) continue; + const number = Number(value); + if (!/^(?:\d+(?:\.\d*)?|\.\d+)$/.test(value) || !Number.isFinite(number) || number < 1 || number > maximum) { + malformed(name); + return { enabled: false, ttl: 30, securityTtl: 5 }; + } + result[field] = number; + } + return result; +} + +export async function requireRevisionSchema(db) { + await db.prepare("SELECT domain, revision, updated_at FROM control_revisions LIMIT 1").all(); +} + +/** Only trusted, fixed storage statements are accepted. D1 batch rolls back a failed CAS. */ +export async function commitRevision(db, domain, statements, expectedRevision = null) { + const domains = Array.isArray(domain) ? domain : [domain]; + if (!domains.length || new Set(domains).size !== domains.length + || !domains.every(name => DOMAINS.includes(name) && name !== "auto_routes") || !statements.length + || expectedRevision !== null && (!Number.isSafeInteger(expectedRevision) || expectedRevision < 0)) { + throw new Error("Invalid revision transaction"); + } + await requireRevisionSchema(db); + const now = new Date().toISOString(); + const batch = domains.flatMap(name => [ + db.prepare("INSERT OR IGNORE INTO control_revisions (domain, revision, updated_at) VALUES (?, 0, ?)").bind(name, now), + db.prepare(`UPDATE control_revisions SET revision=revision+1, updated_at=? + WHERE domain=? AND revision < 9007199254740991 ${expectedRevision === null ? "" : "AND revision=?"}`) + .bind(...(expectedRevision === null ? [now, name] : [now, name, expectedRevision])), + // A failed CAS aborts the entire batch before any configuration statement runs. + db.prepare("INSERT INTO control_revisions (domain, revision, updated_at) SELECT ?, -1, ? WHERE changes() != 1").bind(name, now), + ]); + batch.push(...statements); + try { + await db.batch(batch); + return true; + } catch (error) { + if (String(error?.message).includes("revision_cas_confirmed")) return false; + throw error; + } +} + +export async function handleRevisionMetadata(db, body) { + if (Object.keys(body).length !== 3 || body.version !== 1 || body.operation !== "revisions" + || !Array.isArray(body.domains) || !body.domains.length || body.domains.length > DOMAINS.length + || new Set(body.domains).size !== body.domains.length || !body.domains.every(name => DOMAINS.includes(name))) return null; + await requireRevisionSchema(db); + const statements = body.domains.map(domain => domain === "auto_routes" + ? db.prepare("SELECT COALESCE((SELECT revision FROM config_snapshot_revisions WHERE domain='auto_routes'), 0) AS revision") + : db.prepare("SELECT COALESCE((SELECT revision FROM control_revisions WHERE domain=?), 0) AS revision").bind(domain)); + const rows = await db.batch(statements); + return Response.json({ version: 1, revisions: Object.fromEntries(body.domains.map((domain, index) => [domain, rows[index].results[0].revision])) }, + { headers: { "cache-control": "no-store" } }); +} + +/** Private auto-route dispatch injects its handler and bounded body reader. */ +export async function handleRevisionedAutoRoutes(request, env, { handleAutoRoutesRequest, boundedBody }) { + if (!revisionSyncSettings(env).enabled) return handleAutoRoutesRequest(request, env); + const url = new URL(request.url); + if (request.method !== "POST" || url.origin !== "http://intelligence.internal" || url.pathname !== "/v1/auto-routes" + || url.search || url.hash || url.username || url.password) return handleAutoRoutesRequest(request, env); + try { + if (!env.INTELLIGENCE_DB) throw new Error("Missing storage"); + await requireRevisionSchema(env.INTELLIGENCE_DB); + } catch { + return Response.json({ version: 1, error: { code: "config_revision_storage_unavailable", message: "Configuration revision storage is unavailable" } }, + { status: 503, headers: { "cache-control": "no-store" } }); + } + if (["1", "true", "yes", "on"].includes(String(env.CONFIG_SNAPSHOTS_ENABLED || "").trim().toLowerCase())) { + return handleAutoRoutesRequest(request, env); + } + // Reuse the snapshot counter only for a normal write, never snapshot operations. + try { + const body = JSON.parse(await boundedBody(request.clone(), 16384)); + if (body?.operation === "put") return handleAutoRoutesRequest(request, { ...env, CONFIG_SNAPSHOTS_ENABLED: "true" }); + } catch { + // The original handler owns validation and the unchanged error envelope. + } + return handleAutoRoutesRequest(request, env); +} + +const SECURITY_DOMAINS = new Set(["model_overrides", "key_controls", "model_grants"]); +const validMetadata = (value, domains) => value && typeof value === "object" && !Array.isArray(value) + && Object.keys(value).length === domains.length && domains.every(domain => Object.hasOwn(value, domain) + && Number.isSafeInteger(value[domain]) && value[domain] >= 0 && value[domain] <= 9007199254740991); + +/** Native dispatch injects strict cache installers and calls the freshness guard. */ +export class RevisionConsumer { + constructor(env, { read, refreshers = {}, clock = () => performance.now() / 1000, jitter = Math.random } = {}) { + this.settings = revisionSyncSettings(env); + this.read = read || (async domains => { + const response = await handleRevisionMetadata(env.INTELLIGENCE_DB, { version: 1, operation: "revisions", domains }); + return (await response.json()).revisions; + }); + this.clock = clock; + this.jitter = jitter; + this.domains = Object.fromEntries(DOMAINS.map(domain => [domain, { + refresh: refreshers[domain], security: SECURITY_DOMAINS.has(domain), revision: null, observed: null, checked: null, due: 0, + }])); + this.inFlight = null; + } + + async tick() { + if (!this.settings.enabled) return; + if (this.inFlight) return this.inFlight; + this.inFlight = this.poll(); + try { await this.inFlight; } + finally { this.inFlight = null; } + } + + async poll() { + for (const security of [true, false]) await this.pollGroup(security); + } + + async pollGroup(security) { + const started = this.clock(); + const due = Object.keys(this.domains).filter(name => this.domains[name].security === security && started >= this.domains[name].due); + if (!due.length) return; + for (const name of due) { + const ttl = security ? this.settings.securityTtl : this.settings.ttl; + this.domains[name].due = started + ttl * (0.8 + 0.2 * Math.min(1, Math.max(0, this.jitter()))); + } + let revisions; + try { + revisions = await this.read(due); + if (!validMetadata(revisions, due)) throw new Error("Invalid revision metadata"); + } catch { return; } + const changed = {}; + for (const name of due) { + const state = this.domains[name], revision = revisions[name]; + if (state.observed !== null && revision < state.observed) { state.checked = null; continue; } + state.observed = revision; + if (revision === state.revision) { state.checked = started; continue; } + state.checked = null; + if (typeof state.refresh !== "function") continue; + try { + await state.refresh(revision); + changed[name] = revision; + } catch { /* A failed installer cannot renew freshness. */ } + } + const names = Object.keys(changed); + if (!names.length) return; + let confirmed; + try { + confirmed = await this.read(names); + if (!validMetadata(confirmed, names)) throw new Error("Invalid revision confirmation"); + } catch { return; } + for (const name of names) { + this.domains[name].observed = Math.max(this.domains[name].observed || 0, confirmed[name]); + if (confirmed[name] === changed[name]) { + this.domains[name].revision = changed[name]; + this.domains[name].checked = started; + } + } + } + + requireFreshSecurity() { + if (!this.settings.enabled || [...SECURITY_DOMAINS].every(name => this.domains[name].checked !== null + && this.clock() - this.domains[name].checked <= this.settings.securityTtl)) return null; + return Response.json({ error: { code: "config_security_stale", message: "Security configuration freshness could not be verified" } }, + { status: 503, headers: { "cache-control": "no-store" } }); + } + + status() { + const now = this.clock(); + return { enabled: this.settings.enabled, domains: Object.fromEntries(Object.entries(this.domains).map(([name, state]) => [name, { + revision: state.revision, age_seconds: state.checked === null ? null : Math.max(0, now - state.checked), + stale: state.checked === null || now - state.checked > (state.security ? this.settings.securityTtl : this.settings.ttl), + security: state.security, next_poll_seconds: Math.max(0, state.due - now), + }])) }; + } +} diff --git a/worker/container-env.mjs b/worker/container-env.mjs index 9b20c4eb..bc0ce3ae 100644 --- a/worker/container-env.mjs +++ b/worker/container-env.mjs @@ -23,6 +23,9 @@ const DIRECT_ENV_KEYS = [ "INTELLIGENCE_NANOGPT_SUBSCRIPTION_API_KEY", "WORKBENCH_WORKER_URL", "CONNECTION_PROFILES_DB_PATH", + "CONFIG_REVISION_SYNC_ENABLED", + "CONFIG_SYNC_TTL_SECONDS", + "CONFIG_SECURITY_TTL_SECONDS", "RATE_LIMIT_ENABLED", "RATE_LIMIT_RPM", "RATE_LIMIT_TPM", diff --git a/worker/control-state-d1.mjs b/worker/control-state-d1.mjs index 83794924..3bfaf35e 100644 --- a/worker/control-state-d1.mjs +++ b/worker/control-state-d1.mjs @@ -8,6 +8,7 @@ import { boundedBody } from "./control-users-d1.mjs"; import { logFailure } from "./log.mjs"; import { handlePromptTemplates, promptTemplatesEnabled } from "./prompt-templates-d1.mjs"; +import { commitRevision, handleRevisionMetadata, revisionSyncSettings } from "./config-revision.mjs"; const CONTROL = /[\x00-\x1f\x7f]/; const HASH = /^[0-9a-f]{64}$/; @@ -134,7 +135,11 @@ async function login(db, body) { } } -async function models(db, body) { +async function models(db, body, env) { + if (body.operation === "revisions") { + if (!revisionSyncSettings(env).enabled) return reply({ error: "not_found" }, 404); + return handleRevisionMetadata(db, body); + } if (body.operation === "list" && fields(body, ["version", "operation"])) { const { results } = await db.prepare(`SELECT model_id, status FROM control_model_overrides ORDER BY model_id LIMIT ${MAX_OVERRIDES}`).all(); return reply({ overrides: results }); @@ -142,9 +147,12 @@ async function models(db, body) { if (body.operation === "put" && fields(body, ["version", "operation", "model_id", "status", "updated_at"]) && matches(MODEL_ID, body.model_id) && body.model_id.includes(":") && ["available", "disabled"].includes(body.status) && matches(TIMESTAMP, body.updated_at)) { - await db.prepare(`INSERT INTO control_model_overrides (model_id, status, updated_at) VALUES (?, ?, ?) + const statement = db.prepare(`INSERT INTO control_model_overrides (model_id, status, updated_at) VALUES (?, ?, ?) ON CONFLICT(model_id) DO UPDATE SET status=excluded.status, updated_at=excluded.updated_at`) - .bind(body.model_id, body.status, body.updated_at).run(); + .bind(body.model_id, body.status, body.updated_at); + if (revisionSyncSettings(env).enabled) { + if (!await commitRevision(db, "model_overrides", [statement])) return reply({ error: "revision_conflict" }, 409); + } else await statement.run(); return reply({ stored: true }); } return null; @@ -211,7 +219,7 @@ async function workbench(db, body) { } } -async function catalog(db, body) { +async function catalog(db, body, env) { switch (body.operation) { case "list": { if (!fields(body, ["version", "operation"])) return null; @@ -236,9 +244,12 @@ async function catalog(db, body) { chunks.push(body.data.slice(offset, offset + SNAPSHOT_CHUNK_CHARS)); } // The new snapshot replaces the previous one in a single transaction. - await db.batch([db.prepare("DELETE FROM control_provider_catalog WHERE provider = ?").bind(body.provider), + const statements = [db.prepare("DELETE FROM control_provider_catalog WHERE provider = ?").bind(body.provider), ...chunks.map((data, chunk) => db.prepare(`INSERT INTO control_provider_catalog (provider, chunk, data, updated_at) - VALUES (?, ?, ?, ?)`).bind(body.provider, chunk, data, body.updated_at))]); + VALUES (?, ?, ?, ?)`).bind(body.provider, chunk, data, body.updated_at))]; + if (revisionSyncSettings(env).enabled) { + if (!await commitRevision(db, "provider_catalog", statements)) return reply({ error: "revision_conflict" }, 409); + } else await db.batch(statements); return reply({ stored: true }); } default: @@ -250,7 +261,7 @@ const DOMAINS = Object.freeze({ "prompt-templates": { handle: handlePromptTemplates, maxBytes: 524288, operations: ["create", "get", "list"] }, limits: { handle: limits, maxBytes: 16384, operations: ["sync"] }, login: { handle: login, maxBytes: 4096, operations: ["check", "failure", "success"] }, - models: { handle: models, maxBytes: 4096, operations: ["list", "put"] }, + models: { handle: models, maxBytes: 4096, operations: ["list", "put", "revisions"] }, quotas: { handle: quotas, maxBytes: 16384, operations: ["list", "block"] }, workbench: { handle: workbench, maxBytes: 32768, operations: ["profiles", "save_profile", "reports", "save_report"] }, catalog: { handle: catalog, maxBytes: 262144, operations: ["list", "get", "put"] }, @@ -275,9 +286,9 @@ export async function handleControlStateRequest(request, env) { if (!object(body) || body.version !== 1) return reply({ error: "invalid_request" }, 400); } catch { return reply({ error: "invalid_request" }, 400); } try { - return await domain.handle(env.INTELLIGENCE_DB, body) ?? reply({ error: "invalid_request" }, 400); + return await domain.handle(env.INTELLIGENCE_DB, body, env) ?? reply({ error: "invalid_request" }, 400); } catch (error) { - logFailure("control_state_storage_failed", error, { domain: name, + logFailure("control_state_storage_failed", revisionSyncSettings(env).enabled ? new Error("Revision storage unavailable") : error, { domain: name, operation: domain.operations.includes(body.operation) ? body.operation : "unknown" }); return reply({ error: "storage_unavailable" }, 503); } diff --git a/worker/control-users-d1.mjs b/worker/control-users-d1.mjs index 31d67ed6..ef364728 100644 --- a/worker/control-users-d1.mjs +++ b/worker/control-users-d1.mjs @@ -2,6 +2,7 @@ * Durable dashboard accounts (username, key hash and metadata) in D1, reachable only * through the Container's private outbound handler. Fixed statements, no client SQL. */ +import { commitRevision, revisionSyncSettings } from "./config-revision.mjs"; import { logFailure } from "./log.mjs"; import { AUDIT_OPERATIONS, handleAuditOperation } from "./control-audit-d1.mjs"; @@ -204,10 +205,11 @@ export async function handleControlUsersRequest(request, env) { case "list": { if (!fields(body, ["version", "operation", "after", "limit"]) || !optionalText(body.after, 128) || !Number.isSafeInteger(body.limit) || body.limit < 1 || body.limit > MAX_PAGE) break; - const users = await readUsers(async columns => (await (body.after === null + const query = async columns => (await (body.after === null ? db.prepare(`SELECT ${columns} FROM control_users ORDER BY username LIMIT ?`).bind(body.limit) : db.prepare(`SELECT ${columns} FROM control_users WHERE username > ? ORDER BY username LIMIT ?`).bind(body.after, body.limit) - ).all()).results); + ).all()).results; + const users = revisionSyncSettings(env).enabled ? await read(() => query(COLUMNS)) : await readUsers(query); return reply({ version: 1, users }); } case "get": { @@ -232,7 +234,10 @@ export async function handleControlUsersRequest(request, env) { for (const end of [USER_FIELDS.length, USER_FIELDS.length - 1, USER_FIELDS.length - 2, 12]) { if (USER_FIELDS.slice(end).some(name => user[name] !== null)) throw new Error("Account controls require migration"); try { - await db.batch([upsertStatement(db, user, USER_FIELDS.slice(0, end)), audit(db, "upsert", "stored", user)]); + const statements = [upsertStatement(db, user, USER_FIELDS.slice(0, end)), audit(db, "upsert", "stored", user)]; + if (revisionSyncSettings(env).enabled) { + if (!await commitRevision(db, ["key_controls", "model_grants"], statements)) return reply({ error: "revision_conflict" }, 409); + } else await db.batch(statements); break; } catch (error) { if (!missingColumn(error) || end === 12) throw error; @@ -242,11 +247,23 @@ export async function handleControlUsersRequest(request, env) { } case "delete": { if (!fields(body, ["version", "operation", "username"]) || !text(body.username, 128)) break; - const [result] = await db.batch([db.prepare("DELETE FROM control_users WHERE username=?").bind(body.username), + const statements = [db.prepare("DELETE FROM control_users WHERE username=?").bind(body.username), db.prepare(`INSERT INTO control_user_audit (at, operation, outcome, username) VALUES (?, 'delete', CASE WHEN changes() = 1 THEN 'deleted' ELSE 'missing' END, ?)`) - .bind(new Date().toISOString(), body.username)]); - return reply({ version: 1, deleted: result.meta.changes === 1 }); + .bind(new Date().toISOString(), body.username)]; + let deleted; + if (revisionSyncSettings(env).enabled) { + // Capture the DELETE result from the same atomic revision batch. + let results; + const capture = { prepare: sql => db.prepare(sql), batch: async batch => { results = await db.batch(batch); } }; + if (!await commitRevision(capture, ["key_controls", "model_grants"], statements)) return reply({ error: "revision_conflict" }, 409); + // The final two statements are DELETE and its audit row. + deleted = results.at(-2).meta.changes === 1; + } else { + const [result] = await db.batch(statements); + deleted = result.meta.changes === 1; + } + return reply({ version: 1, deleted }); } case "touch": { if (!fields(body, ["version", "operation", "username", "last_used_at", "last_used_ip"]) diff --git a/worker/d1-schema.mjs b/worker/d1-schema.mjs index 6d50ae6e..4a0a37b0 100644 --- a/worker/d1-schema.mjs +++ b/worker/d1-schema.mjs @@ -10,7 +10,7 @@ export const REQUIRED_D1_TABLES = Object.freeze(["intelligence_policy", "intelli "usage_events", "usage_daily", "usage_batches", "media_jobs", "media_job_items", "shadow_eval_samples", "shadow_eval_results", "shadow_eval_config", "shadow_eval_policy_backups", "prompt_templates", - "config_snapshot_revisions", "config_snapshots", "config_snapshot_applications"]); + "config_snapshot_revisions", "config_snapshots", "config_snapshot_applications", "control_revisions"]); /** * Readiness for the D1 schema. A deploy that skipped `wrangler d1 migrations apply` reports