From 287f9298208bb8f3438c6cf1b6e30f20f9066aee Mon Sep 17 00:00:00 2001 From: Abhyudaya Baiju Tharakan Date: Fri, 9 Oct 2026 02:26:24 +0530 Subject: [PATCH] feat(routing): add opt-in reviewed configuration snapshots CONFIG_SNAPSHOTS_ENABLED (default off) adds admin snapshots of auto-route candidate orders with a paginated diff and a confirmed apply. Apply runs as one D1 batch that checks the revision and stored route state, writes the routes, bumps the revision and appends an audit record, so a stale review returns 409. Rollback is a new snapshot. Snapshots use migration 0015 and fail closed with JSON 503 when its tables are missing. --- app.py | 2 + docs/config-snapshots.md | 103 +++++++++ .../0015_config_snapshots.sql | 26 +++ package.json | 2 +- routes/config_snapshots.py | 58 ++++++ services/auto_route_d1.py | 21 ++ services/config_snapshots.py | 181 ++++++++++++++++ tests/fixtures/secret_firewall_coverage.json | 5 + tests/test_config_snapshots.py | 183 ++++++++++++++++ tests/test_config_snapshots_worker.mjs | 154 ++++++++++++++ worker/auto-routes-d1.mjs | 196 +++++++++++++++++- worker/container-env.mjs | 1 + worker/d1-schema.mjs | 3 +- 13 files changed, 926 insertions(+), 9 deletions(-) create mode 100644 docs/config-snapshots.md create mode 100644 intelligence-migrations/0015_config_snapshots.sql create mode 100644 routes/config_snapshots.py create mode 100644 services/config_snapshots.py create mode 100644 tests/test_config_snapshots.py create mode 100644 tests/test_config_snapshots_worker.mjs diff --git a/app.py b/app.py index 7acf98b7..58860680 100644 --- a/app.py +++ b/app.py @@ -18,6 +18,7 @@ is_api_request_path, login_required, ) +from routes.config_snapshots import register_config_snapshot_routes from routes.core import register_core_routes from routes.dashboard_security import register_dashboard_security_routes from routes.intelligence_media import register_intelligence_media_routes @@ -139,6 +140,7 @@ def create_app() -> Flask: register_core_routes(app) register_dashboard_security_routes(app) register_workbench_routes(app) + register_config_snapshot_routes(app) register_shadow_eval_routes(app, csrf, AuthService, MetricsService, ProxyService) register_knowledge_routes(app, csrf) register_gateway_mcp_routes(app, csrf) diff --git a/docs/config-snapshots.md b/docs/config-snapshots.md new file mode 100644 index 00000000..77586547 --- /dev/null +++ b/docs/config-snapshots.md @@ -0,0 +1,103 @@ +# Control configuration snapshots + +`CONFIG_SNAPSHOTS_ENABLED` defaults to false. All snapshot paths return 404 +while disabled, and existing auto-route saves keep their current payload, +storage and response behavior. No provider calls or generation are involved. + +The public administration paths run in Flask for both direct administration +and Worker-forwarded Container traffic. They use the existing session login, +administrator check and CSRF protection. They are not gateway-key API routes. +The existing private `auto_routes` adapter stores configuration in D1; there is +no new public Worker dispatch or private outbound endpoint. + +Before enabling, apply `intelligence-migrations/0015_config_snapshots.sql` +(`npx wrangler d1 migrations apply multillm-intelligence --remote`), and configure +`INTELLIGENCE_STORAGE_BACKEND=d1` for Flask. Enable the flag on both Worker +and Container. Enabled snapshots without durable D1 or any required table +fail closed with JSON HTTP 503. Enabled ordinary D1 route saves also fail +closed if the snapshot migration is missing. Local SQLite/PostgreSQL snapshot +storage is outside this initial private-D1 implementation boundary. + +## Review and apply + +`GET /admin/config/snapshots` returns `current_revision`, snapshot metadata, +and the latest 100 application audit records. It never includes stored +configuration documents or account objects. The only supported domain is +`auto_routes`; its revision begins at zero when the feature is introduced. + +Create a review with `POST /admin/config/snapshots`: + +```json +{"domain":"auto_routes","base_revision":0,"configuration":{"routes":[{"route_id":"auto:review","candidates":["openai:model-a","nanogpt:model-b"]}]}} +``` + +HTTP 201 returns the immutable snapshot metadata and ID. Creation requires +that `base_revision` still matches; it does not activate any route. +Configuration is a patch containing only the listed route IDs and their +ordered candidate model IDs. Omitted routes and seeded defaults remain active. +No deletion or wholesale replacement is performed. + +`GET /admin/config/snapshots//diff` returns only route IDs and the approved +before/after model ID lists, plus the base and current revisions. Flask displays +the effective seeded defaults when there is no durable override. Diff pages +contain at most 20 routes; use `?offset=` until `next_offset` is +null. A review spanning pages must be restarted if `current_revision` changes. +An unknown snapshot returns 404. + +Apply with `POST /admin/config/snapshots//apply`: + +```json +{"current_revision":0,"confirm":true} +``` + +Both literal `confirm=true` and the current integer revision are required. +A mismatch, stale review, repeat application, or patch that would exceed the +existing 200-row durable route read bound returns 409. The snapshot base +must equal the submitted revision. The transactional D1 batch compares the +revision and exact durable route state, writes the reviewed routes, increments +the revision, and appends an audit record with an opaque administrator digest. +A route write failure rolls back the revision, audit and route updates together. +Ordinary enabled D1 route saves advance the same revision. A fingerprint also +rejects a review if routes were saved while the flag was temporarily disabled. + +A successful apply clears this Container’s route cache. Other Containers can +retain their last-good route copy for the existing 30-second cache interval; +an outage preserves their last-good routing. There is no cluster-wide immediate +cache invalidation. Seeded candidate orders retired by the existing auto-route +service are rejected at creation, rather than claiming an apply that the route +service would subsequently replace with current defaults. + +To roll back, create a new snapshot containing the previous candidate order +at the current revision, review its diff and explicitly apply it. Historical +snapshots and application records are retained; rollback adds a revision. + +## Bounds, secrets and errors + +There are at most 100 snapshots per domain. Each configuration is bounded to +256 KiB, 200 routes and 16 distinct candidates per route; model IDs are bounded +to 256 characters. The existing private transport also bounds its complete +request envelope to 256 KiB, so configurations at that boundary require room +for metadata. Oversized requests return 413. Count exhaustion or a concurrent +revision change at creation returns 409. No automatic archival or deletion is +performed and there is no delete endpoint: once a domain has 100 snapshots, +creation returns 409 until unapplied snapshots are removed from D1 (applied +ones are referenced by their audit records). Application records are append-only and grow with +successful applies; list responses are bounded to the latest 100 records. + +Only the exact route configuration schema is accepted. Provider keys, gateway +keys, headers, prompts, connections, environment fields and arbitrary metadata +are rejected, not redacted into storage. Known credential-shaped identifiers +are also rejected. Snapshots contain configuration, a durable-state digest and +opaque administrator attribution only; no runtime account or environment +objects are read into snapshot documents. + +Malformed configuration returns 400, non-admin sessions return 403, and +unauthenticated sessions follow the existing login flow. Every snapshot response +uses `Cache-Control: no-store`. Storage failures return JSON 503 with +`config_snapshot_storage_unavailable`; they do not claim a completed apply. +Private submissions are never retried. If a response is lost after submission, +check the current revision and audit records before making a new review. + +Snapshot operations consume D1 reads/writes and bounded private transport +capacity. They make no upstream generation calls, but D1 usage can have storage +and request costs. diff --git a/intelligence-migrations/0015_config_snapshots.sql b/intelligence-migrations/0015_config_snapshots.sql new file mode 100644 index 00000000..a3d6a4ab --- /dev/null +++ b/intelligence-migrations/0015_config_snapshots.sql @@ -0,0 +1,26 @@ +-- Reviewed configuration only. No keys, headers, connection profiles or request content. +CREATE TABLE IF NOT EXISTS config_snapshot_revisions ( + domain TEXT PRIMARY KEY CHECK(domain = 'auto_routes'), + revision INTEGER NOT NULL CHECK(revision >= 0 AND revision <= 9007199254740991) +); +CREATE TABLE IF NOT EXISTS config_snapshots ( + id TEXT PRIMARY KEY, + domain TEXT NOT NULL CHECK(domain = 'auto_routes'), + base_revision INTEGER NOT NULL CHECK(base_revision >= 0), + configuration TEXT NOT NULL CHECK(json_valid(configuration)), + base_fingerprint TEXT NOT NULL CHECK(length(base_fingerprint) = 64), + created_at TEXT NOT NULL, + created_by TEXT NOT NULL, + size_bytes INTEGER NOT NULL CHECK(size_bytes > 0 AND size_bytes <= 262144) +); +CREATE INDEX IF NOT EXISTS config_snapshots_domain_created ON config_snapshots(domain, created_at, id); +CREATE TABLE IF NOT EXISTS config_snapshot_applications ( + id TEXT PRIMARY KEY, + domain TEXT NOT NULL CHECK(domain = 'auto_routes'), + snapshot_id TEXT NOT NULL REFERENCES config_snapshots(id), + base_revision INTEGER NOT NULL, + revision INTEGER NOT NULL, + applied_at TEXT NOT NULL, + applied_by TEXT NOT NULL, + UNIQUE(domain, revision) +); diff --git a/package.json b/package.json index 0335d9f0..ca8f8663 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "lint": "ruff check . --select E9,F63,F7,F82", "typecheck": "mypy --ignore-missing-imports --follow-imports=skip --disable-error-code import-untyped --disable-error-code var-annotated --disable-error-code arg-type --disable-error-code assignment --disable-error-code operator app.py routes services providers streaming", "test": "python -m pytest -q", - "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prompt_templates_worker.mjs", + "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs", "db:migrate": "python scripts/validate_sqlite_schema.py --apply", "db:validate": "python scripts/validate_sqlite_schema.py", "security:secrets": "python scripts/check_static_secrets.py", diff --git a/routes/config_snapshots.py b/routes/config_snapshots.py new file mode 100644 index 00000000..d015b0ec --- /dev/null +++ b/routes/config_snapshots.py @@ -0,0 +1,58 @@ +"""Administrator-only snapshot review; existing session and CSRF protections apply.""" +from functools import wraps + +from flask import jsonify, request + +from error_handlers import APIError + +from route_helpers import login_required +from routes.core import require_admin_dashboard_user +from services import config_snapshots + + +def json_errors(view): + @wraps(view) + def wrapped(*args, **kwargs): + try: + return view(*args, **kwargs) + except APIError as error: + return jsonify(error.to_dict()), error.status_code + return wrapped + + +def register_config_snapshot_routes(app): + def disabled_config_snapshots(): + if (request.path == "/admin/config/snapshots" or request.path.startswith("/admin/config/snapshots/")) and not config_snapshots.enabled(): + return jsonify({"error": "not_found"}), 404 + + # Feature-off requests must be 404 even before session and CSRF checks. + app.before_request_funcs.setdefault(None, []).insert(0, disabled_config_snapshots) + + @app.after_request + def private_config_snapshots(response): + if request.path == "/admin/config/snapshots" or request.path.startswith("/admin/config/snapshots/"): + response.headers["Cache-Control"] = "no-store" + return response + + @app.route("/admin/config/snapshots", methods=["GET", "POST"]) + @json_errors + @login_required + def admin_config_snapshots(): + user = require_admin_dashboard_user() + if request.method == "GET": + return jsonify(config_snapshots.call("snapshot_list")) + return jsonify(config_snapshots.create(request.get_json(silent=True), user, app.config["API_BASE_URLS"])), 201 + + @app.get("/admin/config/snapshots//diff") + @json_errors + @login_required + def admin_config_snapshot_diff(identifier): + require_admin_dashboard_user() + return jsonify(config_snapshots.diff(identifier, request.args.get("offset", "0"))) + + @app.post("/admin/config/snapshots//apply") + @json_errors + @login_required + def admin_config_snapshot_apply(identifier): + user = require_admin_dashboard_user() + return jsonify(config_snapshots.apply(identifier, request.get_json(silent=True), user)) diff --git a/services/auto_route_d1.py b/services/auto_route_d1.py index e7348a69..f3b6659e 100644 --- a/services/auto_route_d1.py +++ b/services/auto_route_d1.py @@ -86,3 +86,24 @@ def save_route(route_id, candidates, updated_at): def reset_cache(): with _lock: _cache["routes"], _cache["expires"] = None, 0.0 + + +def snapshot_request(operation, **values): + """Snapshot operations fail closed, without the routing read cache or a retry.""" + try: + if not using_d1(): + raise ValueError("Durable snapshots require D1") + result = intelligence_d1_store.request_private_intelligence( + {"operation": operation, **values}, endpoint="auto_routes") + if (not isinstance(result, dict) or type(result.get("version")) is not int + or result["version"] != 1 or "error" in result): + raise ValueError("Invalid snapshot storage response") + return result + except intelligence_d1_store.PrivateIntelligenceError as error: + if error.status in {400, 404, 409, 413}: + raise APIError("Snapshot request was rejected", error.status, + {"error": error.code}) from None + except Exception: + pass + raise APIError("Snapshot storage is unavailable; apply was not confirmed", 503, + {"error": "config_snapshot_storage_unavailable"}) from None diff --git a/services/config_snapshots.py b/services/config_snapshots.py new file mode 100644 index 00000000..0eab3432 --- /dev/null +++ b/services/config_snapshots.py @@ -0,0 +1,181 @@ +"""Bounded, configuration-only administration through the private D1 adapter.""" +import hashlib +import json +import os +import re + +from error_handlers import APIError +from services import auto_route_d1 +from services.auto_route_service import AutoRouteService, DEFAULT_AUTO_ROUTES, LEGACY_DEFAULT_AUTO_ROUTES +from services.redaction import SECRET_TEXT_PATTERNS + +MAX_SNAPSHOT_BYTES = 256 * 1024 +MAX_ROUTES = 200 +MAX_REVISION = 2**53 - 1 +ACTOR = re.compile(r"[0-9a-f]{64}\Z") +TIMESTAMP = re.compile(r"[0-9T:.+\-Z]{10,40}\Z") +ID = re.compile(r"[0-9a-f]{32}\Z") +CREDENTIAL_ID = re.compile(r"mllm_[A-Za-z0-9_-]{8,}|gh[pousr]_[A-Za-z0-9_]{8,}") + + +def enabled(): + return os.environ.get("CONFIG_SNAPSHOTS_ENABLED", "").strip().lower() in {"1", "true", "yes", "on"} + + +def fail(message, status=400, code="invalid_config_snapshot"): + raise APIError(message, status, {"error": code}) + + +def revision(value): + return type(value) is int and 0 <= value <= MAX_REVISION + + +def validate_configuration(value, base_urls): + if not isinstance(value, dict) or set(value) != {"routes"}: + fail("Only auto-route configuration is accepted") + try: + encoded = json.dumps(value, separators=(",", ":"), ensure_ascii=True, allow_nan=False).encode("utf-8") + except (TypeError, ValueError, RecursionError, UnicodeError): + fail("Invalid configuration") + if len(encoded) > MAX_SNAPSHOT_BYTES: + fail("Snapshot exceeds 256 KiB", 413) + rows = value["routes"] + if not isinstance(rows, list) or not 1 <= len(rows) <= MAX_ROUTES: + fail("Snapshot must contain between 1 and 200 routes") + routes, seen = [], set() + for row in rows: + if not isinstance(row, dict) or set(row) != {"route_id", "candidates"}: + fail("Only route IDs and candidate model IDs are accepted") + try: + route_id = AutoRouteService.normalize_route_id(row["route_id"]) + candidates = list(AutoRouteService.normalize_candidates(row["candidates"], base_urls)) + except ValueError: + fail("Invalid route or candidate model ID") + if tuple(candidates) in LEGACY_DEFAULT_AUTO_ROUTES.get(route_id, ()): + fail("Retired seeded candidate orders cannot be applied; review the current model IDs") + if route_id in seen: + fail("Duplicate route ID") + for identifier in [route_id, *candidates]: + if CREDENTIAL_ID.search(identifier) or any(pattern.search(identifier) for pattern in SECRET_TEXT_PATTERNS): + fail("Credential-shaped identifiers are not accepted") + seen.add(route_id) + routes.append({"route_id": route_id, "candidates": candidates}) + return {"routes": sorted(routes, key=lambda row: row["route_id"])} + + +def actor_identity(user): + # Stable administrator attribution without storing usernames or account/key objects. + name = user.get("username") + if not isinstance(name, str) or not name: + fail("Administrator identity is unavailable", 403) + return hashlib.sha256(name.encode("utf-8")).hexdigest() + + +def call(operation, **values): + if not enabled(): + fail("Not found", 404, "not_found") + result = auto_route_d1.snapshot_request(operation, **values) + if not valid_response(operation, result): + fail("Snapshot storage returned an invalid response", 503, "config_snapshot_storage_unavailable") + return result + + +def safe_identifier(value, pattern): + return isinstance(value, str) and bool(pattern.fullmatch(value)) and not CREDENTIAL_ID.search(value) and not any( + pattern.search(value) for pattern in SECRET_TEXT_PATTERNS) + + +def valid_metadata(row): + return (isinstance(row, dict) and set(row) == {"id", "domain", "base_revision", "created_at", "created_by", "size_bytes"} + and safe_identifier(row["id"], ID) and row["domain"] == "auto_routes" and revision(row["base_revision"]) + and safe_identifier(row["created_by"], ACTOR) and safe_identifier(row["created_at"], TIMESTAMP) + and type(row["size_bytes"]) is int and 0 < row["size_bytes"] <= MAX_SNAPSHOT_BYTES) + + +def valid_audit(row): + return (isinstance(row, dict) and set(row) == {"id", "snapshot_id", "base_revision", "revision", "applied_at", "applied_by"} + and safe_identifier(row["id"], ID) and safe_identifier(row["snapshot_id"], ID) + and revision(row["base_revision"]) and revision(row["revision"]) + and row["revision"] == row["base_revision"] + 1 + and safe_identifier(row["applied_by"], ACTOR) and safe_identifier(row["applied_at"], TIMESTAMP)) + + +def valid_change(row): + if not isinstance(row, dict) or set(row) != {"route_id", "before", "after"} or not safe_identifier(row["route_id"], auto_route_d1._ROUTE_ID): + return False + for name in ("before", "after"): + models = row[name] + if (not isinstance(models, list) or not (0 if name == "before" else 1) <= len(models) <= 16 + or not all(safe_identifier(model, auto_route_d1._MODEL_ID) for model in models) + or len(set(models)) != len(models)): + return False + return True + + +def valid_response(operation, result): + if operation == "snapshot_create": + return set(result) == {"version", "snapshot"} and valid_metadata(result["snapshot"]) + if operation == "snapshot_apply": + return (set(result) == {"version", "applied", "revision", "application_id"} + and result["applied"] is True and revision(result["revision"]) and safe_identifier(result["application_id"], ID)) + if operation == "snapshot_list": + return (set(result) == {"version", "domain", "current_revision", "snapshots", "applications"} + and result["domain"] == "auto_routes" and revision(result["current_revision"]) + and isinstance(result["snapshots"], list) and len(result["snapshots"]) <= 100 + and all(valid_metadata(row) for row in result["snapshots"]) + and isinstance(result["applications"], list) and len(result["applications"]) <= 100 + and all(valid_audit(row) for row in result["applications"])) + if operation == "snapshot_diff": + return (set(result) == {"version", "id", "domain", "base_revision", "current_revision", "changes", "next_offset"} + and safe_identifier(result["id"], ID) and result["domain"] == "auto_routes" + and revision(result["base_revision"]) and revision(result["current_revision"]) + and isinstance(result["changes"], list) and len(result["changes"]) <= 20 + and all(valid_change(row) for row in result["changes"]) + and (result["next_offset"] is None or type(result["next_offset"]) is int and 0 < result["next_offset"] < MAX_ROUTES)) + return False + + +def create(body, user, base_urls): + if not isinstance(body, dict) or set(body) != {"domain", "base_revision", "configuration"}: + fail("Expected domain, base_revision and configuration") + if body["domain"] != "auto_routes" or not revision(body["base_revision"]) or body["base_revision"] >= MAX_REVISION: + fail("Invalid snapshot domain or base revision") + configuration = validate_configuration(body["configuration"], base_urls) + values = dict(domain="auto_routes", base_revision=body["base_revision"], + configuration=configuration, actor=actor_identity(user)) + # The existing private transport also bounds the complete envelope to 256 KiB. + if len(json.dumps({"version": 1, "operation": "snapshot_create", **values}, separators=(",", ":")).encode()) > MAX_SNAPSHOT_BYTES: + fail("Snapshot exceeds the private transport byte limit", 413) + result = call("snapshot_create", **values) + if result["snapshot"]["base_revision"] != body["base_revision"]: + fail("Snapshot storage did not confirm the base revision", 503, "config_snapshot_storage_unavailable") + return result + + +def diff(identifier, offset): + if not ID.fullmatch(identifier): + fail("Not found", 404, "not_found") + if not isinstance(offset, str) or not offset.isascii() or not offset.isdecimal() or len(offset) > 3 or int(offset) > MAX_ROUTES: + fail("Invalid diff offset") + result = call("snapshot_diff", id=identifier, offset=int(offset)) + for change in result.get("changes", []): + route_id = change["route_id"] + before = tuple(change["before"]) + if route_id in DEFAULT_AUTO_ROUTES and (not before or before in LEGACY_DEFAULT_AUTO_ROUTES.get(route_id, ())): + change["before"] = list(DEFAULT_AUTO_ROUTES[route_id]) + return result + + +def apply(identifier, body, user): + if not ID.fullmatch(identifier): + fail("Not found", 404, "not_found") + if (not isinstance(body, dict) or set(body) != {"current_revision", "confirm"} + or body.get("confirm") is not True or not revision(body.get("current_revision")) + or body["current_revision"] >= MAX_REVISION): + fail("Explicit confirm=true and current_revision are required", 409, "revision_conflict") + result = call("snapshot_apply", id=identifier, current_revision=body["current_revision"], + confirm=True, actor=actor_identity(user)) + if result["revision"] != body["current_revision"] + 1: + fail("Snapshot storage did not confirm the apply", 503, "config_snapshot_storage_unavailable") + auto_route_d1.reset_cache() + return result diff --git a/tests/fixtures/secret_firewall_coverage.json b/tests/fixtures/secret_firewall_coverage.json index 4e99aa60..9e8b6138 100644 --- a/tests/fixtures/secret_firewall_coverage.json +++ b/tests/fixtures/secret_firewall_coverage.json @@ -7,6 +7,11 @@ "routes/cascades.py": [ "app.route('/admin/cascades', methods=['GET', 'PUT'])" ], + "routes/config_snapshots.py": [ + "app.get('/admin/config/snapshots//diff')", + "app.post('/admin/config/snapshots//apply')", + "app.route('/admin/config/snapshots', methods=['GET', 'POST'])" + ], "routes/core.py": [ "app.route('/')", "app.route('/admin/metrics/requests')", diff --git a/tests/test_config_snapshots.py b/tests/test_config_snapshots.py new file mode 100644 index 00000000..070b0310 --- /dev/null +++ b/tests/test_config_snapshots.py @@ -0,0 +1,183 @@ +"""Snapshot routes use only scoped configuration and private durable storage.""" +import importlib +import json +from unittest import TestCase +from unittest.mock import patch + +from flask import url_for + +import pytest + +from error_handlers import APIError +from services import auto_route_d1, intelligence_d1_store +from services.intelligence_d1_store import PrivateIntelligenceError + + +checks = TestCase() + + +@pytest.fixture +def client(monkeypatch, tmp_path): + monkeypatch.setenv("FLASK_SECRET_KEY", "synthetic-snapshot-session") + monkeypatch.setenv("JWT_SECRET", "synthetic-snapshot-jwt") + monkeypatch.setenv("ADMIN_API_KEY", "synthetic-snapshot-admin") + monkeypatch.setenv("AUTH_DB_PATH", str(tmp_path / "auth.sqlite3")) + monkeypatch.setenv("MODEL_REGISTRY_DB_PATH", str(tmp_path / "models.sqlite3")) + monkeypatch.setenv("RATE_LIMIT_DB_PATH", str(tmp_path / "limits.sqlite3")) + monkeypatch.delenv("CONFIG_SNAPSHOTS_ENABLED", raising=False) + with patch("config.load_runtime_env"), patch("env_loader.load_runtime_env"), \ + patch("services.usage_ledger.start"), patch("requests.Session.send", side_effect=AssertionError("No network")): + module = importlib.import_module("app") + with patch.object(module, "load_runtime_env"): + app = module.create_app() + app.config.update(TESTING=True, WTF_CSRF_ENABLED=False) + with patch.object(module.AuthService, "is_authenticated", return_value=True), \ + patch.object(module.AuthService, "get_current_user", return_value={"username": "operator", "is_admin": True}): + yield app.test_client() + + +def test_disabled_registered_paths_are_404_without_storage(client, monkeypatch): + with patch.object(intelligence_d1_store, "request_private_intelligence") as private: + for method, path in [("get", ""), ("post", ""), ("get", "/" + "a" * 32 + "/diff"), + ("post", "/" + "a" * 32 + "/apply")]: + checks.assertEqual(getattr(client, method)('/admin/config/snapshots' + path, json={}).status_code, 404) + private.assert_not_called() + + +def test_enabled_requires_admin(client, monkeypatch): + monkeypatch.setenv("CONFIG_SNAPSHOTS_ENABLED", "true") + with patch("services.auth_service.AuthService.get_current_user", return_value={"is_admin": False}), \ + patch.object(intelligence_d1_store, "request_private_intelligence") as private: + checks.assertEqual(client.get('/admin/config/snapshots').status_code, 403) + private.assert_not_called() + + +def test_missing_storage_returns_clear_json_503(client, monkeypatch): + monkeypatch.setenv("CONFIG_SNAPSHOTS_ENABLED", "true") + monkeypatch.setenv("INTELLIGENCE_STORAGE_BACKEND", "d1") + with patch.object(intelligence_d1_store, "request_private_intelligence", + side_effect=PrivateIntelligenceError(503, "storage_unavailable")): + response = client.get("/admin/config/snapshots") + checks.assertEqual(response.status_code, 503) + checks.assertEqual(response.json['error'], 'config_snapshot_storage_unavailable') + + +CONFIG = {"routes": [{"route_id": "auto:review", "candidates": ["openai:model-a"]}]} + + +def test_registered_create_diff_apply_use_private_adapter(client, monkeypatch): + monkeypatch.setenv("CONFIG_SNAPSHOTS_ENABLED", "true") + monkeypatch.setenv("INTELLIGENCE_STORAGE_BACKEND", "d1") + identifier = "a" * 32 + replies = [{"version": 1, "snapshot": {"id": identifier, "domain": "auto_routes", "base_revision": 0, "created_at": "2026-10-09T00:00:00Z", "created_by": "a" * 64, "size_bytes": 80}}, + {"version": 1, "id": "a" * 32, "domain": "auto_routes", "base_revision": 0, "next_offset": None, "current_revision": 0, "changes": [{"route_id": "auto:review", "before": [], "after": ["openai:model-a"]}]}, + {"version": 1, "applied": True, "revision": 1, "application_id": "b" * 32}] + with patch.object(intelligence_d1_store, "request_private_intelligence", side_effect=replies) as private, \ + patch.object(auto_route_d1, "reset_cache") as reset: + created = client.post("/admin/config/snapshots", json={"domain": "auto_routes", "base_revision": 0, "configuration": CONFIG}) + checks.assertEqual(created.status_code, 201) + diff = client.get(f"/admin/config/snapshots/{identifier}/diff") + checks.assertEqual(diff.status_code, 200) + applied = client.post(f"/admin/config/snapshots/{identifier}/apply", json={"current_revision": 0, "confirm": True}) + checks.assertTrue(applied.status_code == 200 and applied.json['revision'] == 1) + reset.assert_called_once() + checks.assertEqual([call.kwargs['endpoint'] for call in private.call_args_list], ['auto_routes'] * 3) + checks.assertEqual(private.call_args_list[0].args[0]['operation'], 'snapshot_create') + checks.assertEqual(len(private.call_args.args[0]['actor']), 64) + checks.assertNotIn('operator', json.dumps(private.call_args.args)) + + +@pytest.mark.parametrize("extra", ["api_key", "gateway_key", "headers", "prompts", "connections", "env", "provider_keys"]) +def test_secrets_and_unapproved_fields_never_reach_storage(client, monkeypatch, extra): + monkeypatch.setenv("CONFIG_SNAPSHOTS_ENABLED", "true") + configuration = {**CONFIG, extra: "synthetic-private-value"} + with patch.object(intelligence_d1_store, "request_private_intelligence") as private: + response = client.post("/admin/config/snapshots", json={"domain": "auto_routes", "base_revision": 0, "configuration": configuration}) + checks.assertEqual(response.status_code, 400) + checks.assertNotIn('synthetic-private-value', response.get_data(as_text=True)) + private.assert_not_called() + + +@pytest.mark.parametrize("body", [{}, {"confirm": "true", "current_revision": 0}, {"confirm": True}, + {"confirm": True, "current_revision": False}]) +def test_apply_requires_literal_confirmation_and_revision(client, monkeypatch, body): + monkeypatch.setenv("CONFIG_SNAPSHOTS_ENABLED", "true") + with patch.object(intelligence_d1_store, "request_private_intelligence") as private: + checks.assertEqual(client.post('/admin/config/snapshots/' + 'a' * 32 + '/apply', json=body).status_code, 409) + private.assert_not_called() + + +def test_conflict_and_uncertain_apply_do_not_claim_success(client, monkeypatch): + monkeypatch.setenv("CONFIG_SNAPSHOTS_ENABLED", "true") + monkeypatch.setenv("INTELLIGENCE_STORAGE_BACKEND", "d1") + for error, status in [(PrivateIntelligenceError(409, "revision_conflict"), 409), (RuntimeError("private detail"), 503)]: + with patch.object(intelligence_d1_store, "request_private_intelligence", side_effect=error) as private, \ + patch.object(auto_route_d1, "reset_cache") as reset: + result = client.post("/admin/config/snapshots/" + "a" * 32 + "/apply", json={"confirm": True, "current_revision": 0}) + checks.assertTrue(result.status_code == status and 'private detail' not in result.get_data(as_text=True)) + checks.assertEqual(private.call_count, 1) + reset.assert_not_called() + + +def test_default_save_payload_and_cache_are_unchanged(monkeypatch): + monkeypatch.delenv("CONFIG_SNAPSHOTS_ENABLED", raising=False) + auto_route_d1.reset_cache() + with patch.object(intelligence_d1_store, "request_private_intelligence", return_value={"version": 1, "stored": True}) as private: + auto_route_d1.save_route("auto:x", ["openai:model"], "2026-10-09T00:00:00Z") + checks.assertEqual(private.call_args.args[0], {'operation': 'put', 'route_id': 'auto:x', 'candidates': ['openai:model'], 'updated_at': '2026-10-09T00:00:00Z'}) + checks.assertEqual(auto_route_d1.stored_routes()['auto:x'][0], ('openai:model',)) + auto_route_d1.reset_cache() + + +def test_limits_and_secret_model_ids(): + from services.config_snapshots import validate_configuration + for config in [{"routes": []}, {"routes": CONFIG["routes"] * 201}, + {"routes": [{"route_id": "auto:x", "candidates": ["openai:sk-syntheticsecret123"]}]}, + {"routes": [{"route_id": "auto:x", "candidates": ["openai:model"], "api_key": "synthetic"}]}]: + with pytest.raises(APIError): + validate_configuration(config, {"openai": "https://synthetic.invalid"}) + checks.assertEqual(validate_configuration(CONFIG, {'openai': 'https://synthetic.invalid'}), CONFIG) + + +def test_disabled_paths_are_404_even_with_csrf_enabled(client): + client.application.config["WTF_CSRF_ENABLED"] = True + checks.assertEqual(client.post('/admin/config/snapshots', json={}).status_code, 404) + + +def test_enabled_apply_keeps_csrf_protection(client, monkeypatch): + monkeypatch.setenv("CONFIG_SNAPSHOTS_ENABLED", "true") + client.application.config["WTF_CSRF_ENABLED"] = True + with patch.object(intelligence_d1_store, "request_private_intelligence") as private: + checks.assertEqual(client.post('/admin/config/snapshots/' + 'a' * 32 + '/apply', json={'confirm': True, 'current_revision': 0}).status_code, 400) + private.assert_not_called() + + +def test_diff_reviews_effective_seeded_defaults(client, monkeypatch): + from services.auto_route_service import DEFAULT_AUTO_ROUTES + monkeypatch.setenv("CONFIG_SNAPSHOTS_ENABLED", "true") + monkeypatch.setenv("INTELLIGENCE_STORAGE_BACKEND", "d1") + result = {"version": 1, "id": "a" * 32, "domain": "auto_routes", "base_revision": 0, "next_offset": None, "current_revision": 0, "changes": [{"route_id": "auto:image", "before": [], "after": ["openai:model"]}]} + with patch.object(intelligence_d1_store, "request_private_intelligence", return_value=result): + response = client.get("/admin/config/snapshots/" + "a" * 32 + "/diff") + checks.assertEqual(response.json['changes'][0]['before'], list(DEFAULT_AUTO_ROUTES['auto:image'])) + + +@pytest.mark.parametrize("result", [{"version": 1}, {"version": 1, "api_key": "synthetic-private-value"}]) +def test_partial_or_secret_storage_response_is_not_success(client, monkeypatch, result): + monkeypatch.setenv("CONFIG_SNAPSHOTS_ENABLED", "true") + monkeypatch.setenv("INTELLIGENCE_STORAGE_BACKEND", "d1") + with patch.object(intelligence_d1_store, "request_private_intelligence", return_value=result): + response = client.post("/admin/config/snapshots", json={"domain": "auto_routes", "base_revision": 0, "configuration": CONFIG}) + checks.assertEqual(response.status_code, 503) + checks.assertNotIn("synthetic-private-value", response.get_data(as_text=True)) + + +def test_unauthenticated_review_does_not_reach_storage(client, monkeypatch): + monkeypatch.setenv("CONFIG_SNAPSHOTS_ENABLED", "true") + with patch("services.auth_service.AuthService.is_authenticated", return_value=False), \ + patch.object(intelligence_d1_store, "request_private_intelligence") as private: + response = client.get("/admin/config/snapshots") + checks.assertEqual(response.status_code, 302) + with client.application.test_request_context(): + checks.assertEqual(response.headers["Location"], url_for("login", next="/admin/config/snapshots")) + private.assert_not_called() diff --git a/tests/test_config_snapshots_worker.mjs b/tests/test_config_snapshots_worker.mjs new file mode 100644 index 00000000..c15c012f --- /dev/null +++ b/tests/test_config_snapshots_worker.mjs @@ -0,0 +1,154 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { convertV4MiniflareOptions, Miniflare } from "miniflare"; +import { handleIntelligenceOutbound } from "../worker/intelligence-outbound.mjs"; +import { applyMigrations } from "./d1_migrations.mjs"; + +const config = candidates => ({ routes: [{ route_id: "auto:review", candidates }] }); +async function database(t, { missing = false, enabled = true } = {}) { + const mf = new Miniflare(convertV4MiniflareOptions({ modules: true, script: "export default {fetch(){return new Response(\"ok\")}}", d1Databases: ["INTELLIGENCE_DB"] })); + t.after(() => mf.dispose()); + const db = await mf.getD1Database("INTELLIGENCE_DB"); + await applyMigrations(db, { skip: missing ? ["0015_config_snapshots.sql"] : [] }); + const env = { INTELLIGENCE_DB: db, ...(enabled ? { CONFIG_SNAPSHOTS_ENABLED: "true" } : {}) }; + const call = async body => { + const response = await handleIntelligenceOutbound(new Request("http://intelligence.internal/v1/auto-routes", { + method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ version: 1, ...body }) }), env); + return { status: response.status, body: await response.json() }; + }; + const create = async (revision, candidates = ["openai:model-a"]) => call({ operation: "snapshot_create", domain: "auto_routes", base_revision: revision, configuration: config(candidates), actor: "a".repeat(64) }); + const apply = async (id, revision, confirm = true) => call({ operation: "snapshot_apply", id, current_revision: revision, confirm, actor: "a".repeat(64) }); + return { db, env, call, create, apply }; +} + +test("disabled snapshots make no statements and legacy saves stay unchanged", async t => { + const { db, call } = await database(t, { enabled: false, missing: true }); + assert.equal((await call({ operation: "snapshot_list" })).status, 404); + const route = { operation: "put", route_id: "auto:old", candidates: ["openai:model"], updated_at: "2026-10-09T00:00:00Z" }; + assert.deepEqual((await call(route)).body, { version: 1, stored: true }); + assert.deepEqual((await call({ operation: "list" })).body.routes, [{ route_id: route.route_id, candidates: route.candidates, updated_at: route.updated_at }]); + assert.equal((await db.prepare("SELECT COUNT(*) AS n FROM auto_routes").first()).n, 1); +}); + +test("missing migration fails closed including enabled legacy save", async t => { + const { call, db } = await database(t, { missing: true }); + assert.equal((await call({ operation: "snapshot_list" })).status, 503); + assert.equal((await call({ operation: "put", route_id: "auto:x", candidates: ["openai:model"], updated_at: "2026-10-09T00:00:00Z" })).status, 503); + assert.equal((await db.prepare("SELECT COUNT(*) AS n FROM auto_routes").first()).n, 0); +}); + +test("preview is read only, concurrent CAS applies once, rollback adds history", async t => { + const { db, call, create, apply } = await database(t); + const [first, other] = await Promise.all([create(0), create(0, ["openai:model-b"])]); + assert.equal(first.status, 200); + assert.equal(other.status, 200); + const id = first.body.snapshot.id; + const listed = await call({ operation: "snapshot_list" }); + assert.equal(listed.body.current_revision, 0); + assert.equal(listed.body.snapshots.length, 2); + assert.ok(listed.body.snapshots.every(row => !Object.hasOwn(row, "configuration"))); + const diff = await call({ operation: "snapshot_diff", id }); + assert.deepEqual(diff.body.changes, [{ route_id: "auto:review", before: [], after: ["openai:model-a"] }]); + assert.equal((await call({ operation: "list" })).body.routes.length, 0); + assert.equal((await apply(id, 0, false)).status, 409); + const attempts = await Promise.all([apply(id, 0), apply(other.body.snapshot.id, 0)]); + assert.deepEqual(attempts.map(item => item.status).sort(), [200, 409]); + const current = (await call({ operation: "list" })).body.routes[0].candidates; + assert.equal((await create(0)).status, 409); + const next = await create(1, ["openai:changed"]); + assert.equal((await apply(next.body.snapshot.id, 1)).body.revision, 2); + const rollback = await create(2, current); + assert.equal((await apply(rollback.body.snapshot.id, 2)).body.revision, 3); + assert.deepEqual((await call({ operation: "list" })).body.routes[0].candidates, current); + assert.equal((await db.prepare("SELECT COUNT(*) AS n FROM config_snapshot_applications").first()).n, 3); + assert.equal((await db.prepare("SELECT COUNT(*) AS n FROM config_snapshots").first()).n, 4); + assert.equal((await apply(rollback.body.snapshot.id, 3)).status, 409); +}); + +test("ordinary enabled save invalidates pending snapshots and retains old routes", async t => { + const { call, create, apply, db } = await database(t); + await db.prepare("INSERT INTO auto_routes VALUES (?, ?, ?)").bind("auto:untouched", "[\"openai:old\"]", "2026-10-09T00:00:00Z").run(); + const pending = await create(0); + assert.equal((await call({ operation: "put", route_id: "auto:ordinary", candidates: ["openai:new"], updated_at: "2026-10-09T00:00:00Z" })).status, 200); + assert.equal((await apply(pending.body.snapshot.id, 0)).status, 409); + const reviewed = await create(1); + assert.equal((await apply(reviewed.body.snapshot.id, 1)).status, 200); + assert.equal((await call({ operation: "list" })).body.routes.length, 3); +}); + +test("unapproved secrets and invalid configuration are rejected without persistence", async t => { + const { create, call, db } = await database(t); + for (const extra of ["api_key", "headers", "prompts", "connections", "env", "gateway_keys"]) { + const result = await call({ operation: "snapshot_create", domain: "auto_routes", base_revision: 0, + configuration: { ...config(["openai:model"]), [extra]: "synthetic-secret" }, actor: "a".repeat(64) }); + assert.equal(result.status, 400); + assert.equal(JSON.stringify(result.body).includes("synthetic-secret"), false); + } + for (const models of [["openai:sk-syntheticsecret123"], ["openai:model", "openai:model"], ["auto:nested"], ["bad"], []]) { + assert.equal((await create(0, models)).status, 400); + } + assert.equal((await db.prepare("SELECT COUNT(*) AS n FROM config_snapshots").first()).n, 0); +}); + +test("snapshot count and bytes are bounded without automatic deletion", async t => { + const { call, create, db } = await database(t); + const huge = { routes: Array.from({ length: 200 }, (_, i) => ({ route_id: `auto:r${i}`, candidates: Array.from({ length: 16 }, (_, j) => `openai:m${j}${"x".repeat(230)}`) })) }; + assert.equal((await call({ operation: "snapshot_create", domain: "auto_routes", base_revision: 0, configuration: huge, actor: "a".repeat(64) })).status, 413); + await db.batch(Array.from({ length: 99 }, (_, i) => db.prepare("INSERT INTO config_snapshots (id,domain,base_revision,configuration,created_at,created_by,size_bytes,base_fingerprint) VALUES (?,?,0,?,?,?,?,?)") + .bind(i.toString(16).padStart(32, "0"), "auto_routes", JSON.stringify(config(["openai:model"])), "2026-10-09T00:00:00Z", "a".repeat(64), 79, "b".repeat(64)))); + const finalSlots = await Promise.all([create(0), create(0)]); + assert.deepEqual(finalSlots.map(result => result.status).sort(), [200, 409]); + assert.equal((await create(0)).status, 409); + assert.equal((await db.prepare("SELECT COUNT(*) AS n FROM config_snapshots").first()).n, 100); +}); + +test("failed atomic apply keeps route, revision and audit unchanged", async t => { + const { db, create, apply, call } = await database(t); + const snapshot = await create(0); + await db.prepare("CREATE TRIGGER refuse_route BEFORE INSERT ON auto_routes BEGIN SELECT RAISE(ABORT, 'synthetic failure'); END").run(); + const result = await apply(snapshot.body.snapshot.id, 0); + assert.equal(result.status, 503); + assert.equal(JSON.stringify(result.body).includes("synthetic failure"), false); + assert.equal((await call({ operation: "snapshot_list" })).body.current_revision, 0); + assert.equal((await db.prepare("SELECT COUNT(*) AS n FROM config_snapshot_applications").first()).n, 0); +}); + + +test("saves while disabled invalidate old reviews when re-enabled", async t => { + const { env, call, create, apply } = await database(t); + const pending = await create(0); + delete env.CONFIG_SNAPSHOTS_ENABLED; + assert.equal((await call({ operation: "put", route_id: "auto:review", candidates: ["openai:newer"], updated_at: "2026-10-09T00:00:00Z" })).status, 200); + env.CONFIG_SNAPSHOTS_ENABLED = "true"; + assert.equal((await apply(pending.body.snapshot.id, 0)).status, 409); + assert.deepEqual((await call({ operation: "list" })).body.routes[0].candidates, ["openai:newer"]); +}); + + +test("apply cannot exceed the active route read bound", async t => { + const { db, create, apply, call } = await database(t); + await db.batch(Array.from({ length: 200 }, (_, i) => db.prepare("INSERT INTO auto_routes VALUES (?, ?, ?)") + .bind(`auto:r${i}`, "[\"openai:model\"]", "2026-10-09T00:00:00Z"))); + const pending = await create(0); + assert.equal((await apply(pending.body.snapshot.id, 0)).status, 409); + assert.equal((await call({ operation: "snapshot_list" })).body.current_revision, 0); + assert.equal((await db.prepare("SELECT COUNT(*) AS n FROM auto_routes").first()).n, 200); +}); + +test("diff pages are bounded and malformed legacy storage never leaks", async t => { + const { db, call } = await database(t); + const configuration = { routes: Array.from({ length: 21 }, (_, i) => ({ route_id: `auto:r${i.toString().padStart(2, "0")}`, candidates: ["openai:model"] })) }; + const made = await call({ operation: "snapshot_create", domain: "auto_routes", base_revision: 0, configuration, actor: "a".repeat(64) }); + const id = made.body.snapshot.id; + const first = await call({ operation: "snapshot_diff", id }); + assert.equal(first.body.changes.length, 20); + assert.equal(first.body.next_offset, 20); + const last = await call({ operation: "snapshot_diff", id, offset: 20 }); + assert.equal(last.body.changes.length, 1); + assert.equal(last.body.next_offset, null); + await db.prepare("INSERT INTO auto_routes VALUES (?, ?, ?)") + .bind("auto:r00", "[\"openai:sk-syntheticsecret123\"]", "2026-10-09T00:00:00Z").run(); + const broken = await call({ operation: "snapshot_diff", id }); + assert.equal(broken.status, 503); + assert.equal(JSON.stringify(broken.body).includes("syntheticsecret123"), false); +}); diff --git a/worker/auto-routes-d1.mjs b/worker/auto-routes-d1.mjs index 78cd8c98..31d39cc2 100644 --- a/worker/auto-routes-d1.mjs +++ b/worker/auto-routes-d1.mjs @@ -11,6 +11,25 @@ const MODEL_ID = /^[A-Za-z0-9][A-Za-z0-9._:/+@-]{0,255}$/; const MAX_CANDIDATES = 16; const MAX_ROUTES = 200; const MAX_BODY_BYTES = 16384; +const SNAPSHOT_BYTES = 256 * 1024; +const SNAPSHOT_COUNT = 100; +const DIFF_PAGE = 20; +const DOMAIN = "auto_routes"; +const SNAPSHOT_ID = /^[0-9a-f]{32}$/; +const ACTOR = /^[0-9a-f]{64}$/; +const SECRET_ID = /sk-[A-Za-z0-9_-]{8,}|AIza[0-9A-Za-z_-]{12,}|mllm_[A-Za-z0-9_-]{8,}|gh[pousr]_[A-Za-z0-9_]{8,}/; +const snapshotEnabled = env => ["1", "true", "yes", "on"].includes(String(env.CONFIG_SNAPSHOTS_ENABLED || "").trim().toLowerCase()); +const validRevision = value => Number.isSafeInteger(value) && value >= 0 && value <= Number.MAX_SAFE_INTEGER - 1; +const identifier = () => crypto.randomUUID().replaceAll("-", ""); +const metadataColumns = "id, domain, base_revision, created_at, created_by, size_bytes"; +const routeStateSQL = `SELECT json_group_array(json_object('route_id',route_id,'candidates',candidates,'updated_at',updated_at)) + FROM (SELECT route_id,candidates,updated_at FROM auto_routes ORDER BY route_id)`; +async function routeState(db) { + const row = await db.prepare(`SELECT (${routeStateSQL}) AS state`).first(); + const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(row.state)); + const fingerprint = Array.from(new Uint8Array(digest), byte => byte.toString(16).padStart(2, "0")).join(""); + return { state: row.state, fingerprint }; +} const TIMESTAMP = /^[0-9T:.+\-Z]{10,40}$/; const reply = (value, status = 200) => Response.json(value.error ? { version: 1, error: { code: value.error, message: "Auto route storage operation failed" } } @@ -30,11 +49,17 @@ export async function handleAutoRoutesRequest(request, env) { if (!env.INTELLIGENCE_DB) return reply({ error: "storage_unavailable" }, 503); let body; try { - body = JSON.parse(await boundedBody(request, MAX_BODY_BYTES)); - if (!body || Array.isArray(body) || typeof body !== "object" || body.version !== 1) return reply({ error: "invalid_request" }, 400); - } catch { return reply({ error: "invalid_request" }, 400); } + body = JSON.parse(await boundedBody(request, snapshotEnabled(env) ? SNAPSHOT_BYTES : MAX_BODY_BYTES)); + if (!body || Array.isArray(body) || typeof body !== "object" || body.version !== 1) { + return reply({ error: "invalid_request" }, 400); + } + } catch (error) { + return reply({ error: "invalid_request" }, snapshotEnabled(env) && error?.message === "Oversized body" ? 413 : 400); + } + if (String(body.operation).startsWith("snapshot_") && !snapshotEnabled(env)) return reply({ error: "not_found" }, 404); const db = env.INTELLIGENCE_DB; try { + if (String(body.operation).startsWith("snapshot_")) return await snapshotOperation(db, body); if (body.operation === "list" && fields(body, ["version", "operation"])) { const { results } = await db.prepare(`SELECT route_id, candidates, updated_at FROM auto_routes ORDER BY route_id LIMIT ${MAX_ROUTES}`).all(); @@ -44,14 +69,171 @@ export async function handleAutoRoutesRequest(request, env) { if (body.operation === "put" && fields(body, ["version", "operation", "route_id", "candidates", "updated_at"]) && typeof body.route_id === "string" && ROUTE_ID.test(body.route_id) && validCandidates(body.candidates) && typeof body.updated_at === "string" && TIMESTAMP.test(body.updated_at)) { - await db.prepare(`INSERT INTO auto_routes (route_id, candidates, updated_at) VALUES (?, ?, ?) - ON CONFLICT(route_id) DO UPDATE SET candidates=excluded.candidates, updated_at=excluded.updated_at`) - .bind(body.route_id, JSON.stringify(body.candidates), body.updated_at).run(); + if (snapshotEnabled(env)) { + await revisionedSave(db, body); + } else { + await db.prepare(`INSERT INTO auto_routes (route_id, candidates, updated_at) VALUES (?, ?, ?) + ON CONFLICT(route_id) DO UPDATE SET candidates=excluded.candidates, updated_at=excluded.updated_at`) + .bind(body.route_id, JSON.stringify(body.candidates), body.updated_at).run(); + } return reply({ version: 1, stored: true }); } return reply({ error: "invalid_request" }, 400); } catch (error) { - logFailure("auto_route_storage_failed", error, { operation: body.operation === "list" ? "list" : "put" }); + logFailure("auto_route_storage_failed", snapshotEnabled(env) ? new Error("Snapshot storage is unavailable") : error, + { operation: body.operation === "list" ? "list" : "put" }); return reply({ error: "storage_unavailable" }, 503); } } + + +function validConfiguration(value) { + if (!value || typeof value !== "object" || Array.isArray(value) || !fields(value, ["routes"]) + || !Array.isArray(value.routes) || !value.routes.length || value.routes.length > MAX_ROUTES) return false; + const ids = new Set(); + return value.routes.every(row => { + if (!row || typeof row !== "object" || !fields(row, ["route_id", "candidates"]) + || typeof row.route_id !== "string" || !ROUTE_ID.test(row.route_id) || ids.has(row.route_id) + || SECRET_ID.test(row.route_id) || !validCandidates(row.candidates) + || row.candidates.some(model => SECRET_ID.test(model) || !model.includes(":") || model.startsWith("auto:") + || model.includes("://"))) return false; + ids.add(row.route_id); + return true; + }); +} + +async function requireSnapshotSchema(db) { + // Probe every required table before a write; deployment does not apply migrations. + await db.prepare(`SELECT (SELECT COUNT(*) FROM config_snapshots) AS snapshots, + (SELECT COUNT(*) FROM config_snapshot_applications) AS applications, + (SELECT COUNT(*) FROM config_snapshot_revisions) AS domains`).first(); +} + +const initializeRevision = db => db.prepare(`INSERT OR IGNORE INTO config_snapshot_revisions + (domain, revision) VALUES ('auto_routes', 0)`); +const changed = result => result?.success === true && result.meta?.changes === 1; + +async function revisionedSave(db, body) { + await requireSnapshotSchema(db); + const statements = [initializeRevision(db), + db.prepare(`UPDATE config_snapshot_revisions SET revision=revision+1 WHERE domain=? + AND revision < ?`).bind(DOMAIN, Number.MAX_SAFE_INTEGER), + db.prepare(`INSERT INTO auto_routes (route_id, candidates, updated_at) + SELECT ?, ?, ? WHERE changes()=1 + ON CONFLICT(route_id) DO UPDATE SET candidates=excluded.candidates, updated_at=excluded.updated_at`) + .bind(body.route_id, JSON.stringify(body.candidates), body.updated_at)]; + const results = await db.batch(statements); + if (!changed(results[1]) || !changed(results[2])) throw new Error("Revisioned save was not confirmed"); +} + +async function snapshotOperation(db, body) { + const operation = body.operation; + if (operation === "snapshot_create") { + if (!fields(body, ["version", "operation", "domain", "base_revision", "configuration", "actor"]) + || body.domain !== DOMAIN || !validRevision(body.base_revision) || typeof body.actor !== "string" + || !ACTOR.test(body.actor) || !validConfiguration(body.configuration)) return reply({ error: "invalid_request" }, 400); + const size = new TextEncoder().encode(JSON.stringify(body.configuration)).byteLength; + if (size > SNAPSHOT_BYTES) return reply({ error: "snapshot_too_large" }, 413); + await requireSnapshotSchema(db); + return await createSnapshot(db, body, size); + } + if (operation === "snapshot_list" && fields(body, ["version", "operation"])) { + await requireSnapshotSchema(db); + const results = await db.batch([ + db.prepare("SELECT COALESCE((SELECT revision FROM config_snapshot_revisions WHERE domain=?), 0) AS revision").bind(DOMAIN), + db.prepare(`SELECT ${metadataColumns} FROM config_snapshots WHERE domain=? ORDER BY created_at DESC, id DESC LIMIT ?`).bind(DOMAIN, SNAPSHOT_COUNT), + db.prepare(`SELECT id, snapshot_id, base_revision, revision, applied_at, applied_by FROM config_snapshot_applications + WHERE domain=? ORDER BY revision DESC LIMIT ?`).bind(DOMAIN, SNAPSHOT_COUNT)]); + return reply({ version: 1, domain: DOMAIN, current_revision: results[0].results[0].revision, + snapshots: results[1].results, applications: results[2].results }); + } + if (operation === "snapshot_diff" && (fields(body, ["version", "operation", "id"]) + || fields(body, ["version", "operation", "id", "offset"])) + && typeof body.id === "string" && SNAPSHOT_ID.test(body.id) + && (body.offset === undefined || Number.isInteger(body.offset) && body.offset >= 0 && body.offset <= MAX_ROUTES)) { + await requireSnapshotSchema(db); + return await diffSnapshot(db, body.id, body.offset || 0); + } + if (operation === "snapshot_apply") { + if (!fields(body, ["version", "operation", "id", "current_revision", "confirm", "actor"]) + || body.confirm !== true || !validRevision(body.current_revision)) return reply({ error: "revision_conflict" }, 409); + if (typeof body.id !== "string" || !SNAPSHOT_ID.test(body.id) + || typeof body.actor !== "string" || !ACTOR.test(body.actor)) return reply({ error: "invalid_request" }, 400); + await requireSnapshotSchema(db); + return await applySnapshot(db, body); + } + return reply({ error: "invalid_request" }, 400); +} + +async function createSnapshot(db, body, size) { + const id = identifier(), now = new Date().toISOString(); + const configuration = JSON.stringify({ routes: [...body.configuration.routes].sort((a, b) => a.route_id.localeCompare(b.route_id)) }); + const baseline = await routeState(db); + const results = await db.batch([initializeRevision(db), + db.prepare(`INSERT INTO config_snapshots (${metadataColumns}, configuration, base_fingerprint) + SELECT ?, ?, ?, ?, ?, ?, ?, ? WHERE (${routeStateSQL})=? AND + (SELECT revision FROM config_snapshot_revisions WHERE domain=?)=? + AND (SELECT COUNT(*) FROM config_snapshots WHERE domain=?) < ?`) + .bind(id, DOMAIN, body.base_revision, now, body.actor, size, configuration, baseline.fingerprint, baseline.state, DOMAIN, body.base_revision, DOMAIN, SNAPSHOT_COUNT)]); + if (!changed(results[1])) return reply({ error: "snapshot_limit_or_revision_conflict" }, 409); + return reply({ version: 1, snapshot: { id, domain: DOMAIN, base_revision: body.base_revision, + created_at: now, created_by: body.actor, size_bytes: size } }); +} + +async function readSnapshot(db, id) { + const row = await db.prepare(`SELECT ${metadataColumns}, configuration, base_fingerprint FROM config_snapshots WHERE id=? AND domain=?`).bind(id, DOMAIN).first(); + if (!row) return null; + const configuration = JSON.parse(row.configuration); + if (!validConfiguration(configuration) || new TextEncoder().encode(row.configuration).byteLength > SNAPSHOT_BYTES) { + throw new Error("Invalid stored snapshot"); + } + return { ...row, configuration }; +} + +async function diffSnapshot(db, id, offset) { + const snapshot = await readSnapshot(db, id); + if (!snapshot) return reply({ error: "not_found" }, 404); + const results = await db.batch([ + db.prepare("SELECT COALESCE((SELECT revision FROM config_snapshot_revisions WHERE domain=?), 0) AS revision").bind(DOMAIN), + db.prepare("SELECT route_id, candidates FROM auto_routes ORDER BY route_id")]); + const stored = new Map(results[1].results.map(row => [row.route_id, JSON.parse(row.candidates)])); + const changes = snapshot.configuration.routes.slice(offset, offset + DIFF_PAGE).map(row => { + const before = stored.get(row.route_id) || []; + if (before.length && (!validCandidates(before) || before.some(model => SECRET_ID.test(model)))) throw new Error("Invalid stored route"); + return { route_id: row.route_id, before, after: row.candidates }; + }); + const next = offset + DIFF_PAGE < snapshot.configuration.routes.length ? offset + DIFF_PAGE : null; + return reply({ version: 1, id, domain: DOMAIN, base_revision: snapshot.base_revision, + current_revision: results[0].results[0].revision, changes, next_offset: next }); +} + +async function applySnapshot(db, body) { + const snapshot = await readSnapshot(db, body.id); + if (!snapshot) return reply({ error: "not_found" }, 404); + if (snapshot.base_revision !== body.current_revision) return reply({ error: "revision_conflict" }, 409); + const baseline = await routeState(db); + if (snapshot.base_fingerprint !== baseline.fingerprint) return reply({ error: "revision_conflict" }, 409); + const action = identifier(), now = new Date().toISOString(); + const revision = body.current_revision + 1; + const statements = [initializeRevision(db), + db.prepare(`UPDATE config_snapshot_revisions SET revision=revision+1 WHERE domain=? AND revision=? + AND (${routeStateSQL})=? + AND (SELECT COUNT(*) FROM auto_routes) + (SELECT COUNT(*) FROM json_each(?) AS incoming + WHERE NOT EXISTS (SELECT 1 FROM auto_routes WHERE route_id=json_extract(incoming.value, '$.route_id'))) <= ?`) + .bind(DOMAIN, body.current_revision, baseline.state, JSON.stringify(snapshot.configuration.routes), MAX_ROUTES), + db.prepare(`INSERT INTO config_snapshot_applications + (id, domain, snapshot_id, base_revision, revision, applied_at, applied_by) + SELECT ?, ?, ?, ?, ?, ?, ? WHERE changes()=1`) + .bind(action, DOMAIN, body.id, body.current_revision, revision, now, body.actor)]; + for (const row of snapshot.configuration.routes) { + statements.push(db.prepare(`INSERT INTO auto_routes (route_id, candidates, updated_at) + SELECT ?, ?, ? WHERE EXISTS (SELECT 1 FROM config_snapshot_applications WHERE id=?) + ON CONFLICT(route_id) DO UPDATE SET candidates=excluded.candidates, updated_at=excluded.updated_at`) + .bind(row.route_id, JSON.stringify(row.candidates), now, action)); + } + // D1 batch is transactional: route failures roll back the revision and audit together. + const results = await db.batch(statements); + if (!changed(results[1])) return reply({ error: "revision_conflict" }, 409); + if (!changed(results[2]) || results.slice(3).some(result => !changed(result))) throw new Error("Apply was not confirmed"); + return reply({ version: 1, applied: true, application_id: action, revision }); +} diff --git a/worker/container-env.mjs b/worker/container-env.mjs index cbef46d7..6d267547 100644 --- a/worker/container-env.mjs +++ b/worker/container-env.mjs @@ -53,6 +53,7 @@ const DIRECT_ENV_KEYS = [ "PROMPT_TEMPLATES_ENABLED", "AUTO_ROUTE_ORDERING", "AUTO_ROUTE_ORDERING_OVERRIDES", + "CONFIG_SNAPSHOTS_ENABLED", "AUTO_ROUTE_HEALTH_HALF_LIFE_SECONDS", "AUTO_ROUTE_LATENCY_WEIGHT", "AUTO_ROUTE_COST_WEIGHT", diff --git a/worker/d1-schema.mjs b/worker/d1-schema.mjs index 06058f36..6d50ae6e 100644 --- a/worker/d1-schema.mjs +++ b/worker/d1-schema.mjs @@ -9,7 +9,8 @@ export const REQUIRED_D1_TABLES = Object.freeze(["intelligence_policy", "intelli "route_health", "route_health_snapshot", "usage_events", "usage_daily", "usage_batches", "media_jobs", "media_job_items", "shadow_eval_samples", "shadow_eval_results", - "shadow_eval_config", "shadow_eval_policy_backups", "prompt_templates"]); + "shadow_eval_config", "shadow_eval_policy_backups", "prompt_templates", + "config_snapshot_revisions", "config_snapshots", "config_snapshot_applications"]); /** * Readiness for the D1 schema. A deploy that skipped `wrangler d1 migrations apply` reports