diff --git a/docs/gateway-alerts.md b/docs/gateway-alerts.md new file mode 100644 index 00000000..029e7441 --- /dev/null +++ b/docs/gateway-alerts.md @@ -0,0 +1,96 @@ +# Spend, pool and health alerts + +Alerts report gateway spend estimates, unknown-price coverage, open provider circuits, +exhausted credential pools and recent provider health failures. They never start a +generation or a health probe. Paid health probes require their own explicit action. + +Alerts are disabled by default. Empty environment values use the defaults: + +```text +GATEWAY_ALERTS_ENABLED=false +GATEWAY_ALERT_WEBHOOK_ALLOWLIST=[] +``` + +Enable the feature and allow an exact HTTPS origin before configuring delivery: + +```text +GATEWAY_ALERTS_ENABLED=true +GATEWAY_ALERT_WEBHOOK_ALLOWLIST=["https://hooks.example"] +``` + +The allowlist accepts at most 20 public DNS origins on port 443. IP literals, local +hostnames, credentials, query strings, fragments and redirects are rejected. A +destination can have a path, but it must belong to an allowlisted origin. Addresses +are stored as private configuration and are omitted from status and alert payloads. +Use an operator-controlled origin with public DNS; this origin policy does not pin DNS +answers or certify the destination's downstream behavior. Remove the origin or clear +the allowlist to stop delivery without changing generation traffic. + +`GET /admin/alerts` requires an administrator's existing dashboard session. It returns +the current revision, normalized rules, whether a webhook is configured, and up to +100 recent event states. All responses have `Cache-Control: no-store`. POST retains +the dashboard's CSRF protection; supply the existing CSRF token with the request. +Use the revision from GET to replace the configuration: + +```json +{ + "revision": 0, + "destination": "https://hooks.example/operator-hook", + "rules": [ + {"kind": "spend", "period": "day", "budget_usd": 25}, + {"kind": "unknown_price", "period": "day"}, + {"kind": "provider_circuit", "provider": "openai"}, + {"kind": "pool_exhaustion", "provider": "nanogpt"}, + {"kind": "provider_health", "provider": "openai", "failures": 3} + ] +} +``` + +Spend rules use an operator-supplied global gateway budget in USD, with default +thresholds of 85% and 100%. `day` and `month` use UTC ledger periods. At most 20 rules +and four distinct spend thresholds per rule are accepted. Set `thresholds` explicitly +to change the spend percentages. An unknown-price rule defaults to alerting whenever +any recorded request lacks a price; `threshold_percent` can raise that minimum. +Provider names must belong to the reviewed provider catalog. An empty rule list stops +new observations. Replacing configuration fails waiting events from the old revision. + +Spend payloads label their basis as `gateway_cost_estimate`; they are estimates from +recorded usage and configured model prices, never actual provider invoices. Coverage +payloads label `unknown_price_coverage`; an unpriced request is unknown, not free. +Provider alerts label `observed_health`. Health failure counts expire after one hour. +No prompt, response, tool content, raw credential, username or recipient appears in +an alert payload, status response or delivery diagnostic. Numeric observations can +still reveal aggregate traffic and spend; protect the webhook's access and retention. + +The durable store deduplicates each event type, rule, threshold, period and revision +for 15 minutes. A sustained condition can create a new event after that interval. +Delivery is limited to ten events per scheduled run and three attempts per event, +with a hard three-second timeout per attempt. Retries are due after 60 seconds and +only run on the next scheduled invocation. Delivery claims prevent concurrent +runners from sending the same event simultaneously. Timeouts and process termination +can leave ambiguous delivery: consumers should deduplicate the stable event identity +and rule/revision/occurrence fields. A successful delivery means the destination +returned HTTP 2xx; it does not prove an operator read the notification. + +States distinguish waiting, delivered and failed events; `attempts`, +`last_attempt_at` and a fixed `error_code` expose failures without response bodies. +Each event stores at most three attempt timestamps. History is logically retained +for seven days, physically pruned in batches of 100 during scheduled runs, and capped +at 2,000 dedupe records. A full queue cannot accept a new distinct event until pruning +makes room. Delivery JSON and configuration requests are bounded to 8 KiB; status +responses are bounded to 64 KiB. + +Disabled or malformed settings return HTTP 404. Invalid rules and destinations return +400, revision conflicts 409, oversized configuration 413, and unavailable durable +state 503 with `gateway_alert_storage_unavailable`. The feature requires the D1 +migration and private state dispatcher. An enabled flag with absent tables fails +closed. Configuration and observation do not deliver webhooks synchronously. + +The Worker scheduler calls `runAlertDelivery` with a fixed aggregate collector and +redirect-rejecting transport. The collector receives normalized rules and returns +only `{rule_id, value, basis, window}` observations; a provider window is `current`. +The private dispatcher calls `handleAlertState` only after internal authentication. +The Flask registrar mounts the admin route and exposes `gateway_alert_observer`; +free checks observe existing results through the same service. Generation requests +are never replayed or delayed to send alerts. No live webhook or paid metering check +is needed to run the deterministic tests. diff --git a/intelligence-migrations/0022_gateway_alerts.sql b/intelligence-migrations/0022_gateway_alerts.sql new file mode 100644 index 00000000..ec3fc76c --- /dev/null +++ b/intelligence-migrations/0022_gateway_alerts.sql @@ -0,0 +1,25 @@ +-- Numeric alert evidence; webhook addresses are private configuration only. +CREATE TABLE IF NOT EXISTS gateway_alert_rules ( + id INTEGER PRIMARY KEY CHECK(id = 1), + revision INTEGER NOT NULL CHECK(revision > 0 AND revision < 9007199254740991), + configuration TEXT NOT NULL CHECK(length(configuration) <= 8192), + updated_at REAL NOT NULL +); +CREATE TABLE IF NOT EXISTS gateway_alert_events ( + dedupe_key TEXT PRIMARY KEY, + event_id TEXT NOT NULL, + rule_id TEXT NOT NULL, + revision INTEGER NOT NULL, + payload TEXT NOT NULL CHECK(length(payload) <= 8192), + created_at REAL NOT NULL, + state TEXT NOT NULL CHECK(state IN ('pending', 'delivered', 'failed')), + attempts INTEGER NOT NULL DEFAULT 0 CHECK(attempts BETWEEN 0 AND 3), + attempt_times TEXT NOT NULL DEFAULT '[]' CHECK(length(attempt_times) <= 128), + last_attempt_at REAL, + next_attempt_at REAL NOT NULL, + claim_token TEXT, + lease_until REAL NOT NULL DEFAULT 0, + error_code TEXT +); +CREATE INDEX IF NOT EXISTS gateway_alert_events_due ON gateway_alert_events(state, next_attempt_at, lease_until); +CREATE INDEX IF NOT EXISTS gateway_alert_events_retention ON gateway_alert_events(created_at); diff --git a/package.json b/package.json index 56f892f0..4a24cbcf 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "lint": "ruff check . --select E9,F63,F7,F82", "typecheck": "mypy --ignore-missing-imports --follow-imports=skip --disable-error-code import-untyped --disable-error-code var-annotated --disable-error-code arg-type --disable-error-code assignment --disable-error-code operator app.py routes services providers streaming", "test": "python -m pytest -q", - "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_session_tiers_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_usage_reservations_worker.mjs tests/test_shared_generation_cache_worker.mjs tests/test_deferred_tools_worker.mjs", + "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_gateway_alerts_worker.mjs tests/test_session_tiers_worker.mjs tests/test_usage_reservations_worker.mjs tests/test_shared_generation_cache_worker.mjs tests/test_deferred_tools_worker.mjs", "db:migrate": "python scripts/validate_sqlite_schema.py --apply", "db:validate": "python scripts/validate_sqlite_schema.py", "security:secrets": "python scripts/check_static_secrets.py", diff --git a/routes/alerts.py b/routes/alerts.py new file mode 100644 index 00000000..6706481d --- /dev/null +++ b/routes/alerts.py @@ -0,0 +1,39 @@ +"""Administrator alert controls retain existing session and CSRF protection.""" +from flask import jsonify, request + +from services.auth_service import AuthService +from services import gateway_alerts + + +def register_alert_routes(app): + def disabled_alerts(): + if request.path == "/admin/alerts" and not gateway_alerts.settings()[0]: + return jsonify({"error": {"code": "not_found"}}), 404 + if request.path == "/admin/alerts" and not AuthService.is_authenticated(): + return jsonify({"error": {"code": "authentication_required"}}), 401 + return None + + app.before_request_funcs.setdefault(None, []).insert(0, disabled_alerts) + + @app.after_request + def private_alerts(response): + if request.path == "/admin/alerts": + response.headers["Cache-Control"] = "no-store" + return response + + @app.route("/admin/alerts", methods=["GET", "POST"]) + def admin_alerts(): + if not AuthService.is_authenticated(): + return jsonify({"error": {"code": "authentication_required"}}), 401 + if not (AuthService.get_current_user() or {}).get("is_admin"): + return jsonify({"error": {"code": "admin_required"}}), 403 + try: + if request.method == "GET": + result = gateway_alerts.call("get") + else: + if request.content_length is not None and request.content_length > gateway_alerts.MAX_BYTES: + raise gateway_alerts.AlertError("gateway_alert_too_large", 413) + result = gateway_alerts.configure(request.get_json(silent=True), set(app.config["API_BASE_URLS"])) + return jsonify(result) + except gateway_alerts.AlertError as error: + return jsonify({"error": {"code": error.code}}), error.status diff --git a/services/gateway_alerts.py b/services/gateway_alerts.py new file mode 100644 index 00000000..f87ad82c --- /dev/null +++ b/services/gateway_alerts.py @@ -0,0 +1,269 @@ +"""Content-free alert configuration and aggregate observation through private state.""" +from __future__ import annotations + +import hashlib +import json +import logging +import math +import os +import re +import struct +from datetime import datetime, timezone +from urllib.parse import urlsplit + +from services import control_state_d1 + +logger = logging.getLogger(__name__) +MAX_BYTES = 8192 +MAX_RULES = 20 +MAX_REVISION = 2**53 - 1 +ID = re.compile(r"[0-9a-f]{64}\Z") +WINDOW = re.compile(r"(?:current|[0-9]{4}-[0-9]{2}(?:-[0-9]{2})?)\Z") +KINDS = {"spend", "unknown_price", "provider_circuit", "pool_exhaustion", "provider_health"} +BASES = {"spend": "gateway_cost_estimate", "unknown_price": "unknown_price_coverage"} +_warned: set[str] = set() + + +class AlertError(Exception): + def __init__(self, code="invalid_gateway_alert", status=400): + self.code, self.status = code, status + super().__init__(code) + + +def warn_once(name): + if name not in _warned: + _warned.add(name) + logger.warning("Invalid gateway alert setting (%s)", name) + + +def number(value): + return type(value) in (int, float) and 0 <= value <= 1e12 and math.isfinite(value) + + +def revision(value): + return type(value) is int and 0 <= value < MAX_REVISION + + +def encoded(value, limit=MAX_BYTES): + try: + text = json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False) + if len(text.encode()) > limit: + raise AlertError("gateway_alert_too_large", 413) + return text + except (TypeError, ValueError, RecursionError, UnicodeError): + raise AlertError() from None + + +def origin(value): + if not isinstance(value, str) or len(value) > 2048 or any(ord(c) <= 32 or ord(c) >= 127 for c in value) or "\\" in value: + raise AlertError() + try: + parsed = urlsplit(value) + host = parsed.hostname or "" + if (parsed.scheme != "https" or parsed.username is not None or parsed.password is not None + or parsed.query or parsed.fragment or "?" in value or "#" in value + or parsed.port not in (None, 443) or not re.fullmatch(r"[a-z0-9]+(?:[a-z0-9-]*[a-z0-9])?(?:\.[a-z0-9]+(?:[a-z0-9-]*[a-z0-9])?)+", host) + or host.rsplit(".", 1)[-1] in {"localhost", "local", "internal", "invalid", "test", "home", "lan"} + or host.rsplit(".", 1)[-1].isdigit() or host.startswith("metadata.")): + raise AlertError() + return "https://" + host + except ValueError: + raise AlertError() from None + + +def settings(): + raw = os.environ.get("GATEWAY_ALERTS_ENABLED", "").strip().lower() + enabled = raw in {"1", "true", "yes", "on"} + if raw not in {"", "0", "1", "false", "true", "no", "yes", "off", "on"}: + warn_once("GATEWAY_ALERTS_ENABLED") + raw_list = os.environ.get("GATEWAY_ALERT_WEBHOOK_ALLOWLIST", "").strip() or "[]" + try: + values = json.loads(raw_list) + if not isinstance(values, list) or len(values) > 20: + raise AlertError() + allowlist = {origin(value) for value in values} + if any(value not in {origin(value), origin(value) + "/", origin(value) + ":443", origin(value) + ":443/"} for value in values): + raise AlertError() + except (ValueError, TypeError, RecursionError, AlertError): + warn_once("GATEWAY_ALERT_WEBHOOK_ALLOWLIST") + return False, set() + return enabled, allowlist + + +def validate_destination(value, allowlist): + if origin(value) not in allowlist: + raise AlertError("gateway_alert_destination_not_allowed") + return value + + +def numeric_identity(value): + # Binary64 identity keeps integer/decimal rule hashes identical across both runtimes. + return "n:" + struct.pack("!d", value).hex() if type(value) in (int, float) else value + + +def validate_rules(values, providers): + if not isinstance(values, list) or len(values) > MAX_RULES: + raise AlertError() + result, seen = [], set() + for value in values: + if not isinstance(value, dict) or not isinstance(value.get("kind"), str) or value["kind"] not in KINDS: + raise AlertError() + kind = value["kind"] + allowed = {"kind", "period", "budget_usd", "thresholds"} if kind == "spend" else ( + {"kind", "period", "threshold_percent"} if kind == "unknown_price" else {"kind", "provider", "failures"} if kind == "provider_health" else {"kind", "provider"}) + if set(value) - allowed: + raise AlertError() + rule = {"kind": kind} + if kind in {"spend", "unknown_price"}: + if not isinstance(value.get("period"), str) or value["period"] not in {"day", "month"}: + raise AlertError() + rule["period"] = value["period"] + else: + if not isinstance(value.get("provider"), str) or value["provider"] not in providers: + raise AlertError() + rule["provider"] = value["provider"] + if kind == "spend": + budget = value.get("budget_usd") + thresholds = value.get("thresholds", [85, 100]) + if (not number(budget) or budget <= 0 or not isinstance(thresholds, list) or not 1 <= len(thresholds) <= 4 + or any(not number(t) or not 0 < t <= 100 for t in thresholds) or len(set(thresholds)) != len(thresholds)): + raise AlertError() + rule.update(budget_usd=budget, thresholds=sorted(thresholds)) + elif kind == "unknown_price": + threshold = value.get("threshold_percent", 0) + if not number(threshold) or threshold > 100: + raise AlertError() + rule["threshold_percent"] = threshold + elif kind == "provider_health": + failures = value.get("failures", 3) + if type(failures) is not int or not 1 <= failures <= 100: + raise AlertError() + rule["failures"] = failures + canonical = {key: [numeric_identity(v) for v in value] if isinstance(value, list) else numeric_identity(value) + for key, value in rule.items()} + identifier = hashlib.sha256(encoded(canonical).encode()).hexdigest() + if identifier in seen: + raise AlertError() + seen.add(identifier) + result.append({**rule, "id": identifier}) + encoded(result) + return result + + +def valid_payload(value): + if not isinstance(value, dict) or set(value) != {"event", "rule_id", "revision", "occurred_at", "basis", "value", "threshold", "window", "provider"}: + return False + return (value["event"] in KINDS and isinstance(value["rule_id"], str) and bool(ID.fullmatch(value["rule_id"])) + and revision(value["revision"]) and number(value["occurred_at"]) and number(value["value"]) + and number(value["threshold"]) and value["basis"] == BASES.get(value["event"], "observed_health") + and isinstance(value["window"], str) and bool(WINDOW.fullmatch(value["window"])) + and (value["provider"] is None or value["provider"] in known_providers())) + + +def known_providers(): + from config import Config + return set(Config.API_BASE_URLS) + + +def valid_status(value): + if (not isinstance(value, dict) or set(value) != {"version", "revision", "rules", "webhook_configured", "events"} + or type(value["version"]) is not int or value["version"] != 1 or not revision(value["revision"]) + or type(value["webhook_configured"]) is not bool or not isinstance(value["rules"], list)): + return False + try: + rules = validate_rules([{k: v for k, v in row.items() if k != "id"} for row in value["rules"]], known_providers()) + if rules != value["rules"] or not isinstance(value["events"], list) or len(value["events"]) > 100: + return False + for row in value["events"]: + if (not isinstance(row, dict) or set(row) != {"id", "state", "attempts", "last_attempt_at", "error_code", "payload"} + or not isinstance(row["id"], str) or not ID.fullmatch(row["id"]) + or row["state"] not in {"pending", "delivered", "failed"} or type(row["attempts"]) is not int + or not 0 <= row["attempts"] <= 3 or row["error_code"] not in {None, "delivery_failed", "rule_replaced", "invalid_payload", "attempts_exhausted"} + or row["last_attempt_at"] is not None and not number(row["last_attempt_at"]) or not valid_payload(row["payload"])): + return False + encoded(value, limit=65536) + except (AlertError, TypeError, AttributeError): + return False + return True + + +def call(operation, **values): + if not settings()[0]: + raise AlertError("not_found", 404) + if not control_state_d1.using_d1(): + raise AlertError("gateway_alert_storage_unavailable", 503) + try: + result = control_state_d1.call("alerts", operation, **values) + except Exception as error: + if getattr(error, "status", None) == 409: + raise AlertError("gateway_alert_revision_conflict", 409) from None + raise AlertError("gateway_alert_storage_unavailable", 503) from None + if operation in {"get", "configure"}: + valid = valid_status(result) + else: + valid = (isinstance(result, dict) and set(result) == {"version", "queued"} and result["version"] == 1 + and type(result["queued"]) is int and 0 <= result["queued"] <= 80) + if not valid: + raise AlertError("gateway_alert_storage_unavailable", 503) + return result + + +def configure(body, providers): + if not isinstance(body, dict) or set(body) != {"revision", "destination", "rules"} or not revision(body["revision"]): + raise AlertError() + encoded(body) + destination = validate_destination(body["destination"], settings()[1]) + rules = validate_rules(body["rules"], providers) + result = call("configure", revision=body["revision"], configuration={"destination": destination, "rules": rules}) + if result["revision"] != body["revision"] + 1 or result["rules"] != rules: + raise AlertError("gateway_alert_storage_unavailable", 503) + return result + + +def collect_observations(rules, *, usage=None, health=None, now=None): + from services.usage_ledger import LEDGER + from services.health_checks import passive_alert_health + current = datetime.fromtimestamp(datetime.now(timezone.utc).timestamp() if now is None else now, timezone.utc) + store, health_reader = usage if usage is not None else LEDGER.store(), health or passive_alert_health + states = health_reader({r["provider"] for r in rules if "provider" in r}) + summaries, result = {}, [] + for rule in rules: + kind, period = rule["kind"], rule.get("period") + if period and period not in summaries: + since = current.strftime("%Y-%m-%d" if period == "day" else "%Y-%m-01") + rows = store.summary("day", since, current.strftime("%Y-%m-%d"), principal=None, limit=31) + totals = {name: sum(row.get(name, 0) or 0 for row in rows) for name in ("cost_usd", "requests", "priced_requests")} + if any(not number(v) for v in totals.values()) or totals["priced_requests"] > totals["requests"]: + raise AlertError("gateway_alert_usage_unavailable", 503) + summaries[period] = totals + if kind == "spend": + value = summaries[period]["cost_usd"] + elif kind == "unknown_price": + totals = summaries[period] + value = 100 * (totals["requests"] - totals["priced_requests"]) / totals["requests"] if totals["requests"] else 0 + else: + state = states.get(rule["provider"], {}) + field = {"provider_circuit": "circuit_open", "pool_exhaustion": "pool_exhausted", "provider_health": "consecutive_failures"}[kind] + value = int(state.get(field, 0)) + result.append({"rule_id": rule["id"], "value": value, "basis": BASES.get(kind, "observed_health"), + "window": current.strftime("%Y-%m-%d" if period == "day" else "%Y-%m") if period else "current"}) + return result + + +def observe(*, collect=None): + if not settings()[0]: + return {"queued": 0} + status = call("get") + if not status["rules"]: + return {"version": 1, "queued": 0} + observations = (collect or collect_observations)(status["rules"]) + encoded(observations) + return call("observe", revision=status["revision"], observations=observations) + + +def observe_safely(): + if settings()[0]: + try: + observe() + except Exception: + logger.warning("Gateway alert observation unavailable") diff --git a/services/gateway_extensions.py b/services/gateway_extensions.py index 8114fd91..77461053 100644 --- a/services/gateway_extensions.py +++ b/services/gateway_extensions.py @@ -80,3 +80,8 @@ def config_revision_status(): for callback in callbacks: callback(app) + + from routes.alerts import register_alert_routes + from services.gateway_alerts import observe + register_alert_routes(app) + app.extensions["gateway_alert_observer"] = observe diff --git a/services/health_checks.py b/services/health_checks.py index d3a8e462..999f6d74 100644 --- a/services/health_checks.py +++ b/services/health_checks.py @@ -104,4 +104,35 @@ def run_free_checks(base_urls: Mapping[str, str], auth_service_cls, proxy_servic results.extend(checked) else: checked_at = time.time() if now is None else now + from services.gateway_alerts import observe_safely + observe_safely() return {"checked_at": iso_time(checked_at), "results": sorted(results, key=lambda item: item["provider"])} + + +def passive_alert_health(providers, *, pools=None, now=None): + """Read existing circuits and pool eligibility without acquiring a key or probing.""" + from services.credential_pool import CredentialPool + from services.resilience_service import ResilienceService + current = time.time() if now is None else now + result = {} + for provider in sorted(providers): + circuit = ResilienceService.snapshot(provider, now=current) + entry = RouteHealth.snapshot("provider:" + provider) or {} + failures = entry.get("consecutive_failures", 0) + if current - (entry.get("last_failure_at") or 0) > 3600: + failures = 0 + counts = pools(provider) if pools is not None else None + if counts is None and CredentialPool.pooled(provider): + keys = CredentialPool.keys(provider) + counts = (len(keys), len(CredentialPool.available(provider, keys))) + if counts is None and provider == "nanogpt": + from services.auth_service import AuthService + from services.nanogpt_key_pool import NanoGPTKeyPool + keys = list(dict.fromkeys(AuthService.get_api_keys(provider))) + with NanoGPTKeyPool._lock: + eligible = sum(NanoGPTKeyPool._rejected_until.get(key, 0) <= time.monotonic() for key in keys) + counts = (len(keys), eligible) + result[provider] = {"circuit_open": circuit["state"] == "open", + "pool_exhausted": counts is not None and counts[0] > 0 and counts[1] == 0, + "consecutive_failures": failures} + return result diff --git a/tests/fixtures/secret_firewall_coverage.json b/tests/fixtures/secret_firewall_coverage.json index 6610c1a9..40f7b523 100644 --- a/tests/fixtures/secret_firewall_coverage.json +++ b/tests/fixtures/secret_firewall_coverage.json @@ -1,5 +1,8 @@ { "flask_routes": { + "routes/alerts.py": [ + "app.route('/admin/alerts', methods=['GET', 'POST'])" + ], "routes/auto_routes.py": [ "app.route('/admin/auto-routes', methods=['GET', 'PUT'])", "app.route('/admin/auto-routes/catalog', methods=['POST'])" @@ -218,6 +221,7 @@ "worker/access-sso.mjs", "worker/admission-do.mjs", "worker/ai-outbound.mjs", + "worker/alert-delivery.mjs", "worker/api-paths.mjs", "worker/auto-routes-d1.mjs", "worker/build-id.mjs", diff --git a/tests/test_gateway_alerts.py b/tests/test_gateway_alerts.py new file mode 100644 index 00000000..7cd6fa9c --- /dev/null +++ b/tests/test_gateway_alerts.py @@ -0,0 +1,284 @@ +"""Content-free alerts use registered admin paths and private, injected state.""" +import importlib +import json +import sqlite3 +from pathlib import Path +from unittest.mock import Mock, patch + +import pytest + +from services import gateway_alerts as alerts + +MIGRATION = "0022_gateway_alerts.sql" +DESTINATION = "https://hooks.example/recipient-private" +BODY = {"revision": 0, "destination": DESTINATION, "rules": [ + {"kind": "spend", "period": "day", "budget_usd": 10}, + {"kind": "unknown_price", "period": "day"}, + {"kind": "provider_circuit", "provider": "openai"}, + {"kind": "pool_exhaustion", "provider": "openai"}, +]} + + +@pytest.fixture(autouse=True) +def isolated(monkeypatch): + monkeypatch.setenv("GATEWAY_ALERTS_ENABLED", "false") + monkeypatch.setenv("GATEWAY_ALERT_WEBHOOK_ALLOWLIST", '["https://hooks.example"]') + monkeypatch.setenv("CONFIG_REVISION_SYNC_ENABLED", "false") + monkeypatch.setenv("INTELLIGENCE_STORAGE_BACKEND", "") + monkeypatch.setattr(alerts, "_warned", set()) + monkeypatch.setattr("requests.Session.send", Mock(side_effect=AssertionError("No network"))) + + +@pytest.fixture +def client(monkeypatch, tmp_path): + for name, value in {"FLASK_SECRET_KEY": "synthetic-alert-session", "JWT_SECRET": "synthetic-alert-jwt", + "ADMIN_API_KEY": "synthetic-alert-admin", "CONTROL_PLANE_DATABASE_URL": "", + "AUTH_STORAGE_BACKEND": "sql", "AUTH_DB_PATH": str(tmp_path / "auth.sqlite3"), + "MODEL_REGISTRY_DB_PATH": str(tmp_path / "models.sqlite3"), + "RATE_LIMIT_DB_PATH": str(tmp_path / "limits.sqlite3")}.items(): + monkeypatch.setenv(name, value) + with patch("config.load_runtime_env"), patch("env_loader.load_runtime_env"), patch("services.usage_ledger.start"): + module = importlib.import_module("app") + with patch.object(module, "load_runtime_env"): + app = module.create_app() + app.config.update(TESTING=True, WTF_CSRF_ENABLED=False) + core = importlib.import_module("routes.core") + route = importlib.import_module("routes.alerts") + monkeypatch.setattr(core.AuthService, "is_authenticated", lambda: True) + monkeypatch.setattr(route.AuthService, "is_authenticated", lambda: True) + monkeypatch.setattr(route.AuthService, "get_current_user", lambda: {"is_admin": True}) + return app.test_client() + + +def status(revision=0, rules=None): + return {"version": 1, "revision": revision, "rules": rules or [], "webhook_configured": bool(revision), + "events": []} + + +@pytest.mark.parametrize("value", ["", "false", "invalid", "no", "0"]) +def test_disabled_registered_routes_do_not_touch_storage(client, monkeypatch, value): + monkeypatch.setenv("GATEWAY_ALERTS_ENABLED", value) + private = Mock(side_effect=AssertionError("Storage touched")) + monkeypatch.setattr(alerts.control_state_d1, "call", private) + for method in ("GET", "POST"): + response = client.open("/admin/alerts", method=method, json={}) + assert response.status_code == 404 + assert response.headers["Cache-Control"] == "no-store" + private.assert_not_called() + assert client.get("/healthz").json["status"] == "healthy" + + +def test_enabled_registered_config_defaults_and_private_status(client, monkeypatch): + monkeypatch.setenv("GATEWAY_ALERTS_ENABLED", "true") + monkeypatch.setenv("INTELLIGENCE_STORAGE_BACKEND", "d1") + normalized = alerts.validate_rules(BODY["rules"], {"openai"}) + assert normalized[0]["thresholds"] == [85, 100] + private = Mock(side_effect=[status(), status(1, normalized)]) + monkeypatch.setattr(alerts.control_state_d1, "call", private) + assert client.get("/admin/alerts").json == status() + saved = client.post("/admin/alerts", json=BODY) + assert saved.status_code == 200 + assert saved.json == status(1, normalized) + assert private.call_args.args == ("alerts", "configure") + assert private.call_args.kwargs["configuration"]["destination"] == DESTINATION + assert "recipient-private" not in saved.get_data(as_text=True) + + +def test_admin_authorization_and_csrf_are_preserved(client, monkeypatch): + monkeypatch.setenv("GATEWAY_ALERTS_ENABLED", "true") + route = importlib.import_module("routes.alerts") + monkeypatch.setattr(route.AuthService, "get_current_user", lambda: {"is_admin": False}) + private = Mock() + monkeypatch.setattr(alerts.control_state_d1, "call", private) + assert client.get("/admin/alerts").status_code == 403 + assert client.post("/admin/alerts", json=BODY).status_code == 403 + private.assert_not_called() + client.application.config["WTF_CSRF_ENABLED"] = True + assert client.post("/admin/alerts", json=BODY).status_code == 400 + + +@pytest.mark.parametrize("backend", ["", "sql", "d1"]) +def test_enabled_missing_state_fails_closed(client, monkeypatch, backend): + monkeypatch.setenv("GATEWAY_ALERTS_ENABLED", "true") + monkeypatch.setenv("INTELLIGENCE_STORAGE_BACKEND", backend) + monkeypatch.setattr(alerts.control_state_d1, "call", Mock(side_effect=RuntimeError("private-secret"))) + response = client.get("/admin/alerts") + assert response.status_code == 503 + assert response.json["error"]["code"] == "gateway_alert_storage_unavailable" + assert "private-secret" not in response.get_data(as_text=True) + + +@pytest.mark.parametrize("destination", ["http://hooks.example/a", "https://evil.example/a", + "https://hooks.example@localhost/a", "https://127.0.0.1/a", "https://169.254.169.254/a", + "https://[::1]/a", "https://hooks.example/a?secret=x", "https://hooks.example/a#fragment", + "https://hooks.example\\@evil.example/a", "https://hooks.example/\nsecret"]) +def test_ssrf_rejects_before_private_storage(client, monkeypatch, destination): + monkeypatch.setenv("GATEWAY_ALERTS_ENABLED", "true") + monkeypatch.setenv("INTELLIGENCE_STORAGE_BACKEND", "d1") + private = Mock() + monkeypatch.setattr(alerts.control_state_d1, "call", private) + response = client.post("/admin/alerts", json={**BODY, "destination": destination}) + assert response.status_code == 400 + private.assert_not_called() + + +@pytest.mark.parametrize("value", ["", "[]", "invalid", '["https://localhost"]', '["https://127.0.0.1"]']) +def test_allowlist_empty_or_malformed_prevents_configuration(client, monkeypatch, value, caplog): + monkeypatch.setenv("GATEWAY_ALERTS_ENABLED", "true") + monkeypatch.setenv("GATEWAY_ALERT_WEBHOOK_ALLOWLIST", value) + for _ in range(2): + assert client.post("/admin/alerts", json=BODY).status_code == (400 if value in {"", "[]"} else 404) + assert "localhost" not in caplog.text + assert sum("Invalid gateway alert setting" in r.message for r in caplog.records) <= 1 + + +@pytest.mark.parametrize("rules", [[{"kind": "spend", "budget_usd": 0}], + [{"kind": "spend", "period": "day", "budget_usd": float("nan")}], + [{"kind": "spend", "period": "day", "budget_usd": 1, "thresholds": [True]}], + [{"kind": "provider_circuit", "provider": "private-key"}], + [{"kind": "unknown_price", "period": "day", "prompt": "private-content"}], BODY["rules"] * 6]) +def test_invalid_or_content_bearing_rules_are_rejected(rules): + with pytest.raises(alerts.AlertError): + alerts.validate_rules(rules, {"openai"}) + + +def test_registered_response_validation_never_exposes_private_fields(client, monkeypatch): + monkeypatch.setenv("GATEWAY_ALERTS_ENABLED", "true") + monkeypatch.setenv("INTELLIGENCE_STORAGE_BACKEND", "d1") + monkeypatch.setattr(alerts.control_state_d1, "call", Mock(return_value={**status(), "secret": "private-secret"})) + response = client.get("/admin/alerts") + assert response.status_code == 503 + assert "private-secret" not in response.get_data(as_text=True) + + +def test_collector_reads_only_aggregate_usage_and_observed_health(monkeypatch): + rules = alerts.validate_rules(BODY["rules"], {"openai"}) + rows = [{"day": "2026-10-09", "requests": 10, "priced_requests": 6, "cost_usd": 9, + "principal": "private-person", "prompt": "private-content"}] + store = Mock() + store.summary.return_value = rows + states = {"openai": {"circuit_open": True, "pool_exhausted": True, "consecutive_failures": 0}} + observations = alerts.collect_observations(rules, usage=store, health=lambda _: states, + now=1791504000) + assert [row["value"] for row in observations] == [9, 40, 1, 1] + assert observations[0]["basis"] == "gateway_cost_estimate" + assert observations[1]["basis"] == "unknown_price_coverage" + text = json.dumps(observations) + assert "private" not in text + assert all(call.kwargs.get("principal") is None for call in store.summary.call_args_list) + + +def test_observer_is_inert_when_disabled_and_does_not_start_probes(monkeypatch): + private, collect = Mock(), Mock() + monkeypatch.setattr(alerts.control_state_d1, "call", private) + assert alerts.observe(collect=collect) == {"queued": 0} + private.assert_not_called() + collect.assert_not_called() + + +def test_enabled_observer_uses_config_revision_and_injected_collector(monkeypatch): + monkeypatch.setenv("GATEWAY_ALERTS_ENABLED", "true") + monkeypatch.setenv("INTELLIGENCE_STORAGE_BACKEND", "d1") + rules = alerts.validate_rules(BODY["rules"], {"openai"}) + private = Mock(side_effect=[status(1, rules), {"version": 1, "queued": 2}]) + monkeypatch.setattr(alerts.control_state_d1, "call", private) + collect = Mock(return_value=[{"rule_id": rules[0]["id"], "value": 9, + "basis": "gateway_cost_estimate", "window": "2026-10-09"}]) + assert alerts.observe(collect=collect) == {"version": 1, "queued": 2} + assert private.call_args.kwargs["revision"] == 1 + collect.assert_called_once_with(rules) + + +def test_additive_migration_rehearsal_keeps_old_rows(): + with sqlite3.connect(":memory:") as db: + db.executescript("CREATE TABLE old_rows (value TEXT); INSERT INTO old_rows VALUES ('existing');") + sql = (Path(__file__).resolve().parents[1] / "intelligence-migrations" / MIGRATION).read_text() + db.executescript(sql) + db.executescript(sql) + assert db.execute("SELECT value FROM old_rows").fetchone() == ("existing",) + assert {r[0] for r in db.execute("SELECT name FROM sqlite_master WHERE type='table'")} >= { + "gateway_alert_rules", "gateway_alert_events"} + + +def test_fractional_configuration_identity_matches_worker(): + rule = alerts.validate_rules([{"kind": "spend", "period": "month", "budget_usd": 0.000001, + "thresholds": [0.001, 85.5, 100.0]}], {"openai"})[0] + assert rule["id"] == "f699ee8b115fb123a1f3f9504e7004a9123d853c7886c9c767c4066f5e648d61" + + +def test_enabled_unauthenticated_admin_path_is_json_401(client, monkeypatch): + monkeypatch.setenv("GATEWAY_ALERTS_ENABLED", "true") + route = importlib.import_module("routes.alerts") + monkeypatch.setattr(route.AuthService, "is_authenticated", lambda: False) + response = client.get("/admin/alerts") + assert response.status_code == 401 + assert response.json["error"]["code"] == "authentication_required" + + +def test_default_raw_body_headers_and_callback_order_are_unchanged(monkeypatch): + from flask import Flask, Response, request + from services.gateway_extensions import register_gateway_extensions + called = [] + app = Flask(__name__) + register_gateway_extensions(app, callbacks=(lambda _: called.append("first"), lambda _: called.append("second"))) + assert called == ["first", "second"] + app.add_url_rule("/v1/raw", "raw", lambda: Response(request.get_data(), headers={"X-Test": "original"}), methods=["POST"]) + response = app.test_client().post("/v1/raw", data=b'raw\x00content') + assert response.data == b'raw\x00content' + assert response.headers["X-Test"] == "original" + assert "Cache-Control" not in response.headers + + +def test_passive_health_and_pool_counts_never_probe(monkeypatch): + health = importlib.import_module("services.health_checks") + resilience = importlib.import_module("services.resilience_service") + route_health = importlib.import_module("services.route_health") + monkeypatch.setattr(resilience.ResilienceService, "snapshot", lambda *args, **kwargs: {"state": "open"}) + monkeypatch.setattr(route_health.RouteHealth, "snapshot", lambda *args: {"consecutive_failures": 3, "last_failure_at": 100}) + result = health.passive_alert_health({"openai"}, pools=lambda _: (2, 0), now=101) + assert result == {"openai": {"circuit_open": True, "pool_exhausted": True, "consecutive_failures": 3}} + assert health.passive_alert_health({"openai"}, pools=lambda _: (0, 0), now=4000)["openai"] == { + "circuit_open": True, "pool_exhausted": False, "consecutive_failures": 0} + + +def test_observation_failure_does_not_change_free_health_results(monkeypatch): + health = importlib.import_module("services.health_checks") + monkeypatch.setenv("GATEWAY_ALERTS_ENABLED", "true") + monkeypatch.setattr(health.AutoRouteService, "list_routes", lambda: []) + monkeypatch.setattr(alerts, "observe", Mock(side_effect=alerts.AlertError("gateway_alert_storage_unavailable", 503))) + proxy = Mock(side_effect=AssertionError("No provider requests")) + result = health.run_free_checks({}, Mock(), proxy, now=100) + assert result == {"checked_at": health.iso_time(100), "results": []} + proxy.assert_not_called() + + +@pytest.mark.parametrize("period", [[], {}, 1, None]) +def test_malformed_period_is_json_400_before_storage(client, monkeypatch, period): + monkeypatch.setenv("GATEWAY_ALERTS_ENABLED", "true") + private = Mock() + monkeypatch.setattr(alerts.control_state_d1, "call", private) + response = client.post("/admin/alerts", json={**BODY, "rules": [ + {"kind": "spend", "period": period, "budget_usd": 10}]}) + assert response.status_code == 400 + private.assert_not_called() + + +def test_oversized_registered_configuration_is_413_before_storage(client, monkeypatch): + monkeypatch.setenv("GATEWAY_ALERTS_ENABLED", "true") + private = Mock(side_effect=AssertionError("Storage touched")) + monkeypatch.setattr(alerts.control_state_d1, "call", private) + response = client.post("/admin/alerts", json={**BODY, "destination": "https://hooks.example/" + "a" * 8192}) + assert response.status_code == 413 + assert response.json["error"]["code"] == "gateway_alert_too_large" + assert response.headers["Cache-Control"] == "no-store" + private.assert_not_called() + + +def test_registered_revision_conflict_is_json_409(client, monkeypatch): + monkeypatch.setenv("GATEWAY_ALERTS_ENABLED", "true") + monkeypatch.setenv("INTELLIGENCE_STORAGE_BACKEND", "d1") + failure = alerts.control_state_d1.intelligence_d1_store.PrivateIntelligenceError(409, "gateway_alert_revision_conflict") + monkeypatch.setattr(alerts.control_state_d1, "call", Mock(side_effect=failure)) + response = client.post("/admin/alerts", json=BODY) + assert response.status_code == 409 + assert response.json["error"]["code"] == "gateway_alert_revision_conflict" diff --git a/tests/test_gateway_alerts_worker.mjs b/tests/test_gateway_alerts_worker.mjs new file mode 100644 index 00000000..c6b3774b --- /dev/null +++ b/tests/test_gateway_alerts_worker.mjs @@ -0,0 +1,322 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { DatabaseSync } from "node:sqlite"; +import { readFileSync } from "node:fs"; +import { handleAlertState, runAlertDelivery, alertSettings, validateDestination, normalizeAlertRules } from "../worker/alert-delivery.mjs"; + +const MIGRATION = "0022_gateway_alerts.sql"; +const destination = "https://hooks.example/private-recipient"; +const envBase = { GATEWAY_ALERTS_ENABLED: "true", GATEWAY_ALERT_WEBHOOK_ALLOWLIST: '["https://hooks.example"]' }; +const rules = [{ kind: "spend", period: "day", budget_usd: 10 }, + { kind: "unknown_price", period: "day" }, { kind: "provider_circuit", provider: "openai" }, + { kind: "pool_exhaustion", provider: "openai" }, { kind: "provider_health", provider: "openai" }]; +// Exercise the production SQL against SQLite without starting a network-capable runtime. +function database(t, migrate = true) { + const sqlite = new DatabaseSync(":memory:"); + t.after(() => sqlite.close()); + function execute(statements, script = null) { + if (script) sqlite.exec(script); + sqlite.exec("BEGIN"); + try { + const results = statements.map(statement => { + const prepared = sqlite.prepare(statement.sql); + const rows = prepared.columns().length ? prepared.all(...statement.values) : (prepared.run(...statement.values), []); + return { results: rows, meta: { changes: sqlite.prepare("SELECT changes() AS n").get().n } }; + }); + sqlite.exec("COMMIT"); + return results; + } catch (error) { sqlite.exec("ROLLBACK"); throw error; } + } + const db = { prepare(sql) { return { sql, values: [], bind(...values) { this.values = values; return this; }, + async run() { return execute([this])[0]; }, async all() { return execute([this])[0]; }, + async first() { return execute([this])[0].results[0] ?? null; } }; }, + async batch(statements) { return execute(statements); } }; + execute([], "CREATE TABLE old_rows(value TEXT); INSERT INTO old_rows VALUES ('existing');"); + const migration = readFileSync(new URL(`../intelligence-migrations/${MIGRATION}`, import.meta.url), "utf8"); + if (migrate) { execute([], migration); execute([], migration); } + return { db, migrate: () => execute([], migration) }; +} +async function setup(t) { + const { db } = database(t); + const env = { ...envBase, INTELLIGENCE_DB: db }; + const now = [1791504000]; + const call = async body => { + const response = await handleAlertState(env, { version: 1, ...body }, { clock: () => now[0] }); + return { status: response.status, body: await response.json() }; + }; + const saved = await call({ operation: "configure", revision: 0, configuration: { destination, rules } }); + assert.equal(saved.status, 200); + return { db, env, now, call, rules: saved.body.rules }; +} +const observation = (rule, value, window = "2026-10-09") => ({ rule_id: rule.id, value, window, + basis: rule.kind === "spend" ? "gateway_cost_estimate" : rule.kind === "unknown_price" ? "unknown_price_coverage" : "observed_health" }); + + +test("disabled and empty allowlist never read state, collect or transport", async () => { + for (const flag of [undefined, "", "false", "invalid"]) { + const response = await handleAlertState({ GATEWAY_ALERTS_ENABLED: flag }, { operation: "get" }); + assert.equal(response.status, 404); + } + for (const allowlist of ["", "[]", "invalid"]) { + const result = await runAlertDelivery({ ...envBase, GATEWAY_ALERT_WEBHOOK_ALLOWLIST: allowlist }, { + collect() { throw Error("collector touched"); }, transport() { throw Error("transport touched"); }, + }); + assert.deepEqual(result, { attempted: 0, delivered: 0, failed: 0 }); + } +}); + + +test("strict destinations reject SSRF and redirects before transport", () => { + const settings = alertSettings(envBase); + for (const value of ["http://hooks.example", "https://evil.example", "https://localhost", "https://127.0.0.1", + "https://169.254.169.254", "https://[::1]", "https://hooks.example@evil.example", "https://hooks.example/?secret=1", + "https://hooks.example/#x", "https://hooks.example\\@evil.example", "https://hooks.example/\nprivate"]) { + assert.throws(() => validateDestination(value, settings.allowlist)); + } + assert.equal(validateDestination(destination, settings.allowlist), destination); +}); + + +test("configuration CAS defaults, bounded private status and additive migration", async t => { + const { db, call, rules: saved } = await setup(t); + assert.deepEqual(saved[0].thresholds, [85, 100]); + assert.equal((await call({ operation: "configure", revision: 0, configuration: { destination, rules } })).status, 409); + assert.equal((await db.prepare("SELECT value FROM old_rows").first()).value, "existing"); + const status = (await call({ operation: "get" })).body; + assert.equal(status.webhook_configured, true); + assert.equal(JSON.stringify(status).includes("private-recipient"), false); + assert.deepEqual(status.events, []); +}); + + +test("missing tables and private driver failures return sanitized 503", async t => { + const { db, migrate } = database(t, false); + for (const operation of ["get", "configure", "observe"]) { + const response = await handleAlertState({ ...envBase, INTELLIGENCE_DB: db }, { + version: 1, operation, revision: 0, configuration: { destination, rules }, observations: [], + }); + assert.equal(response.status, 503); + assert.equal((await response.json()).error.code, "gateway_alert_storage_unavailable"); + } + migrate(); + assert.equal((await db.prepare("SELECT value FROM old_rows").first()).value, "existing"); + const response = await handleAlertState({ ...envBase, INTELLIGENCE_DB: { prepare() { throw Error("private-secret"); } } }, + { version: 1, operation: "get" }); + assert.equal(response.status, 503); + assert.equal((await response.text()).includes("private-secret"), false); +}); + + +test("thresholds dedupe per event, rule and revision for 15 minutes", async t => { + const { call, rules: saved, now } = await setup(t); + const observe = value => call({ operation: "observe", revision: 1, observations: [observation(saved[0], value)] }); + assert.equal((await observe(8.49)).body.queued, 0); + assert.equal((await observe(8.5)).body.queued, 1); + assert.equal((await observe(9)).body.queued, 0); + assert.equal((await observe(10)).body.queued, 1); + now[0] += 899; + assert.equal((await observe(10)).body.queued, 0); + now[0] += 1; + assert.equal((await observe(10)).body.queued, 2); + assert.equal((await call({ operation: "configure", revision: 1, configuration: { destination, rules } })).status, 200); + assert.equal((await call({ operation: "observe", revision: 1, observations: [] })).status, 409); + assert.equal((await call({ operation: "observe", revision: 2, observations: [observation(saved[0], 9)] })).body.queued, 1); +}); + + +test("unknown prices and passive provider health payloads contain no recipient or content", async t => { + const { call, env, rules: saved, now } = await setup(t); + const observations = [observation(saved[1], 30), observation(saved[2], 1, "current"), + observation(saved[3], 1, "current"), observation(saved[4], 3, "current")]; + assert.equal((await call({ operation: "observe", revision: 1, observations })).body.queued, 4); + const payloads = []; + const result = await runAlertDelivery(env, { clock: () => now[0], transport: async (url, options) => { + assert.equal(url, destination); + assert.equal(options.redirect, "error"); + assert.equal(options.timeoutMs, 3000); + assert.equal(options.signal instanceof AbortSignal, true); + assert.ok(new TextEncoder().encode(options.body).length <= 8192); + payloads.push(JSON.parse(options.body)); + return { status: 204 }; + } }); + assert.equal(result.delivered, 4); + assert.equal(payloads[0].basis, "unknown_price_coverage"); + assert.equal(payloads.some(row => JSON.stringify(row).includes("private")), false); + assert.equal((await call({ operation: "get" })).body.events.every(row => row.state === "delivered"), true); +}); + + +test("one scheduled run attempts at most ten events and three attempts total", async t => { + const { call, env, rules: saved, now } = await setup(t); + for (let index = 1; index <= 12; index++) { + await call({ operation: "observe", revision: 1, observations: [observation(saved[0], 9, `2026-10-${String(index).padStart(2, "0")}`)] }); + } + let calls = 0; + const transport = async () => { calls++; throw Error("private failure"); }; + assert.equal((await runAlertDelivery(env, { clock: () => now[0], transport })).attempted, 10); + for (let index = 0; index < 4; index++) { + now[0] += 61; + await runAlertDelivery(env, { clock: () => now[0], transport }); + } + assert.equal(calls, 36); + const status = (await call({ operation: "get" })).body; + assert.equal(status.events.length, 12); + assert.equal(status.events.every(row => row.state === "failed" && row.attempts === 3), true); + assert.equal(JSON.stringify(status).includes("private failure"), false); + const attempts = (await dbRows(env.INTELLIGENCE_DB)).map(row => JSON.parse(row.attempt_times)); + assert.equal(attempts.every(row => row.length === 3), true); +}); +async function dbRows(db) { return (await db.prepare("SELECT attempt_times FROM gateway_alert_events").all()).results; } + + +test("redirect status is a visible delivery failure without a second destination", async t => { + const { call, env, rules: saved, now } = await setup(t); + await call({ operation: "observe", revision: 1, observations: [observation(saved[0], 9)] }); + let calls = 0; + await runAlertDelivery(env, { clock: () => now[0], transport: async () => { calls++; return { status: 302 }; } }); + assert.equal(calls, 1); + const event = (await call({ operation: "get" })).body.events[0]; + assert.equal(event.state, "pending"); + assert.equal(event.error_code, "delivery_failed"); +}); + + +test("hard timeout bounds a transport ignoring abort", async t => { + const { call, env, rules: saved, now } = await setup(t); + await call({ operation: "observe", revision: 1, observations: [observation(saved[0], 9)] }); + let signal; + const started = Date.now(); + const result = await runAlertDelivery(env, { clock: () => now[0], transport: (_url, options) => { + signal = options.signal; return new Promise(() => {}); + } }); + assert.equal(result.attempted, 1); + assert.ok(Date.now() - started < 4500); + assert.equal(signal.aborted, true); + assert.equal((await call({ operation: "get" })).body.events[0].error_code, "delivery_failed"); +}); + + +test("concurrent observers and delivery runners claim only once", async t => { + const { call, env, rules: saved, now } = await setup(t); + const results = await Promise.all(Array.from({ length: 4 }, () => call({ operation: "observe", revision: 1, + observations: [observation(saved[0], 9)] }))); + assert.equal(results.reduce((sum, row) => sum + row.body.queued, 0), 1); + let calls = 0; + const transport = async () => { calls++; return { status: 204 }; }; + await Promise.all(Array.from({ length: 3 }, () => runAlertDelivery(env, { clock: () => now[0], transport }))); + assert.equal(calls, 1); +}); + + +test("untrusted observations and stored oversized payloads never reach transport", async t => { + const { call, env, rules: saved, now, db } = await setup(t); + for (const change of [{ prompt: "private" }, { value: Infinity }, { basis: "actual_invoice" }, + { rule_id: "private-person" }, { window: "private-person" }]) { + assert.equal((await call({ operation: "observe", revision: 1, observations: [{ ...observation(saved[0], 9), ...change }] })).status, 400); + } + await call({ operation: "observe", revision: 1, observations: [observation(saved[0], 9)] }); + await assert.rejects(db.prepare("UPDATE gateway_alert_events SET payload = ?").bind(JSON.stringify({ prompt: "x".repeat(9000) })).run()); + await db.prepare("UPDATE gateway_alert_events SET payload = ?").bind(JSON.stringify({ prompt: "漢".repeat(3000) })).run(); + let calls = 0; + const result = await runAlertDelivery(env, { clock: () => now[0], transport: async () => { calls++; return { status: 204 }; } }); + assert.equal(calls, 0); + assert.equal(result.failed, 1); +}); + + +test("allowlist changes and absent transport do not consume attempts", async t => { + const { call, env, rules: saved, now } = await setup(t); + await call({ operation: "observe", revision: 1, observations: [observation(saved[0], 9)] }); + assert.equal((await runAlertDelivery(env, { clock: () => now[0] })).attempted, 0); + const result = await runAlertDelivery({ ...env, GATEWAY_ALERT_WEBHOOK_ALLOWLIST: '["https://other.example"]' }, { + clock: () => now[0], transport() { throw Error("must not send"); }, + }); + assert.equal(result.attempted, 0); + assert.equal((await call({ operation: "get" })).body.events[0].attempts, 0); +}); + + +test("fractional configuration has the same canonical rule identity in both runtimes", async () => { + const [rule] = await normalizeAlertRules([{ kind: "spend", period: "month", budget_usd: 0.000001, thresholds: [0.001, 85.5, 100.0] }]); + assert.equal(rule.id, "f699ee8b115fb123a1f3f9504e7004a9123d853c7886c9c767c4066f5e648d61"); +}); + + +test("scheduled collector is a bounded aggregate collaborator", async t => { + const { env, rules: saved, now, call } = await setup(t); + let calls = 0; + const result = await runAlertDelivery(env, { clock: () => now[0], + collect: async rules => { assert.deepEqual(rules, saved); return [observation(rules[0], 9)]; }, + transport: async () => { calls++; return { status: 204 }; }, + }); + assert.equal(result.delivered, 1); + assert.equal(calls, 1); + const rejected = await runAlertDelivery(env, { clock: () => now[0], collect: async () => [{ prompt: "private-content" }], + transport: async () => { throw Error("No delivery for invalid observations"); }, + }); + assert.equal(rejected.error, "gateway_alert_observation_unavailable"); + assert.equal((await call({ operation: "get" })).body.events.length, 1); +}); + + +test("expired history is never delivered and physical pruning is bounded", async t => { + const { env, now, rules: saved, call, db } = await setup(t); + await call({ operation: "observe", revision: 1, observations: [observation(saved[0], 9)] }); + const row = await db.prepare("SELECT * FROM gateway_alert_events").first(); + await db.prepare("UPDATE gateway_alert_events SET created_at = ?, next_attempt_at = ?").bind(now[0] - 8 * 86400, now[0] - 8 * 86400).run(); + await db.prepare(`WITH RECURSIVE nums(n) AS (SELECT 1 UNION ALL SELECT n+1 FROM nums WHERE n < 150) + INSERT INTO gateway_alert_events(dedupe_key,event_id,rule_id,revision,payload,created_at,state,next_attempt_at) + SELECT printf('%064x',n), printf('%064x',n), ?, 1, ?, ?, 'pending', ? FROM nums`) + .bind(row.rule_id, row.payload, now[0] - 8 * 86400, now[0] - 8 * 86400).run(); + let calls = 0; + const result = await runAlertDelivery(env, { clock: () => now[0], transport: async () => { calls++; return { status: 204 }; } }); + assert.equal(result.attempted, 0); + assert.equal(calls, 0); + assert.equal((await db.prepare("SELECT count(*) AS n FROM gateway_alert_events").first()).n, 51); + assert.equal((await call({ operation: "get" })).body.events.length, 0); +}); + + +test("configuration replacement cancels old waiting deliveries", async t => { + const { env, now, rules: saved, call } = await setup(t); + await call({ operation: "observe", revision: 1, observations: [observation(saved[0], 9)] }); + assert.equal((await call({ operation: "configure", revision: 1, configuration: { destination, rules: [] } })).status, 200); + const event = (await call({ operation: "get" })).body.events[0]; + assert.equal(event.state, "failed"); + assert.equal(event.error_code, "rule_replaced"); + assert.equal((await runAlertDelivery(env, { clock: () => now[0], transport() { throw Error("Old delivery must not run"); } })).attempted, 0); +}); + +test("status rejects malformed stored delivery metadata without exposing private values", async t => { + const { db, call, rules: saved } = await setup(t); + await call({ operation: "observe", revision: 1, observations: [observation(saved[0], 9)] }); + for (const [column, value] of [["event_id", "private-recipient"], ["error_code", "private-response"], + ["attempts", 1.5], ["last_attempt_at", "private-recipient"]]) { + await db.prepare(`UPDATE gateway_alert_events SET ${column} = ?`).bind(value).run(); + const response = await call({ operation: "get" }); + assert.equal(response.status, 503, column); + assert.equal(response.body.error.code, "gateway_alert_storage_unavailable"); + assert.equal(JSON.stringify(response.body).includes("private"), false); + await db.prepare(`UPDATE gateway_alert_events SET event_id = ?, error_code = NULL, attempts = 0, + last_attempt_at = NULL`).bind("a".repeat(64)).run(); + } +}); + +test("the durable queue cap rejects new distinct events and permits dedupe replacement", async t => { + const { db, call, now, rules: saved } = await setup(t); + await call({ operation: "observe", revision: 1, observations: [observation(saved[0], 9)] }); + const row = await db.prepare("SELECT * FROM gateway_alert_events").first(); + await db.prepare(`WITH RECURSIVE nums(n) AS (SELECT 1 UNION ALL SELECT n+1 FROM nums WHERE n < 1999) + INSERT INTO gateway_alert_events(dedupe_key,event_id,rule_id,revision,payload,created_at,state,next_attempt_at) + SELECT printf('%064x',n), printf('%064x',n), ?, 1, ?, ?, 'pending', ? FROM nums`) + .bind(row.rule_id, row.payload, now[0], now[0]).run(); + assert.equal((await call({ operation: "observe", revision: 1, + observations: [observation(saved[0], 9, "2026-10-10")] })).body.queued, 0); + now[0] += 900; + assert.equal((await call({ operation: "observe", revision: 1, + observations: [observation(saved[0], 9)] })).body.queued, 1); + assert.equal((await db.prepare("SELECT count(*) AS n FROM gateway_alert_events").first()).n, 2000); + const status = (await call({ operation: "get" })).body; + assert.equal(status.events.length, 100); + assert.ok(new TextEncoder().encode(JSON.stringify(status)).length <= 65536); +}); diff --git a/worker/alert-delivery.mjs b/worker/alert-delivery.mjs new file mode 100644 index 00000000..b960943b --- /dev/null +++ b/worker/alert-delivery.mjs @@ -0,0 +1,306 @@ +/** Content-free durable alerts; private state and scheduled delivery are fixed collaborators. */ +const MAX_BYTES = 8192; +const MAX_RULES = 20; +const MAX_EVENTS = 2000; +const RETENTION_SECONDS = 7 * 86400; +const DEDUPE_SECONDS = 900; +const TIMEOUT_MS = 3000; +const encoder = new TextEncoder(); +const warned = new Set(); +const ID = /^[0-9a-f]{64}$/; +const WINDOW = /^(?:current|[0-9]{4}-[0-9]{2}(?:-[0-9]{2})?)$/; +const KINDS = new Set(["spend", "unknown_price", "provider_circuit", "pool_exhaustion", "provider_health"]); +const PROVIDERS = new Set(["openai", "anthropic", "cerebras", "googleai", "xai", "groq", "together", "azure", + "scaleway", "hyperbolic", "sambanova", "openrouter", "opencode", "mimo", "nanogpt", "navyai", "linkapi", + "aihubmix", "codex-easy", "kimi-code", "cline-pass", "palm", "nineteen", "chutes", "gemini", "gemma"]); +const basisFor = kind => kind === "spend" ? "gateway_cost_estimate" : kind === "unknown_price" ? "unknown_price_coverage" : "observed_health"; +const object = value => value !== null && typeof value === "object" && !Array.isArray(value); +const fields = (value, names) => object(value) && Object.keys(value).length === names.length && names.every(key => Object.hasOwn(value, key)); +const number = value => typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1e12; +const revision = value => Number.isSafeInteger(value) && value >= 0 && value < Number.MAX_SAFE_INTEGER; +const matched = (pattern, value) => typeof value === "string" && pattern.test(value); +const reply = (value, status = 200) => Response.json({ version: 1, ...value }, { status, headers: { "cache-control": "no-store" } }); +const error = (code, status = 400) => reply({ error: { code } }, status); +function warn(name) { + if (!warned.has(name)) { warned.add(name); console.warn(`Invalid gateway alert setting (${name})`); } +} +function bounded(value, limit = MAX_BYTES) { + const text = JSON.stringify(value); + if (typeof text !== "string" || encoder.encode(text).length > limit) throw Error("Invalid alert size"); + return text; +} +async function digest(value) { + return Array.from(new Uint8Array(await crypto.subtle.digest("SHA-256", encoder.encode(value)))) + .map(byte => byte.toString(16).padStart(2, "0")).join(""); +} +function numericIdentity(value) { + if (typeof value !== "number") return value; + const buffer = new ArrayBuffer(8); + new DataView(buffer).setFloat64(0, value); + return "n:" + Array.from(new Uint8Array(buffer)).map(byte => byte.toString(16).padStart(2, "0")).join(""); +} +function canonical(rule) { + return JSON.stringify(Object.fromEntries(Object.keys(rule).sort().map(key => [key, + Array.isArray(rule[key]) ? rule[key].map(numericIdentity) : numericIdentity(rule[key])]))); +} +function origin(value) { + if (typeof value !== "string" || value.length > 2048 || /[^\x21-\x7e]|\\/.test(value)) throw Error("Invalid destination"); + const url = new URL(value); + const host = url.hostname; + if (url.protocol !== "https:" || url.username || url.password || url.port || value.includes("?") || value.includes("#") + || !/^[a-z0-9]+(?:[a-z0-9-]*[a-z0-9])?(?:\.[a-z0-9]+(?:[a-z0-9-]*[a-z0-9])?)+$/.test(host) + || /\.(?:localhost|local|internal|invalid|test|home|lan|[0-9]+)$/.test(host) || host.startsWith("metadata.")) throw Error("Invalid destination"); + return url.origin; +} +export function alertSettings(env) { + const flag = String(env.GATEWAY_ALERTS_ENABLED ?? "").trim().toLowerCase(); + let enabled = ["1", "true", "yes", "on"].includes(flag); + if (!["", "0", "1", "false", "true", "no", "yes", "off", "on"].includes(flag)) warn("GATEWAY_ALERTS_ENABLED"); + let allowlist = new Set(); + try { + const list = JSON.parse(String(env.GATEWAY_ALERT_WEBHOOK_ALLOWLIST ?? "").trim() || "[]"); + if (!Array.isArray(list) || list.length > 20) throw Error("Invalid allowlist"); + for (const value of list) { + const item = origin(value); + if (![item, item + "/", item + ":443", item + ":443/"].includes(value)) throw Error("Invalid origin"); + allowlist.add(item); + } + } catch { enabled = false; allowlist = new Set(); warn("GATEWAY_ALERT_WEBHOOK_ALLOWLIST"); } + return { enabled, allowlist }; +} +export function validateDestination(value, allowlist) { + if (!allowlist.has(origin(value))) throw Error("Destination not allowed"); + return value; +} +export async function normalizeAlertRules(values, providers = PROVIDERS) { + if (!Array.isArray(values) || values.length > MAX_RULES) throw Error("Invalid rules"); + const result = [], seen = new Set(); + for (const input of values) { + if (!object(input) || !KINDS.has(input.kind)) throw Error("Invalid kind"); + const { id, ...value } = input; + const kind = value.kind; + const allowed = kind === "spend" ? ["kind", "period", "budget_usd", "thresholds"] + : kind === "unknown_price" ? ["kind", "period", "threshold_percent"] + : kind === "provider_health" ? ["kind", "provider", "failures"] : ["kind", "provider"]; + if (Object.keys(value).some(key => !allowed.includes(key))) throw Error("Invalid rule fields"); + const rule = { kind }; + if (["spend", "unknown_price"].includes(kind)) { + if (!["day", "month"].includes(value.period)) throw Error("Invalid period"); + rule.period = value.period; + } else { + if (!providers.has(value.provider)) throw Error("Invalid provider"); + rule.provider = value.provider; + } + if (kind === "spend") { + const thresholds = value.thresholds ?? [85, 100]; + if (!number(value.budget_usd) || !value.budget_usd || !Array.isArray(thresholds) || thresholds.length < 1 || thresholds.length > 4 + || thresholds.some(t => !number(t) || t <= 0 || t > 100) || new Set(thresholds).size !== thresholds.length) throw Error("Invalid spend rule"); + rule.budget_usd = value.budget_usd; + rule.thresholds = [...thresholds].sort((a, b) => a - b); + } else if (kind === "unknown_price") { + rule.threshold_percent = value.threshold_percent ?? 0; + if (!number(rule.threshold_percent) || rule.threshold_percent > 100) throw Error("Invalid coverage rule"); + } else if (kind === "provider_health") { + rule.failures = value.failures ?? 3; + if (!Number.isSafeInteger(rule.failures) || rule.failures < 1 || rule.failures > 100) throw Error("Invalid health rule"); + } + const identifier = await digest(canonical(rule)); + if (seen.has(identifier) || id !== undefined && id !== identifier) throw Error("Invalid rule identity"); + seen.add(identifier); + result.push({ ...rule, id: identifier }); + } + bounded(result); + return result; +} +function validPayload(value, providers) { + return fields(value, ["event", "rule_id", "revision", "occurred_at", "basis", "value", "threshold", "window", "provider"]) + && KINDS.has(value.event) && matched(ID, value.rule_id) && revision(value.revision) && number(value.occurred_at) + && number(value.value) && number(value.threshold) && value.basis === basisFor(value.event) && matched(WINDOW, value.window) + && (value.provider === null || providers.has(value.provider)); +} +async function readConfig(db, providers) { + const row = await db.prepare("SELECT revision, configuration FROM gateway_alert_rules WHERE id = 1").first(); + if (!row) return { revision: 0, rules: [], destination: null }; + const value = JSON.parse(row.configuration); + if (!revision(row.revision) || !fields(value, ["destination", "rules"])) throw Error("Invalid stored configuration"); + origin(value.destination); + const rules = await normalizeAlertRules(value.rules, providers); + return { revision: row.revision, rules, destination: value.destination }; +} +async function status(db, config, now, providers) { + const { results } = await db.prepare(`SELECT event_id, state, attempts, last_attempt_at, error_code, payload + FROM gateway_alert_events WHERE created_at > ? ORDER BY created_at DESC, event_id LIMIT 100`).bind(now - RETENTION_SECONDS).all(); + const events = results.map(row => { + const payload = JSON.parse(row.payload); + if (!validPayload(payload, providers) || !matched(ID, row.event_id) + || !["pending", "delivered", "failed"].includes(row.state) + || !Number.isSafeInteger(row.attempts) || row.attempts < 0 || row.attempts > 3 + || row.last_attempt_at !== null && !number(row.last_attempt_at) + || ![null, "delivery_failed", "rule_replaced", "invalid_payload", "attempts_exhausted"].includes(row.error_code)) { + throw Error("Invalid stored event"); + } + return { id: row.event_id, state: row.state, attempts: row.attempts, last_attempt_at: row.last_attempt_at, + error_code: row.error_code, payload }; + }); + const result = { revision: config.revision, rules: config.rules, webhook_configured: config.destination !== null, events }; + bounded(result, 65536); + return reply(result); +} +async function configure(db, body, settings, now, providers) { + if (!fields(body, ["version", "operation", "revision", "configuration"]) || !revision(body.revision) + || body.revision >= Number.MAX_SAFE_INTEGER - 1 || !fields(body.configuration, ["destination", "rules"])) return error("invalid_gateway_alert"); + let configuration; + try { + configuration = { destination: validateDestination(body.configuration.destination, settings.allowlist), + rules: await normalizeAlertRules(body.configuration.rules, providers) }; + bounded(configuration); + } catch { return error("invalid_gateway_alert"); } + const next = body.revision + 1; + const results = await db.batch([ + db.prepare(`INSERT INTO gateway_alert_rules(id, revision, configuration, updated_at) + SELECT 1, ?1, ?2, ?3 WHERE ?4 = 0 OR EXISTS (SELECT 1 FROM gateway_alert_rules WHERE id = 1 AND revision = ?4) + ON CONFLICT(id) DO UPDATE SET revision = excluded.revision, configuration = excluded.configuration, updated_at = excluded.updated_at + WHERE gateway_alert_rules.revision = ?4`).bind(next, bounded(configuration), now, body.revision), + db.prepare(`UPDATE gateway_alert_events SET state = 'failed', error_code = 'rule_replaced', claim_token = NULL, lease_until = 0 + WHERE state = 'pending' AND revision <> ?1 AND EXISTS (SELECT 1 FROM gateway_alert_rules WHERE id = 1 AND revision = ?1)`).bind(next), + ]); + if (results[0].meta.changes !== 1) return error("gateway_alert_revision_conflict", 409); + return status(db, { revision: next, ...configuration }, now, providers); +} +function validObservation(row, rule) { + return fields(row, ["rule_id", "value", "basis", "window"]) && matched(ID, row.rule_id) && rule + && number(row.value) && row.basis === basisFor(rule.kind) && matched(WINDOW, row.window) + && (rule.period === "day" ? /^[0-9]{4}-[0-9]{2}-[0-9]{2}$/.test(row.window) + : rule.period === "month" ? /^[0-9]{4}-[0-9]{2}$/.test(row.window) : row.window === "current") + && (rule.kind !== "unknown_price" || row.value <= 100) + && (!["provider_circuit", "pool_exhaustion"].includes(rule.kind) || [0, 1].includes(row.value)); +} +async function enqueue(db, config, observations, now) { + const byId = new Map(config.rules.map(rule => [rule.id, rule])); + if (!Array.isArray(observations) || observations.length > MAX_RULES || new Set(observations.map(row => row?.rule_id)).size !== observations.length + || observations.some(row => !validObservation(row, byId.get(row?.rule_id)))) return error("invalid_gateway_alert"); + const statements = []; + for (const row of observations) { + const rule = byId.get(row.rule_id); + const thresholds = rule.kind === "spend" ? rule.thresholds : [rule.kind === "unknown_price" ? rule.threshold_percent : rule.failures ?? 1]; + const measured = rule.kind === "spend" ? row.value / rule.budget_usd * 100 : row.value; + for (const threshold of thresholds) { + if (measured < threshold || rule.kind === "unknown_price" && row.value === 0) continue; + const dedupe = await digest(canonical({ kind: rule.kind, rule_id: rule.id, revision: config.revision, threshold, window: row.window })); + const payload = { event: rule.kind, rule_id: rule.id, revision: config.revision, occurred_at: now, + basis: row.basis, value: row.value, threshold, window: row.window, provider: rule.provider ?? null }; + const eventId = await digest(dedupe + ":" + crypto.randomUUID()); + statements.push(db.prepare(`INSERT INTO gateway_alert_events + (dedupe_key, event_id, rule_id, revision, payload, created_at, state, next_attempt_at) + SELECT ?1, ?2, ?3, ?4, ?5, ?6, 'pending', ?6 + WHERE EXISTS (SELECT 1 FROM gateway_alert_rules WHERE id = 1 AND revision = ?4) + AND ((SELECT COUNT(*) FROM gateway_alert_events) < ${MAX_EVENTS} OR EXISTS (SELECT 1 FROM gateway_alert_events WHERE dedupe_key = ?1)) + ON CONFLICT(dedupe_key) DO UPDATE SET event_id = excluded.event_id, payload = excluded.payload, + created_at = excluded.created_at, state = 'pending', attempts = 0, attempt_times = '[]', last_attempt_at = NULL, + next_attempt_at = excluded.next_attempt_at, claim_token = NULL, lease_until = 0, error_code = NULL + WHERE gateway_alert_events.created_at <= ?7 AND gateway_alert_events.lease_until <= ?6`) + .bind(dedupe, eventId, rule.id, config.revision, bounded(payload), now, now - DEDUPE_SECONDS)); + } + } + if (!statements.length) return reply({ queued: 0 }); + const results = await db.batch(statements); + return reply({ queued: results.reduce((sum, row) => sum + row.meta.changes, 0) }); +} +/** Call only from the authenticated private managed-state dispatcher. */ +export async function handleAlertState(env, body, { clock = () => Date.now() / 1000, providers = PROVIDERS } = {}) { + const settings = alertSettings(env); + if (!settings.enabled) return error("not_found", 404); + if (!object(body) || body.version !== 1 || !["get", "configure", "observe"].includes(body.operation)) return error("invalid_gateway_alert"); + try { bounded(body); } catch { return error("gateway_alert_too_large", 413); } + const db = env.INTELLIGENCE_DB, now = clock(); + if (!db || !number(now)) return error("gateway_alert_storage_unavailable", 503); + try { + const config = await readConfig(db, providers); + if (body.operation === "get") { + if (!fields(body, ["version", "operation"])) return error("invalid_gateway_alert"); + return await status(db, config, now, providers); + } + if (body.operation === "configure") return await configure(db, body, settings, now, providers); + if (!fields(body, ["version", "operation", "revision", "observations"]) || !revision(body.revision)) return error("invalid_gateway_alert"); + if (body.revision !== config.revision) return error("gateway_alert_revision_conflict", 409); + return await enqueue(db, config, body.observations, now); + } catch { return error("gateway_alert_storage_unavailable", 503); } +} +async function prune(db, now) { + await db.prepare(`DELETE FROM gateway_alert_events WHERE dedupe_key IN ( + SELECT dedupe_key FROM gateway_alert_events WHERE created_at <= ? AND lease_until <= ? LIMIT 100)`) + .bind(now - RETENTION_SECONDS, now).run(); + await db.prepare(`UPDATE gateway_alert_events SET state = 'failed', error_code = 'attempts_exhausted', claim_token = NULL, lease_until = 0 + WHERE state = 'pending' AND attempts >= 3 AND lease_until <= ?`).bind(now).run(); +} +async function claim(db, config, now) { + const token = crypto.randomUUID(); + const result = await db.prepare(`UPDATE gateway_alert_events SET claim_token = ?1, lease_until = ?2, attempts = attempts + 1, + last_attempt_at = ?3, attempt_times = json_insert(attempt_times, '$[#]', ?3) + WHERE dedupe_key IN (SELECT dedupe_key FROM gateway_alert_events WHERE state = 'pending' AND attempts < 3 + AND next_attempt_at <= ?3 AND lease_until <= ?3 AND revision = ?4 AND created_at > ?5 ORDER BY next_attempt_at, created_at LIMIT 10) + AND EXISTS (SELECT 1 FROM gateway_alert_rules WHERE id = 1 AND revision = ?4) + RETURNING event_id, payload, attempts, claim_token`).bind(token, now + 45, now, config.revision, now - RETENTION_SECONDS).all(); + return result.results; +} +async function finish(db, row, now, succeeded, code) { + await db.prepare(`UPDATE gateway_alert_events SET state = ?1, error_code = ?2, next_attempt_at = ?3, + claim_token = NULL, lease_until = 0 WHERE event_id = ?4 AND claim_token = ?5 AND state = 'pending'`) + .bind(succeeded ? "delivered" : row.attempts >= 3 || code === "invalid_payload" ? "failed" : "pending", + succeeded ? null : code, now + 60, row.event_id, row.claim_token).run(); +} +async function deliver(transport, destination, payload) { + const controller = new AbortController(); + let timer; + try { + const timeout = new Promise((_, reject) => { timer = setTimeout(() => { controller.abort(); reject(Error("Delivery timeout")); }, TIMEOUT_MS); }); + const response = await Promise.race([Promise.resolve().then(() => transport(destination, { + method: "POST", headers: { "content-type": "application/json" }, body: bounded(payload), + redirect: "error", signal: controller.signal, timeoutMs: TIMEOUT_MS, + })), timeout]); + return Number.isInteger(response?.status) && response.status >= 200 && response.status < 300 && !response.redirected; + } catch { return false; } + finally { controller.abort(); clearTimeout(timer); } +} +/** The scheduler supplies one reviewed metrics collector and a redirect-rejecting transport. */ +export async function runAlertDelivery(env, { collect, transport, clock = () => Date.now() / 1000, providers = PROVIDERS } = {}) { + const result = { attempted: 0, delivered: 0, failed: 0 }; + const settings = alertSettings(env); + if (!settings.enabled || settings.allowlist.size === 0 || typeof transport !== "function") return result; + const db = env.INTELLIGENCE_DB; + if (!db) return { ...result, error: "gateway_alert_storage_unavailable" }; + try { + const now = clock(); + if (!number(now)) throw Error("Invalid time"); + const config = await readConfig(db, providers); + if (!config.destination || !config.rules.length) return result; + try { validateDestination(config.destination, settings.allowlist); } catch { return result; } + await prune(db, now); + if (typeof collect === "function") { + const observations = await collect(config.rules); + bounded(observations); + const queued = await enqueue(db, config, observations, now); + if (queued.status !== 200) return { ...result, error: "gateway_alert_observation_unavailable" }; + } + const rows = await claim(db, config, now); + // Parallel deliveries finish within one timeout window, before the lease expires. + await Promise.all(rows.map(async row => { + result.attempted++; + let payload; + try { + bounded(row.payload); + payload = JSON.parse(row.payload); + if (!validPayload(payload, providers)) throw Error("Invalid payload"); + } catch { + result.failed++; + await finish(db, row, clock(), false, "invalid_payload"); + return; + } + const succeeded = await deliver(transport, config.destination, payload); + if (succeeded) result.delivered++; + else if (row.attempts === 3) result.failed++; + await finish(db, row, clock(), succeeded, "delivery_failed"); + })); + return result; + } catch { return { ...result, error: "gateway_alert_storage_unavailable" }; } +} diff --git a/worker/container-env.mjs b/worker/container-env.mjs index d57557bc..636827a8 100644 --- a/worker/container-env.mjs +++ b/worker/container-env.mjs @@ -37,6 +37,8 @@ const DIRECT_ENV_KEYS = [ "CONNECTION_PROFILES_DB_PATH", "CONFIG_REVISION_SYNC_ENABLED", "CONFIG_SYNC_TTL_SECONDS", + "GATEWAY_ALERTS_ENABLED", + "GATEWAY_ALERT_WEBHOOK_ALLOWLIST", "CONFIG_SECURITY_TTL_SECONDS", "RATE_LIMIT_ENABLED", "RATE_LIMIT_RPM", diff --git a/worker/d1-schema.mjs b/worker/d1-schema.mjs index d39d522d..e927bcae 100644 --- a/worker/d1-schema.mjs +++ b/worker/d1-schema.mjs @@ -10,7 +10,7 @@ export const REQUIRED_D1_TABLES = Object.freeze(["intelligence_policy", "intelli "usage_events", "usage_daily", "usage_batches", "media_jobs", "media_job_items", "shadow_eval_samples", "shadow_eval_results", "shadow_eval_config", "shadow_eval_policy_backups", "prompt_templates", - "config_snapshot_revisions", "config_snapshots", "config_snapshot_applications", "control_revisions", "tool_grants", "generation_cache", "usage_reservation_budgets", "usage_reservations", "usage_reservation_transitions", "session_tiers"]); + "config_snapshot_revisions", "config_snapshots", "config_snapshot_applications", "control_revisions", "tool_grants", "generation_cache", "usage_reservation_budgets", "usage_reservations", "usage_reservation_transitions", "session_tiers", "gateway_alert_rules", "gateway_alert_events"]); /** * Readiness for the D1 schema. A deploy that skipped `wrangler d1 migrations apply` reports