diff --git a/cloudflare-worker.mjs b/cloudflare-worker.mjs index 4578979d..4a8c358f 100644 --- a/cloudflare-worker.mjs +++ b/cloudflare-worker.mjs @@ -8,6 +8,7 @@ import { handleAiOutbound } from "./worker/ai-outbound.mjs"; import { handleMediaOutbound } from "./worker/media-outbound.mjs"; import { serveSignedMediaFile, signedMediaFileId } from "./worker/media-files.mjs"; import { MediaJobWorkflow } from "./worker/media-workflow.mjs"; +import { AdmissionCoordinator } from "./worker/admission-do.mjs"; import { handleKnowledgeEdgeRequest, isKnowledgeEdgePath } from "./worker/knowledge-edge.mjs"; import { withAccessIdentity } from "./worker/access-sso.mjs"; import { runScheduledShadowEval } from "./worker/shadow-eval-schedule.mjs"; @@ -41,6 +42,7 @@ import { withJanitorGlmReasoningNormalization } from "./worker/janitor-reasoning export { RoleplaySession }; +export { AdmissionCoordinator }; export { MediaJobWorkflow }; import { CORS_ALLOWED_METHODS, CORS_DEFAULT_HEADERS, CORS_EXPOSE_HEADERS } from "./worker/cors-policy.mjs"; diff --git a/docs/admission-leases.md b/docs/admission-leases.md new file mode 100644 index 00000000..a8642de2 --- /dev/null +++ b/docs/admission-leases.md @@ -0,0 +1,118 @@ +# Shared keyed concurrency admission + +Admission is off by default. `ADMISSION_ENABLED=false` (or empty) leaves dispatch, +response bytes, headers and storage unchanged. No lease call or Durable Object access +occurs. Invalid flags or limits disable admission and produce one fixed warning per +process/isolate without the invalid value. `ADMISSION_LIMITS_JSON={}` (or empty) is +unlimited even when the flag is true. There is no default heavy-model gate. + +## Limits and identity + +Enable with `ADMISSION_ENABLED=true` and, for example: + +```json +{"principal": 4, "model_groups": {"heavy": 1, "fast": 3}} +``` + +`principal` limits all active requests for one authenticated key identity across its +model groups. Each `model_groups` entry limits that same principal within the named +**authorized** group. Missing or zero limits are unlimited. Limits are integers from +0 through 10,000; at most 256 group entries are accepted. Unknown fields disable the +configuration. These are concurrency limits, not RPM, token or upstream quota limits. +Both applicable limits are checked atomically before dispatch. Lowering a limit does +not revoke existing work; new acquisitions wait for capacity. + +The authentication/routing registrar supplies an `AdmissionIdentity` (Python) or an +explicit object (Worker) with `principal_hash`, `model_group`, `request_id`, and +`deadline_ms`. The principal is a stable opaque SHA-256 identity for the authenticated +key, shared between runtimes. Do not supply raw keys, usernames, caller headers or +unverified body fields. Resolve/authorize the group before admission. The request ID +must identify one generation, including both runtimes when forwarding. The deadline +is a Unix timestamp in milliseconds, after now and at most 24 hours away; the registrar +must supply the actual eventual generation deadline, not a new deadline on renewal. + +## Authority and private operations + +`ADMISSION_COORDINATOR` uses `AdmissionCoordinator` with SQLite Durable Object storage. +The binding and `v3_admission` migration are additive; existing bindings/migrations +stay unchanged. The class is inert while disabled: construction, fetch and alarm make +no storage writes and schedule no alarms. Limits for a principal share one of 64 +stable shards, selected from its hash, so native Worker and Flask callers compete for +the same capacity. Each shard holds at most 10,000 outstanding leases, independent of +configured per-principal limits. This safety bound also returns 429. + +The fixed Container egress endpoint is `POST http://intelligence.internal/v1/admission`. +Container egress isolation is its authentication boundary, as with the other private +intelligence operations. It must remain absent from the public Worker router. Native +Worker hooks call the bound authority directly, after authentication. Durable Object +fetch accepts only `POST http://admission.internal/v1/admission`; it is not a public +HTTP authority. A public origin, query, fragment, alternate method, unknown field or +client-supplied limit/key is rejected. The strict version-1 body contains the identity +fields and `operation` (`acquire`, `renew`, `release`); renew/release also require +`lease_id`. Lease IDs cannot release or renew another principal, group, request or +deadline. Repeated acquisition of a live request returns its existing lease without +extending it; changing the group/deadline is rejected. Release is idempotent. + +Standalone Flask has no global local semaphore. It needs an explicit private +`ADMISSION_AUTHORITY_URL`, for example `http://127.0.0.1:8081/v1/admission`, pointing to +a trusted private adapter implementing these same shared authority operations. That +adapter is an integration requirement, not provided as a public Flask route here. +Only private IPs, localhost or `.internal` hostnames with `/v1/admission` are accepted; +credentials, redirects, query strings and fragments are forbidden. The adapter must +be isolated/authenticated at the network or service boundary. Missing authority while +limited and enabled fails with 503. Container D1 mode uses the fixed private egress +URL by default. Explicit URLs take precedence. Requests bypass environment proxies, +use one POST with no retries, 2-second connect/3-second inactivity timeouts, and bound +response decoding to 4 KiB and a 5-second decoding budget. The Requests timeout is +not a hard overall wall-clock deadline during a stalled setup. Worker authority calls +have a 5-second wait bound; a lost reply is never permission to replay generation. + +## Lease lifecycle + +A lease lives for 30 seconds, capped by the generation deadline. Renew every 10 seconds. +Python has one lazy daemon renewal scheduler per process, shared by clients, with +bounded local lease bookkeeping. It performs private operations sequentially, so very +large active sets or a slow authority can cause leases to expire; it never admits on +that basis. Worker leases own a renewal timer and an expiry timer. Renewal cannot +extend the generation deadline or revive an expired lease. Lost renewal stops further +output and calls the injected cancellation collaborator. Wire `on_lost` (Python) or +`onLost` (Worker) to the actual upstream cancellation owner. A blocking dispatch cannot +be forcibly interrupted by a lease helper without that collaborator. + +`dispatch_with_admission(identity, dispatch, client, on_lost=...)` wraps a normalized +Flask Response; `runWithAdmission(identity, env, dispatch, options)` wraps native Worker +dispatch. Worker options accept the request `signal`, `onLost`, and an injectable +millisecond clock. Use one lifecycle owner per generation. Forwarded work must be +admitted by Flask only; the Worker must not acquire an additional lease for forwarding. +A pre-dispatch/preflight rejection, response construction failure, normal completion, +stream error, disconnect or deadline releases once. Streaming wrappers pass chunks +unchanged and do not buffer the full response. Closing a Flask stream before its first +iteration still closes its source and releases its lease. WSGI detects disconnects +only on writes/iteration/close; cancellation during blocking setup needs the transport +collaborator. Release failure is not replayed and does not rewrite provider output; +the authority reclaims the slot at lease expiry. A process crash has the same bound. + +Expired leases stop counting immediately and are removed on the next authority +operation using an expiry index. No cleanup alarms are scheduled. An idle shard may +retain expired metadata until its next operation. Only opaque principal identity, +authorized group, request ID, lease ID, deadline and expiry are stored; no request, +prompt, response, tool content or credential is stored or logged. SQLite records and +private calls incur normal platform storage/request costs. Admission does not claim +zero generation cost, successful output or replay permission after a partial result. + +## Errors and integration + +Capacity denial is a real 429 `admission_denied` with integer `Retry-After` seconds +until the earliest blocking lease expiry (minimum 1). Unavailable authority/storage or +unconfirmed acquisition while enabled is 503 `admission_unavailable`. Lease identity +mismatch is 403; expired renewal is 409 on the private API, translated to 503 by the +generation lifecycle. No success body is synthesized. Error responses are no-store. +Raw provider headers are preserved; only gateway denial adds Retry-After. + +Native Worker generations acquire their lease at the edge. Requests forwarded to the +Container acquire only in Flask, after authentication and before dispatch, so one +request never holds two leases. No public route is added; `/v1/admission` exists only +on the private intelligence transport. Set `ADMISSION_ENABLED`, `ADMISSION_LIMITS_JSON` +and, for a standalone Flask deployment, `ADMISSION_AUTHORITY_URL` on both the Worker and +the Container. No D1 tables are added. `wrangler deploy` applies the `v3_admission` +Durable Object migration; the class stays inert until the flag is enabled. diff --git a/middleware/admission.py b/middleware/admission.py new file mode 100644 index 00000000..47fd3fe9 --- /dev/null +++ b/middleware/admission.py @@ -0,0 +1,84 @@ +"""Explicit post-authentication admission hook for the Flask registrar.""" +import json + +from flask import Response + +from services.admission_leases import AdmissionError + + +def admission_error_response(error): + headers = {"Cache-Control": "no-store"} + if error.status == 429 and error.retry_after is not None: + headers["Retry-After"] = str(error.retry_after) + return Response(json.dumps({"error": {"code": error.code, "message": "Concurrency admission failed."}}), + status=error.status, headers=headers, content_type="application/json") + + +class _LeasedIterable: + def __init__(self, source, lease): + self.source, self.lease = source, lease + self.iterator = None + self.closed = False + + def __iter__(self): + return self + + def __next__(self): + if self.closed: + raise StopIteration + try: + self.lease.check() + if self.iterator is None: + self.iterator = iter(self.source) + chunk = next(self.iterator) + self.lease.check() + return chunk + except BaseException: + self.close() + raise + + def close(self): + if self.closed: + return + self.closed = True + try: + close = getattr(self.source, "close", None) + if close: + close() + finally: + self.lease.release() + + +def dispatch_with_admission(identity, dispatch, client, *, on_lost=None): + """Identity is supplied after auth/routing; dispatch returns a normalized Response. + + The registrar calls this before dispatch/preflight and supplies transport cancellation + through on_lost. Flask/WSGI detects disconnects on iteration or response close. + """ + try: + lease = client.acquire(identity, on_lost=on_lost) + except AdmissionError as error: + return admission_error_response(error) + if lease is None: + return dispatch() + response = None + try: + lease.check() + response = dispatch() + lease.check() + if not isinstance(response, Response): + raise TypeError("Admission dispatch must return a normalized Response") + if response.is_streamed: + response.response = _LeasedIterable(response.response, lease) + else: + lease.release() + return response + except BaseException as error: + try: + if response is not None and hasattr(response, "close"): + response.close() + finally: + lease.release() + if isinstance(error, AdmissionError): + return admission_error_response(error) + raise diff --git a/package.json b/package.json index 42ee5f27..4dc59531 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "lint": "ruff check . --select E9,F63,F7,F82", "typecheck": "mypy --ignore-missing-imports --follow-imports=skip --disable-error-code import-untyped --disable-error-code var-annotated --disable-error-code arg-type --disable-error-code assignment --disable-error-code operator app.py routes services providers streaming", "test": "python -m pytest -q", - "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs", + "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_admission_leases_worker.mjs", "db:migrate": "python scripts/validate_sqlite_schema.py --apply", "db:validate": "python scripts/validate_sqlite_schema.py", "security:secrets": "python scripts/check_static_secrets.py", diff --git a/services/admission_leases.py b/services/admission_leases.py new file mode 100644 index 00000000..6e60771e --- /dev/null +++ b/services/admission_leases.py @@ -0,0 +1,275 @@ +"""Private shared concurrency leases; local bookkeeping is never the authority.""" +from __future__ import annotations + +import ipaddress +import json +import logging +import os +import re +import threading +import time +import weakref +from dataclasses import dataclass +from urllib.parse import urlsplit + +import requests + +logger = logging.getLogger(__name__) +_HASH = re.compile(r"[0-9a-f]{64}\Z") +_GROUP = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:/+@-]{0,255}\Z") +_ID = re.compile(r"[A-Za-z0-9_.:-]{1,128}\Z") +_LEASE = re.compile(r"[0-9a-f]{32}\Z") +LEASE_MS = 30_000 +RENEW_MS = 10_000 +MAX_OUTSTANDING = 10_000 +_warned = False +_scheduler_lock = threading.Lock() +_clients: weakref.WeakSet = weakref.WeakSet() +_scheduler = None + + +class AdmissionError(Exception): + def __init__(self, code="admission_unavailable", status=503, retry_after=None): + super().__init__("Concurrency admission failed.") + self.code, self.status, self.retry_after = code, status, retry_after + + +@dataclass(frozen=True) +class AdmissionSettings: + enabled: bool = False + principal: int = 0 + groups: dict | None = None + + def limited(self, group=None): + groups = self.groups or {} + return bool(self.principal or (groups.get(group, 0) if group is not None else any(groups.values()))) + + +def admission_settings(env=None): + global _warned + env = os.environ if env is None else env + flag = str(env.get("ADMISSION_ENABLED", "")).strip().lower() + if flag in {"", "false", "0"}: + return AdmissionSettings() + try: + if flag not in {"true", "1"}: + raise ValueError("flag") + limits = json.loads(str(env.get("ADMISSION_LIMITS_JSON", "")).strip() or "{}") + def valid_limit(value): + return type(value) is int and 0 <= value <= MAX_OUTSTANDING + if not isinstance(limits, dict) or set(limits) - {"principal", "model_groups"}: + raise ValueError("limits") + if "principal" in limits and not valid_limit(limits["principal"]): + raise ValueError("principal") + groups = limits.get("model_groups", {}) + if (not isinstance(groups, dict) or len(groups) > 256 + or any(not _GROUP.fullmatch(key) or not valid_limit(value) for key, value in groups.items())): + raise ValueError("groups") + return AdmissionSettings(True, limits.get("principal", 0), groups) + except (ValueError, TypeError): + if not _warned: + _warned = True + logger.warning("Invalid admission configuration; admission is disabled.") + return AdmissionSettings() + + +@dataclass(frozen=True) +class AdmissionIdentity: + """Opaque authenticated key identity and authorized group, supplied by the registrar.""" + principal_hash: str + model_group: str + request_id: str + deadline_ms: int + + def valid(self): + return (isinstance(self.principal_hash, str) and _HASH.fullmatch(self.principal_hash) + and isinstance(self.model_group, str) and _GROUP.fullmatch(self.model_group) + and isinstance(self.request_id, str) and _ID.fullmatch(self.request_id) + and type(self.deadline_ms) is int and self.deadline_ms > 0) + + def payload(self, operation, lease_id=None): + result = {"version": 1, "operation": operation, "principal_hash": self.principal_hash, + "model_group": self.model_group, "request_id": self.request_id, "deadline_ms": self.deadline_ms} + if lease_id is not None: + result["lease_id"] = lease_id + return result + + +def _authority_url(env): + configured = str(env.get("ADMISSION_AUTHORITY_URL", "")).strip() + if not configured and env.get("INTELLIGENCE_STORAGE_BACKEND", "").strip() == "d1": + configured = "http://intelligence.internal/v1/admission" + parsed = urlsplit(configured) + host = parsed.hostname or "" + try: + private = ipaddress.ip_address(host).is_private + except ValueError: + private = host == "localhost" or host.endswith(".internal") + if (parsed.scheme not in {"http", "https"} or not private or parsed.username or parsed.password + or parsed.query or parsed.fragment or parsed.path != "/v1/admission"): + raise AdmissionError() + return configured + + +def _private_call(env, payload): + """Fixed private operation, one submission, no redirects, proxies or response content logs.""" + url = _authority_url(env) + with requests.Session() as session: + session.trust_env = False + with session.post(url, json=payload, headers={"Accept": "application/json", "Accept-Encoding": "identity"}, + timeout=(2, 3), allow_redirects=False, stream=True) as response: + if (response.headers.get("Content-Type", "").split(";", 1)[0].lower() != "application/json" + or response.headers.get("Content-Encoding", "identity").lower() != "identity"): + raise AdmissionError() + body = bytearray() + deadline = time.monotonic() + 5 + for chunk in response.iter_content(chunk_size=4096): + if len(body) + len(chunk) > 4096 or time.monotonic() >= deadline: + raise AdmissionError() + body.extend(chunk) + result = json.loads(body) + if not isinstance(result, dict) or type(result.get("version")) is not int or result["version"] != 1: + raise AdmissionError() + if response.status_code == 429: + retry = response.headers.get("Retry-After", "") + if not retry.isascii() or not retry.isdecimal() or not 1 <= int(retry) <= 86400: + raise AdmissionError() + raise AdmissionError("admission_denied", 429, int(retry)) + if response.status_code != 200 or "error" in result: + raise AdmissionError() + return result + + +def _renew_loop(): + while True: + time.sleep(1) + with _scheduler_lock: + clients = list(_clients) + for client in clients: + client.renew_due() + + +def _register_scheduler(client): + global _scheduler + with _scheduler_lock: + _clients.add(client) + if _scheduler is None or not _scheduler.is_alive(): + _scheduler = threading.Thread(target=_renew_loop, daemon=True, name="admission-renewal") + _scheduler.start() + + +class AdmissionClient: + def __init__(self, env=None, *, call=None, clock=None, background=True): + self.env = os.environ if env is None else env + self.call = call or (lambda payload: _private_call(self.env, payload)) + self.clock = clock or (lambda: int(time.time() * 1000)) + self.background = background + self._leases: set[AdmissionLease] = set() + self._lock = threading.Lock() + + def acquire(self, identity, *, on_lost=None): + settings = admission_settings(self.env) + if not settings.enabled or not settings.limited(): + return None + if not isinstance(identity, AdmissionIdentity) or not identity.valid(): + raise AdmissionError("admission_identity_invalid", 403) + if not settings.limited(identity.model_group): + return None + if identity.deadline_ms <= self.clock() or identity.deadline_ms > self.clock() + 86_400_000: + raise AdmissionError("admission_deadline_invalid", 403) + # This bound protects local renewal bookkeeping; all admission decisions stay remote. + with self._lock: + if len(self._leases) >= MAX_OUTSTANDING: + raise AdmissionError() + result = self.submit(identity.payload("acquire")) + value = result.get("lease") + self.validate(value, identity) + lease = AdmissionLease(self, identity, value, on_lost) + with self._lock: + self._leases.add(lease) + if self.background: + _register_scheduler(self) + return lease + + def submit(self, payload): + try: + result = self.call(payload) + if not isinstance(result, dict) or type(result.get("version")) is not int or result["version"] != 1 or "error" in result: + raise AdmissionError() + return result + except AdmissionError: + raise + except Exception: + raise AdmissionError() from None + + def validate(self, value, identity, lease_id=None): + if (not isinstance(value, dict) or not isinstance(value.get("lease_id"), str) + or not _LEASE.fullmatch(value["lease_id"]) or type(value.get("expires_at")) is not int + or not self.clock() < value["expires_at"] <= min(self.clock() + LEASE_MS, identity.deadline_ms) + or (lease_id is not None and value["lease_id"] != lease_id)): + raise AdmissionError() + + def forget(self, lease): + with self._lock: + self._leases.discard(lease) + + def renew_due(self): + with self._lock: + leases = list(self._leases) + for lease in leases: + lease.renew_due() + + +class AdmissionLease: + def __init__(self, client, identity, value, on_lost): + self.client, self.identity = client, identity + self.lease_id, self.expires_at = value["lease_id"], value["expires_at"] + self.next_renewal = client.clock() + RENEW_MS + self.on_lost = on_lost + self.closed = False + self.failure = None + self._lock = threading.RLock() + + def check(self): + with self._lock: + if self.failure: + raise self.failure + if self.closed or self.client.clock() >= min(self.expires_at, self.identity.deadline_ms): + raise AdmissionError() + + def renew_due(self): + with self._lock: + if self.closed: + return + now = self.client.clock() + try: + self.check() + if now < self.next_renewal: + return + result = self.client.submit(self.identity.payload("renew", self.lease_id)) + value = result.get("lease") + self.client.validate(value, self.identity, self.lease_id) + self.expires_at = value["expires_at"] + self.next_renewal = self.client.clock() + RENEW_MS + return + except AdmissionError as error: + self.failure = error + # Transport cancellation can wait for readers that call check(); never hold their lock. + self.release() + try: + if self.on_lost: + self.on_lost(self.failure) + except Exception: + logger.warning("Admission cancellation callback failed.") + + def release(self): + with self._lock: + if self.closed: + return + self.closed = True + self.client.forget(self) + try: + self.client.submit(self.identity.payload("release", self.lease_id)) + except AdmissionError: + # A lost release response is not replayed; shared expiry is the recovery path. + logger.warning("Admission release was not confirmed; lease will expire.") diff --git a/tests/fixtures/secret_firewall_coverage.json b/tests/fixtures/secret_firewall_coverage.json index ff0fe87b..97ae2bb0 100644 --- a/tests/fixtures/secret_firewall_coverage.json +++ b/tests/fixtures/secret_firewall_coverage.json @@ -216,6 +216,7 @@ }, "edge_modules": [ "worker/access-sso.mjs", + "worker/admission-do.mjs", "worker/ai-outbound.mjs", "worker/api-paths.mjs", "worker/auto-routes-d1.mjs", diff --git a/tests/helpers/load_cloudflare_worker.mjs b/tests/helpers/load_cloudflare_worker.mjs index 21205186..623f2975 100644 --- a/tests/helpers/load_cloudflare_worker.mjs +++ b/tests/helpers/load_cloudflare_worker.mjs @@ -739,6 +739,7 @@ export async function loadWorkerModule() { .replace('from "./worker/health-schedule.mjs";', `from "${new URL("../../worker/health-schedule.mjs", import.meta.url)}";`) .replace('from "./worker/status-page.mjs";', `from "${new URL("../../worker/status-page.mjs", import.meta.url)}";`) .replace('from "./worker/intelligence-outbound.mjs";', `from "${new URL("../../worker/intelligence-outbound.mjs", import.meta.url)}";`) + .replace('from "./worker/admission-do.mjs";', `from "${new URL("../../worker/admission-do.mjs", import.meta.url)}";`) .replace('from "./worker/client-headers.mjs";', `from "${new URL("../../worker/client-headers.mjs", import.meta.url)}";`) .replace('from "./worker/opencode-session.mjs";', `from "${new URL("../../worker/opencode-session.mjs", import.meta.url)}";`) .replace('from "./worker/opencode/reasoning-request.mjs";', `from "${new URL("../../worker/opencode/reasoning-request.mjs", import.meta.url)}";`) diff --git a/tests/test_admission_leases.py b/tests/test_admission_leases.py new file mode 100644 index 00000000..71a6e02b --- /dev/null +++ b/tests/test_admission_leases.py @@ -0,0 +1,305 @@ +"""Shared admission clients and response lifecycle with an injected private authority.""" +import json +from concurrent.futures import ThreadPoolExecutor +from threading import Lock + +import pytest +from flask import Flask, Response + +from middleware.admission import dispatch_with_admission +from services.admission_leases import AdmissionClient, AdmissionError, AdmissionIdentity, admission_settings + + +def identity(request_id="r1", principal_hash="a" * 64, model_group="heavy", deadline_ms=1_100_000): + return AdmissionIdentity(principal_hash, model_group, request_id, deadline_ms) + + +class Authority: + def __init__(self): + self.calls = [] + self.leases = {} + self.lock = Lock() + self.now = 1_000_000 + self.down = False + + def __call__(self, payload): + with self.lock: + self.calls.append(payload) + if self.down: + raise OSError("unavailable") + key = (payload["principal_hash"], payload["model_group"]) + operation = payload["operation"] + if operation == "acquire": + if key in self.leases: + raise AdmissionError("admission_denied", 429, 30) + lease = {"lease_id": payload["request_id"].encode().hex().ljust(32, "0"), + "expires_at": min(self.now + 30000, payload["deadline_ms"])} + self.leases[key] = lease + return {"version": 1, "lease": lease} + if operation == "release": + return {"version": 1, "released": self.leases.pop(key, None) is not None} + lease = self.leases[key] + lease["expires_at"] = min(self.now + 30000, payload["deadline_ms"]) + return {"version": 1, "lease": lease} + + +@pytest.fixture +def authority(): + return Authority() + + +def client(authority, **kwargs): + return AdmissionClient({"ADMISSION_ENABLED": "true", "ADMISSION_LIMITS_JSON": '{"model_groups":{"heavy":1}}'}, + call=authority, clock=lambda: authority.now, background=False, **kwargs) + + +@pytest.mark.parametrize("env", [{}, {"ADMISSION_ENABLED": ""}, {"ADMISSION_ENABLED": "bad"}, + {"ADMISSION_ENABLED": "true", "ADMISSION_LIMITS_JSON": "bad"}, + {"ADMISSION_ENABLED": "true", "ADMISSION_LIMITS_JSON": "{}"}]) +def test_disabled_unlimited_no_calls_no_headers_and_same_response(env, authority): + admission = AdmissionClient(env, call=authority, background=False) + response = Response("same", headers={"X-Provider": "raw"}) + assert dispatch_with_admission(None, lambda: response, admission) is response + assert list(response.headers) == [("X-Provider", "raw"), ("Content-Type", "text/html; charset=utf-8"), ("Content-Length", "4")] + assert authority.calls == [] + + +def test_two_flask_clients_share_limit_and_release_once(authority): + left, right = client(authority), client(authority) + def acquire(args): + admission, context = args + try: + return admission.acquire(context) + except AdmissionError as error: + return error + with ThreadPoolExecutor(2) as pool: + results = list(pool.map(acquire, [(left, identity()), (right, identity("r2"))])) + lease = next(item for item in results if not isinstance(item, AdmissionError)) + denial = next(item for item in results if isinstance(item, AdmissionError)) + assert (denial.status, denial.retry_after) == (429, 30) + lease.release() + lease.release() + assert [call["operation"] for call in authority.calls].count("release") == 1 + assert right.acquire(identity("r3")) is not None + + +def test_renewal_and_deadline_release_once(authority): + admission = client(authority) + lease = admission.acquire(identity()) + authority.now += 10000 + admission.renew_due() + assert lease.expires_at == 1_040_000 + authority.now = 1_100_000 + admission.renew_due() + assert lease.closed + lease.release() + assert [call["operation"] for call in authority.calls] == ["acquire", "renew", "release"] + + +def test_authority_failure_and_renewal_loss_are_503(authority): + admission = client(authority) + authority.down = True + with pytest.raises(AdmissionError) as error: + admission.acquire(identity()) + assert error.value.status == 503 + authority.down = False + lease = admission.acquire(identity()) + authority.down = True + authority.now += 10000 + admission.renew_due() + with pytest.raises(AdmissionError) as error: + lease.check() + assert error.value.status == 503 + assert lease.closed + + +@pytest.mark.parametrize("context", [identity(principal_hash="raw-key"), identity(model_group="bad group"), + identity(request_id=""), identity(deadline_ms=0)]) +def test_spoofed_or_invalid_identity_never_calls_authority(authority, context): + with pytest.raises(AdmissionError): + client(authority).acquire(context) + assert authority.calls == [] + + +def test_registered_route_denial_outage_headers_and_spoofed_public_headers(authority): + admission = client(authority) + held = admission.acquire(identity("held")) + app = Flask(__name__) + @app.post("/generate") + def generate(): + return dispatch_with_admission(identity("trusted"), lambda: Response("upstream"), admission) + with app.test_client() as browser: + response = browser.post("/generate", headers={"X-Principal": "b" * 64, "X-Model-Group": "small"}) + assert response.status_code == 429 + assert response.headers["Retry-After"] == "30" + assert response.json["error"]["code"] == "admission_denied" + assert authority.calls[-1]["principal_hash"] == "a" * 64 + held.release() + authority.down = True + assert browser.post("/generate").status_code == 503 + + +def test_completion_errors_and_unstarted_stream_close_release_once(authority): + admission = client(authority) + response = dispatch_with_admission(identity(), lambda: Response("body"), admission) + assert response.get_data() == b"body" + assert not authority.leases + def broken(): + raise RuntimeError("response error") + with pytest.raises(RuntimeError): + dispatch_with_admission(identity("error"), broken, admission) + source_closed = [] + class Source: + def __iter__(self): return self + def __next__(self): return b"data: same\n\n" + def close(self): source_closed.append(True) + response = dispatch_with_admission(identity("cancel"), lambda: Response(Source()), admission) + response.close() + response.close() + assert source_closed == [True] + assert not authority.leases + assert [call["operation"] for call in authority.calls].count("release") == 3 + + +def test_stream_iteration_error_and_bytes_preserved(authority): + admission = client(authority) + def source(): + yield b"data: unchanged\n\n" + raise RuntimeError("upstream error") + response = dispatch_with_admission(identity(), lambda: Response(source(), headers={"X-Provider": "raw"}), admission) + iterator = iter(response.response) + assert next(iterator) == b"data: unchanged\n\n" + with pytest.raises(RuntimeError): next(iterator) + response.close() + assert not authority.leases + assert response.headers["X-Provider"] == "raw" + assert [call["operation"] for call in authority.calls].count("release") == 1 + + +def test_standalone_requires_explicit_private_url_and_invalid_config_logs_once(authority, caplog, monkeypatch): + monkeypatch.setattr("services.admission_leases._warned", False) + admission = AdmissionClient({"ADMISSION_ENABLED": "true", "ADMISSION_LIMITS_JSON": '{"principal":1}'}, clock=lambda: authority.now, background=False) + with pytest.raises(AdmissionError) as error: admission.acquire(identity()) + assert error.value.status == 503 + for _ in range(2): + assert not admission_settings({"ADMISSION_ENABLED": "true", "ADMISSION_LIMITS_JSON": "secret malformed"}).enabled + assert caplog.text.count("Invalid admission configuration") == 1 + assert "secret malformed" not in caplog.text + for url in ["https://public.example/v1/admission", "http://127.0.0.1/v1/admission?x=1", "http://user:pass@localhost/v1/admission"]: + admission = AdmissionClient({"ADMISSION_ENABLED": "true", "ADMISSION_LIMITS_JSON": '{"principal":1}', + "ADMISSION_AUTHORITY_URL": url}, clock=lambda: authority.now, background=False) + with pytest.raises(AdmissionError): admission.acquire(identity()) + + +def test_no_identity_content_or_limit_override_in_wire_payload(authority): + lease = client(authority).acquire(identity()) + assert set(authority.calls[0]) == {"version", "operation", "principal_hash", "model_group", "request_id", "deadline_ms"} + lease.release() + assert "raw-key" not in json.dumps(authority.calls) + + +@pytest.mark.parametrize("status,retry,expected", [(200, None, 200), (429, "7", 429), (503, None, 503), + (429, "bad", 503), (302, None, 503)]) +def test_private_transport_single_submission_bounded_status_and_no_redirects(monkeypatch, status, retry, expected): + calls = [] + value = {"version": 1, "lease": {"lease_id": "f" * 32, "expires_at": 1_030_000}} + if status != 200: value = {"version": 1, "error": {"code": "admission_denied" if status == 429 else "down"}} + class Reply: + status_code = status + headers = {"Content-Type": "application/json", **({"Retry-After": retry} if retry else {})} + def __enter__(self): return self + def __exit__(self, *args): pass + def iter_content(self, chunk_size): yield json.dumps(value).encode() + class Session: + def __enter__(self): return self + def __exit__(self, *args): pass + def post(self, url, **kwargs): + assert self.trust_env is False + calls.append((url, kwargs)) + return Reply() + monkeypatch.setattr("services.admission_leases.requests.Session", Session) + admission = AdmissionClient({"ADMISSION_ENABLED": "true", "ADMISSION_LIMITS_JSON": '{"principal":1}', + "ADMISSION_AUTHORITY_URL": "http://127.0.0.1/v1/admission"}, + clock=lambda: 1_000_000, background=False) + if expected == 200: + lease = admission.acquire(identity()) + assert lease.expires_at == 1_030_000 + else: + with pytest.raises(AdmissionError) as error: admission.acquire(identity()) + assert error.value.status == expected + if expected == 429: assert error.value.retry_after == 7 + assert len(calls) == 1 + assert calls[0][1]["allow_redirects"] is False + assert calls[0][1]["timeout"] == (2, 3) + assert calls[0][1]["json"]["principal_hash"] == "a" * 64 + + +def test_renewal_loss_invokes_cancellation_once_and_rejects_remaining_stream(authority): + admission = client(authority) + lost, closed = [], [] + class Source: + def __iter__(self): return self + def __next__(self): return b"chunk" + def close(self): closed.append(True) + response = dispatch_with_admission(identity(), lambda: Response(Source()), admission, on_lost=lost.append) + assert next(iter(response.response)) == b"chunk" + authority.down = True + authority.now += 10000 + admission.renew_due() + admission.renew_due() + with pytest.raises(AdmissionError): next(iter(response.response)) + response.close() + assert len(lost) == 1 + assert closed == [True] + assert [call["operation"] for call in authority.calls].count("release") == 1 + + +def test_loss_during_dispatch_closes_response_and_returns_503(authority): + admission = client(authority) + closed = [] + def dispatch(): + authority.now = 1_100_000 + admission.renew_due() + response = Response("late") + response.call_on_close(lambda: closed.append(True)) + return response + response = dispatch_with_admission(identity(), dispatch, admission) + assert response.status_code == 503 + assert closed == [True] + assert not authority.leases + + +def test_renewal_scheduler_is_single_and_disabled_configuration_never_starts_it(authority, monkeypatch): + starts = [] + class Thread: + def __init__(self, **kwargs): assert kwargs["daemon"] is True + def is_alive(self): return True + def start(self): starts.append(True) + monkeypatch.setattr("services.admission_leases._scheduler", None) + monkeypatch.setattr("services.admission_leases.threading.Thread", Thread) + assert AdmissionClient({}, call=authority).acquire(None) is None + assert starts == [] + env = {"ADMISSION_ENABLED": "true", "ADMISSION_LIMITS_JSON": '{"principal":2}'} + first = AdmissionClient(env, call=authority, clock=lambda: authority.now).acquire(identity()) + second = AdmissionClient(env, call=authority, clock=lambda: authority.now).acquire(identity("small", model_group="small")) + assert starts == [True] + first.release() + second.release() + + + +def test_cancellation_callback_runs_after_unlock_and_release(authority): + admission = client(authority) + observed = [] + pool = ThreadPoolExecutor(1) + def callback(error): + def check(): + with pytest.raises(AdmissionError): lease.check() + return lease.closed + observed.append(pool.submit(check).result(timeout=1)) + lease = admission.acquire(identity(), on_lost=callback) + authority.down = True + authority.now += 10000 + admission.renew_due() + pool.shutdown(wait=True) + assert observed == [True] diff --git a/tests/test_admission_leases_worker.mjs b/tests/test_admission_leases_worker.mjs new file mode 100644 index 00000000..53301093 --- /dev/null +++ b/tests/test_admission_leases_worker.mjs @@ -0,0 +1,239 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { DatabaseSync } from "node:sqlite"; +import { readFileSync } from "node:fs"; +import { AdmissionCoordinator, acquireAdmission, admissionSettings, runWithAdmission } from "../worker/admission-do.mjs"; +import { loadWorkerModule } from "./helpers/load_cloudflare_worker.mjs"; +import { handleIntelligenceOutbound } from "../worker/intelligence-outbound.mjs"; + +const identity = (request_id = "r1", principal_hash = "a".repeat(64), model_group = "heavy") => + ({ principal_hash, model_group, request_id, deadline_ms: 1_100_000 }); +function authority(t, limits = { principal: 2, model_groups: { heavy: 1 } }) { + let now = 1_000_000, touches = 0; + const db = new DatabaseSync(":memory:"); + t.after(() => db.close()); + const storage = { sql: { exec(query, ...args) { + touches++; + const statement = db.prepare(query); + return { toArray: () => statement.columns().length ? statement.all(...args) : (statement.run(...args), []) }; + } }, transactionSync(callback) { + db.exec("BEGIN"); + try { const result = callback(); db.exec("COMMIT"); return result; } + catch (error) { db.exec("ROLLBACK"); throw error; } + } }; + const env = { ADMISSION_ENABLED: "true", ADMISSION_LIMITS_JSON: JSON.stringify(limits) }; + const coordinator = new AdmissionCoordinator({ storage }, env, () => now); + let accesses = 0; + env.ADMISSION_COORDINATOR = { getByName(name) { accesses++; assert.match(name, /^admission-[0-9]+$/); return coordinator; } }; + const call = async body => { + const response = await handleIntelligenceOutbound(new Request("http://intelligence.internal/v1/admission", { + method: "POST", body: JSON.stringify({ version: 1, ...body }), + }), env); + return { status: response.status, headers: response.headers, body: await response.json() }; + }; + return { env, db, storage, coordinator, call, clock(value) { now = value; }, touches: () => touches, accesses: () => accesses }; +} + +test("disabled and malformed settings never touch authority, storage, headers or response identity", async t => { + const a = authority(t); + const response = new Response("unchanged", { headers: { "x-original": "yes" } }); + for (const env of [{}, { ADMISSION_ENABLED: "" }, { ADMISSION_ENABLED: "bad" }, + { ADMISSION_ENABLED: "true", ADMISSION_LIMITS_JSON: "bad" }, + { ADMISSION_ENABLED: "true", ADMISSION_LIMITS_JSON: "{}" }]) { + env.ADMISSION_COORDINATOR = { getByName() { throw Error("unexpected access"); } }; + assert.equal(await runWithAdmission(null, env, () => response), response); + } + a.env.ADMISSION_ENABLED = "false"; + assert.equal((await a.coordinator.fetch(new Request("http://admission.internal/v1/admission", { method: "POST", body: "bad" }))).status, 200); + await a.coordinator.alarm(); + assert.equal(a.touches(), 0); +}); + +test("Flask private and native callers share atomic keyed admission with real retry advice", async t => { + const a = authority(t); + const [first, second] = await Promise.all([ + a.call({ operation: "acquire", ...identity() }), + a.call({ operation: "acquire", ...identity("r2") }), + ]); + assert.deepEqual([first.status, second.status].sort(), [200, 429]); + const denied = first.status === 429 ? first : second; + assert.equal(denied.headers.get("retry-after"), "30"); + await assert.rejects(acquireAdmission(identity("native"), a.env), error => error.status === 429); + const lease = await acquireAdmission(identity("other", "b".repeat(64)), a.env, { clock: () => 1_000_000 }); + await lease.release(); +}); + +test("renewal, expiry, request binding and duplicate release are durable and bounded", async t => { + const a = authority(t); + const first = await a.call({ operation: "acquire", ...identity() }); + const lease_id = first.body.lease.lease_id; + assert.equal((await a.call({ operation: "acquire", ...identity() })).body.lease.lease_id, lease_id); + a.clock(1_020_000); + assert.equal((await a.call({ operation: "renew", ...identity(), lease_id })).body.lease.expires_at, 1_050_000); + a.clock(1_031_000); + assert.equal((await a.call({ operation: "acquire", ...identity("r2") })).status, 429); + for (const change of [{ model_group: "small" }, { request_id: "wrong" }, { principal_hash: "b".repeat(64) }]) { + const bad = await a.coordinator.fetch(new Request("http://admission.internal/v1/admission", { + method: "POST", body: JSON.stringify({ version: 1, operation: "release", ...identity(), lease_id, ...change }), + })); + assert.equal(bad.status, 403); + } + a.clock(1_051_000); + assert.equal((await a.call({ operation: "renew", ...identity(), lease_id })).status, 409); + const replacement = await a.call({ operation: "acquire", ...identity("r2") }); + assert.equal(replacement.status, 200); + const release = { operation: "release", ...identity("r2"), lease_id: replacement.body.lease.lease_id }; + assert.equal((await a.call(release)).body.released, true); + assert.equal((await a.call(release)).body.released, false); +}); + +test("deadline cannot be extended; shard cap and authority outage fail closed", async t => { + const a = authority(t, { principal: 10000 }); + const first = await a.call({ operation: "acquire", ...identity(), deadline_ms: 1_005_000 }); + assert.equal(first.body.lease.expires_at, 1_005_000); + assert.equal((await a.call({ operation: "renew", ...identity(), lease_id: first.body.lease.lease_id })).status, 403); + a.clock(1_005_000); + assert.equal((await a.call({ operation: "renew", ...identity(), deadline_ms: 1_005_000, lease_id: first.body.lease.lease_id })).status, 409); + const insert = a.db.prepare("INSERT INTO admission_leases VALUES (?, ?, ?, ?, ?, ?)"); + for (let n = 0; n < 10000; n++) insert.run(String(n), "c".repeat(64), "heavy", String(n), 1_100_000, 1_035_000); + assert.equal((await a.call({ operation: "acquire", ...identity("cap") })).status, 429); + a.env.ADMISSION_COORDINATOR = { getByName() { throw Error("down"); } }; + await assert.rejects(acquireAdmission(identity(), a.env), error => error.status === 503); +}); + +test("private allowlist rejects public targets, extra fields and raw principal keys", async t => { + const a = authority(t); + for (const [url, method, status] of [["https://public.test/v1/admission", "POST", 400], + ["http://intelligence.internal/v1/admission?x=1", "POST", 400], ["http://intelligence.internal/v1/admission", "GET", 405]]) { + assert.equal((await handleIntelligenceOutbound(new Request(url, { method }), a.env)).status, status); + } + for (const change of [{ principal_hash: "raw-key" }, { model_group: "bad group" }, { limit: 999 }, { version: 2 }, { deadline_ms: NaN }]) { + assert.equal((await a.call({ operation: "acquire", ...identity(), ...change })).status, 400); + } + assert.equal(a.touches(), 0); + assert.equal(admissionSettings({ ADMISSION_ENABLED: "true", ADMISSION_LIMITS_JSON: "" }).enabled, true); +}); + +test("stream completion, response errors and cancellation release once and preserve bytes", async t => { + const a = authority(t); + let releases = 0; + const fetch = a.coordinator.fetch.bind(a.coordinator); + a.coordinator.fetch = async request => { + if ((await request.clone().json()).operation === "release") releases++; + return fetch(request); + }; + const result = await runWithAdmission(identity(), a.env, () => new Response("same bytes", { headers: { "x-provider": "raw" } }), + { clock: () => 1_000_000 }); + assert.equal(await result.text(), "same bytes"); + assert.equal(result.headers.get("x-provider"), "raw"); + assert.equal(releases, 1); + await assert.rejects(runWithAdmission(identity("error"), a.env, () => { throw Error("response failure"); }, + { clock: () => 1_000_000 })); + assert.equal(releases, 2); + const canceled = await runWithAdmission(identity("cancel"), a.env, () => new Response(new ReadableStream({ pull() {} })), + { clock: () => 1_000_000 }); + await canceled.body.cancel(); + assert.equal(releases, 3); + assert.equal(a.db.prepare("SELECT COUNT(*) AS n FROM admission_leases").get().n, 0); +}); + + +test("principal limits span groups and leases survive authority reconstruction", async t => { + const a = authority(t); + const first = await a.call({ operation: "acquire", ...identity() }); + assert.equal(first.status, 200); + assert.equal((await a.call({ operation: "acquire", ...identity("small", "a".repeat(64), "small") })).status, 200); + assert.equal((await a.call({ operation: "acquire", ...identity("third", "a".repeat(64), "tiny") })).status, 429); + const restarted = new AdmissionCoordinator({ storage: a.storage }, a.env, () => 1_000_000); + const denied = await restarted.fetch(new Request("http://admission.internal/v1/admission", { + method: "POST", body: JSON.stringify({ version: 1, operation: "acquire", ...identity("restart") }), + })); + assert.equal(denied.status, 429); + assert.equal((await a.call({ operation: "acquire", ...identity(), model_group: "small" })).status, 403); +}); + +test("public caller headers cannot write the private admission route", async t => { + const a = authority(t); + const { default: worker } = await loadWorkerModule(); + const env = { ...a.env, MULTILLM_PROXY_CONTAINER: { getByName() { return { fetch() { + return Response.json({ error: "not_found" }, { status: 404 }); + } }; } } }; + const response = await worker.fetch(new Request("https://proxy.example/v1/admission", { + method: "POST", headers: { "x-principal-hash": "a".repeat(64), "x-model-group": "heavy" }, + body: JSON.stringify({ version: 1, operation: "acquire", ...identity() }), + }), env); + assert.equal(response.status, 404); + assert.equal(a.accesses(), 0); + assert.equal(a.touches(), 0); +}); + +test("renewal loss cancels pending reads, releases once and reports failure", async t => { + const a = authority(t); + let lease, canceled = 0, lost = 0; + const response = await runWithAdmission(identity(), a.env, acquired => { + lease = acquired; + return new Response(new ReadableStream({ pull() {}, cancel() { canceled++; } })); + }, { clock: () => 1_000_000, onLost() { lost++; throw Error("callback failure"); } }); + const original = a.coordinator.fetch.bind(a.coordinator); + let releases = 0; + a.coordinator.fetch = async request => { + const { operation } = await request.clone().json(); + if (operation === "renew") return Response.json({ version: 1, error: { code: "down" } }, { status: 503 }); + if (operation === "release") releases++; + return original(request); + }; + const read = response.body.getReader().read(); + await lease.renew(); + await assert.rejects(read); + await lease.release(); + assert.equal(canceled, 1); + assert.equal(lost, 1); + assert.equal(releases, 1); +}); + +test("abort before dispatch and while streaming cancels without replay", async t => { + const a = authority(t); + const controller = new AbortController(); + controller.abort(); + let dispatches = 0; + await assert.rejects(runWithAdmission(identity(), a.env, () => { dispatches++; }, + { signal: controller.signal, clock: () => 1_000_000 })); + assert.equal(dispatches, 0); + const active = new AbortController(); + let lease, cancels = 0; + await runWithAdmission(identity("active"), a.env, acquired => { + lease = acquired; + return new Response(new ReadableStream({ pull() {}, cancel() { cancels++; } })); + }, { signal: active.signal, clock: () => 1_000_000 }); + active.abort(); + await new Promise(resolve => setImmediate(resolve)); + assert.equal(lease.closed, true); + assert.equal(cancels, 1); +}); + +test("storage failures are real 503 and unknown groups stay unlimited", async t => { + const a = authority(t, { model_groups: { heavy: 1 } }); + for (const group of ["small", "constructor", "toString"]) { + const value = await acquireAdmission(identity("small", "a".repeat(64), group), a.env); + assert.equal(value, null); + } + assert.equal(a.accesses(), 0); + a.storage.sql.exec = () => { throw Error("storage down"); }; + assert.equal((await a.call({ operation: "acquire", ...identity() })).status, 503); +}); + + +test("Wrangler retains existing bindings and migrations with one additive SQLite class", () => { + const config = JSON.parse(readFileSync(new URL("../wrangler.jsonc", import.meta.url), "utf8")); + assert.deepEqual(config.durable_objects.bindings, [ + { name: "MULTILLM_PROXY_CONTAINER", class_name: "MultiLLMProxyContainer" }, + { name: "ROLEPLAY_SESSION", class_name: "RoleplaySession" }, + { name: "ADMISSION_COORDINATOR", class_name: "AdmissionCoordinator" }, + ]); + assert.deepEqual(config.migrations, [ + { tag: "v1", new_sqlite_classes: ["MultiLLMProxyContainer"] }, + { tag: "v2", new_sqlite_classes: ["RoleplaySession"] }, + { tag: "v3_admission", new_sqlite_classes: ["AdmissionCoordinator"] }, + ]); + assert.equal(config.vars.ADMISSION_ENABLED, undefined); +}); diff --git a/worker/admission-do.mjs b/worker/admission-do.mjs new file mode 100644 index 00000000..46819d03 --- /dev/null +++ b/worker/admission-do.mjs @@ -0,0 +1,306 @@ +import { boundedBody } from "./control-users-d1.mjs"; + +const HASH = /^[0-9a-f]{64}$/; +const ID = /^[A-Za-z0-9_.:-]{1,128}$/; +const GROUP = /^[A-Za-z0-9][A-Za-z0-9._:/+@-]{0,255}$/; +const LEASE = /^[0-9a-f]{32}$/; +const LEASE_MS = 30_000; +const RENEW_MS = 10_000; +const MAX_OUTSTANDING = 10_000; +const MAX_DEADLINE_MS = 86_400_000; +const PRIVATE_URL = "http://admission.internal/v1/admission"; +let warned = false; +const object = value => value !== null && typeof value === "object" && !Array.isArray(value); +const limit = value => Number.isSafeInteger(value) && value >= 0 && value <= MAX_OUTSTANDING; +const fields = (body, names) => object(body) && Object.keys(body).length === names.length && names.every(key => Object.hasOwn(body, key)); +const matches = (pattern, value) => typeof value === "string" && pattern.test(value); + +export function admissionSettings(env = {}) { + const flag = String(env.ADMISSION_ENABLED ?? "").trim().toLowerCase(); + if (["", "false", "0"].includes(flag)) return { enabled: false }; + try { + if (!["true", "1"].includes(flag)) throw Error("flag"); + const limits = JSON.parse(env.ADMISSION_LIMITS_JSON?.trim() || "{}"); + if (!object(limits) || Object.keys(limits).some(key => !["principal", "model_groups"].includes(key)) + || (Object.hasOwn(limits, "principal") && !limit(limits.principal)) + || (Object.hasOwn(limits, "model_groups") && (!object(limits.model_groups) + || Object.keys(limits.model_groups).length > 256 + || Object.entries(limits.model_groups).some(([key, value]) => !GROUP.test(key) || !limit(value))))) throw Error("limits"); + return { enabled: true, principal: limits.principal ?? 0, groups: limits.model_groups ?? {} }; + } catch { + if (!warned) { warned = true; console.warn("Invalid admission configuration; admission is disabled."); } + return { enabled: false }; + } +} +const groupLimit = (settings, group) => Object.hasOwn(settings.groups, group) ? settings.groups[group] : 0; +const unlimited = (settings, group) => !settings.principal && !groupLimit(settings, group); +const validIdentity = value => object(value) && matches(HASH, value.principal_hash) && matches(GROUP, value.model_group) + && matches(ID, value.request_id) && Number.isSafeInteger(value.deadline_ms) && value.deadline_ms > 0; +function validOperation(body) { + const names = ["version", "operation", "principal_hash", "model_group", "request_id", "deadline_ms"]; + if (["renew", "release"].includes(body?.operation)) names.push("lease_id"); + return fields(body, names) && body.version === 1 && ["acquire", "renew", "release"].includes(body.operation) + && validIdentity(body) && (body.operation === "acquire" || matches(LEASE, body.lease_id)); +} +const reply = (body, status = 200, retry = null) => Response.json({ version: 1, ...body }, { + status, headers: { "cache-control": "no-store", ...(retry === null ? {} : { "Retry-After": String(retry) }) }, +}); +const failure = (code, status, retry = null) => reply({ error: { code, message: "Concurrency admission failed." } }, status, retry); +const leaseReply = row => reply({ lease: { lease_id: row.lease_id, expires_at: row.expires_at } }); + +/** Private SQLite authority. Construction and disabled calls perform no storage work. */ +export class AdmissionCoordinator { + constructor(ctx, env, clock = Date.now) { + this.storage = ctx.storage; + this.env = env; + this.clock = clock; + this.ready = false; + } + + rows(sql, ...args) { return this.storage.sql.exec(sql, ...args).toArray(); } + + initialize() { + if (this.ready) return; + this.storage.transactionSync(() => { + this.rows(`CREATE TABLE IF NOT EXISTS admission_leases ( + lease_id TEXT PRIMARY KEY, principal_hash TEXT NOT NULL, model_group TEXT NOT NULL, + request_id TEXT NOT NULL, deadline_ms INTEGER NOT NULL, expires_at INTEGER NOT NULL, + UNIQUE(principal_hash, request_id))`); + this.rows("CREATE INDEX IF NOT EXISTS admission_expiry ON admission_leases(expires_at)"); + this.rows("CREATE INDEX IF NOT EXISTS admission_scope ON admission_leases(principal_hash, model_group, expires_at)"); + }); + this.ready = true; + } + + async fetch(request) { + if (request.url !== PRIVATE_URL || request.method !== "POST") return failure("not_found", 404); + const settings = admissionSettings(this.env); + if (!settings.enabled) return reply({ lease: null }); + let body; + try { body = JSON.parse(await boundedBody(request, 4096)); } catch { return failure("invalid_request", 400); } + if (!validOperation(body)) return failure("invalid_request", 400); + if (body.operation === "acquire" && unlimited(settings, body.model_group)) return reply({ lease: null }); + try { + this.initialize(); + return this.storage.transactionSync(() => this.operate(body, settings)); + } catch { return failure("admission_unavailable", 503); } + } + + operate(body, settings) { + const now = this.clock(); + this.rows("DELETE FROM admission_leases WHERE expires_at <= ?", now); + if (body.operation === "acquire") return this.acquire(body, settings, now); + const [row] = this.rows("SELECT * FROM admission_leases WHERE lease_id = ?", body.lease_id); + if (!row) return body.operation === "release" ? reply({ released: false }) : failure("admission_lease_expired", 409); + if (["principal_hash", "model_group", "request_id", "deadline_ms"].some(key => row[key] !== body[key])) { + return failure("admission_identity_mismatch", 403); + } + if (body.operation === "release") { + this.rows("DELETE FROM admission_leases WHERE lease_id = ?", body.lease_id); + return reply({ released: true }); + } + row.expires_at = Math.min(now + LEASE_MS, row.deadline_ms); + this.rows("UPDATE admission_leases SET expires_at = ? WHERE lease_id = ?", row.expires_at, row.lease_id); + return leaseReply(row); + } + + acquire(body, settings, now) { + if (body.deadline_ms <= now || body.deadline_ms > now + MAX_DEADLINE_MS) return failure("admission_deadline_invalid", 400); + const [existing] = this.rows("SELECT * FROM admission_leases WHERE principal_hash = ? AND request_id = ?", + body.principal_hash, body.request_id); + if (existing) { + if (existing.model_group !== body.model_group || existing.deadline_ms !== body.deadline_ms) { + return failure("admission_identity_mismatch", 403); + } + return leaseReply(existing); + } + const scopes = [ + ["", [], MAX_OUTSTANDING], + ["WHERE principal_hash = ?", [body.principal_hash], settings.principal], + ["WHERE principal_hash = ? AND model_group = ?", [body.principal_hash, body.model_group], groupLimit(settings, body.model_group)], + ]; + for (const [where, args, maximum] of scopes) { + if (!maximum) continue; + const [count] = this.rows(`SELECT COUNT(*) AS n, MIN(expires_at) AS next_expiry FROM admission_leases ${where}`, ...args); + if (count.n >= maximum) return failure("admission_denied", 429, Math.max(1, Math.ceil((count.next_expiry - now) / 1000))); + } + const row = { ...body, lease_id: crypto.randomUUID().replaceAll("-", ""), expires_at: Math.min(now + LEASE_MS, body.deadline_ms) }; + this.rows("INSERT INTO admission_leases VALUES (?, ?, ?, ?, ?, ?)", row.lease_id, row.principal_hash, + row.model_group, row.request_id, row.deadline_ms, row.expires_at); + return leaseReply(row); + } + + async alarm() { + // Expiry is pruned on operations; no alarms are scheduled, including while disabled. + if (!admissionSettings(this.env).enabled || !this.ready) return; + this.rows("DELETE FROM admission_leases WHERE expires_at <= ?", this.clock()); + } +} + +async function authorityCall(body, env) { + let timer; + try { + const shard = parseInt(body.principal_hash.slice(0, 2), 16) % 64; + const stub = env.ADMISSION_COORDINATOR.getByName(`admission-${shard}`); + return await Promise.race([ + stub.fetch(new Request(PRIVATE_URL, { method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify(body), signal: AbortSignal.timeout(5000) })), + new Promise((_, reject) => { timer = setTimeout(() => reject(new AdmissionError()), 5000); }), + ]); + } catch { return failure("admission_unavailable", 503); } + finally { clearTimeout(timer); } +} + +/** Only the authenticated Container egress dispatcher calls this route. */ +export async function handleAdmissionRequest(request, env) { + if (request.url !== "http://intelligence.internal/v1/admission" || request.method !== "POST") return failure("not_found", 404); + const settings = admissionSettings(env); + if (!settings.enabled) return reply({ lease: null }); + let body; + try { body = JSON.parse(await boundedBody(request, 4096)); } catch { return failure("invalid_request", 400); } + if (!validOperation(body)) return failure("invalid_request", 400); + if (body.operation === "acquire" && unlimited(settings, body.model_group)) return reply({ lease: null }); + return authorityCall(body, env); +} + +export class AdmissionError extends Error { + constructor(code = "admission_unavailable", status = 503, retryAfter = null) { + super("Concurrency admission failed."); + this.code = code; + this.status = status; + this.retryAfter = retryAfter; + } + response() { return failure(this.code, this.status, this.retryAfter); } +} + +async function call(body, env) { + const response = await authorityCall(body, env); + try { + const payload = JSON.parse(await boundedBody(response, 4096)); + if (payload.version !== 1) throw Error("version"); + if (response.status !== 200) { + const retry = Number(response.headers.get("Retry-After")); + throw new AdmissionError(response.status === 429 ? "admission_denied" : "admission_unavailable", + response.status === 429 ? 429 : 503, response.status === 429 && Number.isSafeInteger(retry) && retry > 0 ? retry : null); + } + return payload; + } catch (error) { + if (error instanceof AdmissionError) throw error; + throw new AdmissionError(); + } +} + +class AdmissionLease { + constructor(identity, env, value, options) { + this.identity = identity; + this.env = env; + this.clock = options.clock ?? Date.now; + this.onLost = options.onLost ?? (() => {}); + this.closed = false; + this.lost = null; + this.renewing = false; + this.value = value; + this.interval = setInterval(() => { void this.renew(); }, RENEW_MS); + this.timer = setTimeout(() => { void this.lose(new AdmissionError("admission_deadline_exceeded")); }, + Math.max(0, Math.min(value.expires_at, identity.deadline_ms) - this.clock())); + this.interval.unref?.(); + this.timer.unref?.(); + } + async renew() { + if (this.closed || this.renewing) return; + this.renewing = true; + try { + const payload = await call({ version: 1, operation: "renew", ...this.identity, lease_id: this.value.lease_id }, this.env); + if (this.closed) return; + this.validate(payload.lease); + this.value = payload.lease; + clearTimeout(this.timer); + this.timer = setTimeout(() => { void this.lose(new AdmissionError("admission_deadline_exceeded")); }, + Math.max(0, Math.min(this.value.expires_at, this.identity.deadline_ms) - this.clock())); + this.timer.unref?.(); + } catch (error) { await this.lose(error); } + finally { this.renewing = false; } + } + validate(value) { + if (!object(value) || !matches(LEASE, value.lease_id) || value.lease_id !== this.value.lease_id + || !Number.isSafeInteger(value.expires_at) || value.expires_at <= this.clock() + || value.expires_at > Math.min(this.clock() + LEASE_MS, this.identity.deadline_ms)) throw new AdmissionError(); + } + check() { + if (this.lost) throw this.lost; + if (this.closed || this.clock() >= Math.min(this.value.expires_at, this.identity.deadline_ms)) throw new AdmissionError(); + } + async lose(error) { + if (this.closed) return; + this.lost = error instanceof AdmissionError ? error : new AdmissionError(); + await Promise.allSettled([this.release(), Promise.resolve().then(() => this.onLost(this.lost))]); + } + async release() { + if (this.closed) return this.releasePromise; + this.closed = true; + clearInterval(this.interval); + clearTimeout(this.timer); + this.releasePromise = (async () => { + try { await call({ version: 1, operation: "release", ...this.identity, lease_id: this.value.lease_id }, this.env); } + catch { /* An uncertain release is never replayed; the persisted lease expires. */ } + })(); + return this.releasePromise; + } +} + +/** Identity comes from authentication and authorized routing, never request headers. */ +export async function acquireAdmission(identity, env, options = {}) { + const settings = admissionSettings(env); + if (!settings.enabled || (!settings.principal && Object.values(settings.groups).every(value => !value))) return null; + if (!validIdentity(identity) || !fields(identity, ["principal_hash", "model_group", "request_id", "deadline_ms"])) { + throw new AdmissionError("admission_identity_invalid", 403); + } + if (unlimited(settings, identity.model_group)) return null; + const payload = await call({ version: 1, operation: "acquire", ...identity }, env); + const value = payload.lease; + if (!object(value) || !matches(LEASE, value.lease_id) || !Number.isSafeInteger(value.expires_at) + || value.expires_at <= (options.clock ?? Date.now)() + || value.expires_at > Math.min((options.clock ?? Date.now)() + LEASE_MS, identity.deadline_ms)) throw new AdmissionError(); + return new AdmissionLease(identity, env, value, options); +} + +export async function runWithAdmission(identity, env, dispatch, options = {}) { + let reader; + let response; + const lease = await acquireAdmission(identity, env, { ...options, onLost: async error => { + try { await reader?.cancel(error); } finally { await options.onLost?.(error); } + } }); + if (!lease) return dispatch(); + let aborted = false; + const abort = () => { aborted = true; void lease.lose(new AdmissionError("admission_canceled")); void reader?.cancel().catch(() => {}); }; + options.signal?.addEventListener("abort", abort, { once: true }); + const finish = async () => { options.signal?.removeEventListener("abort", abort); await lease.release(); }; + try { + if (options.signal?.aborted) abort(); + lease.check(); + response = await dispatch(lease); + if (aborted) throw new AdmissionError("admission_canceled"); + lease.check(); + if (!response.body) { await finish(); return response; } + reader = response.body.getReader(); + const body = new ReadableStream({ + async pull(controller) { + try { + lease.check(); + const { done, value } = await reader.read(); + lease.check(); + if (done) { await finish(); controller.close(); } + else controller.enqueue(value); + } catch (error) { + controller.error(error); + try { await reader.cancel(); } finally { await finish(); } + } + }, + async cancel(reason) { try { await reader.cancel(reason); } finally { await finish(); } }, + }, { highWaterMark: 0 }); + return new Response(body, { status: response.status, statusText: response.statusText, headers: response.headers }); + } catch (error) { + try { if (reader) await reader.cancel(); else await response?.body?.cancel(); } finally { await finish(); } + throw error; + } +} diff --git a/worker/container-env.mjs b/worker/container-env.mjs index f678771b..88884283 100644 --- a/worker/container-env.mjs +++ b/worker/container-env.mjs @@ -27,6 +27,9 @@ const DIRECT_ENV_KEYS = [ "RATE_LIMIT_RPM", "RATE_LIMIT_TPM", "DAILY_REQUEST_LIMIT", + "ADMISSION_ENABLED", + "ADMISSION_LIMITS_JSON", + "ADMISSION_AUTHORITY_URL", "FREE_ROUTE_EXTRA_PROVIDERS", "FREE_ROUTE_FREE_TIER_PROVIDERS", "FREE_ROUTE_PROVIDER_ORDER", diff --git a/worker/intelligence-outbound.mjs b/worker/intelligence-outbound.mjs index 63aa86cb..63c95bb3 100644 --- a/worker/intelligence-outbound.mjs +++ b/worker/intelligence-outbound.mjs @@ -1,3 +1,4 @@ +import { handleAdmissionRequest } from "./admission-do.mjs"; import { handleIntelligenceStoreRequest } from "./intelligence-d1.mjs"; import { handleIntelligenceAuthRequest } from "./intelligence-auth-d1.mjs"; import { handleControlUsersRequest } from "./control-users-d1.mjs"; @@ -19,6 +20,7 @@ export function handleIntelligenceOutbound(request, env) { return Response.json({ error: { code: "method_not_allowed", message: "Use POST for storage operations." } }, { status: 405, headers: { Allow: "POST" } }); } + if (url.pathname === "/v1/admission") return handleAdmissionRequest(request, env); if (url.pathname === "/v1/store") return handleIntelligenceStoreRequest(request, env); if (url.pathname === "/v1/auth") return handleIntelligenceAuthRequest(request, env); if (url.pathname === "/v1/users") return handleControlUsersRequest(request, env); diff --git a/wrangler.jsonc b/wrangler.jsonc index 58d0cf5e..4049c6e8 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -54,6 +54,10 @@ { "name": "ROLEPLAY_SESSION", "class_name": "RoleplaySession" + }, + { + "name": "ADMISSION_COORDINATOR", + "class_name": "AdmissionCoordinator" } ] }, @@ -69,6 +73,12 @@ "new_sqlite_classes": [ "RoleplaySession" ] + }, + { + "tag": "v3_admission", + "new_sqlite_classes": [ + "AdmissionCoordinator" + ] } ], "observability": {