From 631c5cbc57c2d29e5c6b43bded6f275254bf84cd Mon Sep 17 00:00:00 2001 From: Abhyudaya Baiju Tharakan Date: Fri, 9 Oct 2026 05:51:29 +0530 Subject: [PATCH] feat(gateway): add opt-in end-to-end generation deadlines Carry one monotonic budget from X-MultiLLM-Deadline-Ms through setup, admission, retry waits, provider calls, preflight and stream reads. Expiry returns 504 before commitment or a protocol error event after it, cancels the upstream once and settles usage as unknown. Requests without the header are unchanged. --- docs/generation-deadline.md | 63 +++ package.json | 2 +- route_helpers.py | 2 +- routes/auto_routes.py | 10 +- services/generation_deadline.py | 430 +++++++++++++++++++ services/proxy_service.py | 30 +- tests/fixtures/secret_firewall_coverage.json | 1 + tests/test_generation_deadline.py | 277 ++++++++++++ tests/test_generation_deadline_worker.mjs | 169 ++++++++ worker/container-env.mjs | 1 + worker/cors-policy.mjs | 2 +- worker/generation-deadline.mjs | 249 +++++++++++ 12 files changed, 1228 insertions(+), 8 deletions(-) create mode 100644 docs/generation-deadline.md create mode 100644 services/generation_deadline.py create mode 100644 tests/test_generation_deadline.py create mode 100644 tests/test_generation_deadline_worker.mjs create mode 100644 worker/generation-deadline.mjs diff --git a/docs/generation-deadline.md b/docs/generation-deadline.md new file mode 100644 index 00000000..c1b8412f --- /dev/null +++ b/docs/generation-deadline.md @@ -0,0 +1,63 @@ +# Generation deadlines + +Send `X-MultiLLM-Deadline-Ms` to set a relative budget for one generation. The +header accepts an integer from 1 to `GENERATION_DEADLINE_MAX_MS` (300000 by +default). Setup, token counting, admission, allowed retry waits, provider calls, +stream preflight and body reads all consume that same budget. A shorter key or +route limit takes precedence. + +Without the header, existing timeouts, retries, response bytes and stream behavior +remain unchanged. An empty `GENERATION_DEADLINE_MAX_MS` uses the default. A +malformed maximum disables deadlines and logs one warning without its value. +A malformed request header returns HTTP 400 `invalid_generation_deadline`. + +For example, this request has a ten-second generation budget: + +```sh +curl "$GATEWAY_URL/v1/chat/completions" \ + -H "Authorization: Bearer $GATEWAY_TOKEN" \ + -H 'Content-Type: application/json' \ + -H 'X-MultiLLM-Deadline-Ms: 10000' \ + -d '{"model":"openai:gpt-4.1","messages":[{"role":"user","content":"Hello"}],"stream":true}' +``` + +Omit the deadline header to use the existing timeouts: + +```sh +curl "$GATEWAY_URL/v1/chat/completions" \ + -H "Authorization: Bearer $GATEWAY_TOKEN" \ + -H 'Content-Type: application/json' \ + -d '{"model":"openai:gpt-4.1","messages":[{"role":"user","content":"Hello"}]}' +``` + +Before response commitment, expiry returns HTTP 504 with an error whose code is +`generation_deadline_exceeded`. After an SSE stream is committed, expiry emits +an error event in the selected Chat, Responses or Messages protocol and cancels +the upstream. It does not append a stop event or `[DONE]`, or start a fallback. +Binary streams terminate with a transport error because they have no SSE error +envelope. Chat SSE preflight uses the existing 65536-byte/32-event validation +bounds. Other SSE prefixes inspect at most 65536 bytes before commitment; +transport reads can deliver a larger chunk containing that prefix. +Exhausting the prefix bound or reaching EOF before useful output returns a real +HTTP 502 `upstream_stream_invalid` error and does not authorize replay. + +The budget uses each host's monotonic clock. Forwarding sends only the rounded-down +remaining milliseconds in `X-MultiLLM-Internal-Deadline-Ms`. The forwarding +boundary strips any caller-supplied internal budget. The receiving Flask boundary +accepts that header only after its fixed transport verifier authenticates the +internal hop; public API authentication alone is not proof of an internal hop. +No monotonic timestamp is transmitted or compared across machines. + +The deadline limits waiting and requests local cancellation. It cannot guarantee +that a provider stops computation or billing. Synchronous CPU setup cannot be +preempted; it is checked before further dispatch. Flask transport connects and +reads use bounded timeouts, and the cancellation owner closes a handed-off +response at expiry. A timeout before upstream handoff relies on the transport's +connect/read timeout rather than a separate socket owner. + +Cancellation never grants replay permission. A handoff without confirmed final +usage remains ambiguous, even after clean EOF. Finalizers use +`settlement_information` (Flask) or `settlementInformation` (Worker) to retain +unknown usage and reservation information for settlement. Deadlines do not +create a durable reservation store or estimate a successful charge. They store +no prompt or response content and do not change the request's retention policy. diff --git a/package.json b/package.json index fa5f6aff..dee8ed58 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "lint": "ruff check . --select E9,F63,F7,F82", "typecheck": "mypy --ignore-missing-imports --follow-imports=skip --disable-error-code import-untyped --disable-error-code var-annotated --disable-error-code arg-type --disable-error-code assignment --disable-error-code operator app.py routes services providers streaming", "test": "python -m pytest -q", - "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs", + "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_generation_deadline_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs", "db:migrate": "python scripts/validate_sqlite_schema.py --apply", "db:validate": "python scripts/validate_sqlite_schema.py", "security:secrets": "python scripts/check_static_secrets.py", diff --git a/route_helpers.py b/route_helpers.py index cd72a0b8..6d654b38 100644 --- a/route_helpers.py +++ b/route_helpers.py @@ -26,7 +26,7 @@ CORS_ALLOWED_METHODS = "GET, POST, PUT, DELETE, PATCH, OPTIONS" CORS_DEFAULT_HEADERS = ( - "Authorization, Content-Type, Accept, Origin, X-Requested-With, X-MultiLLM-Tool-Repair, X-MultiLLM-Conversion-Report, X-MultiLLM-Retention, " + "Authorization, Content-Type, Accept, Origin, X-Requested-With, X-MultiLLM-Tool-Repair, X-MultiLLM-Conversion-Report, X-MultiLLM-Retention, X-MultiLLM-Deadline-Ms, " "X-Api-Key, X-Goog-Api-Key, X-MultiLLM-Api-Key, Anthropic-Version, " "Anthropic-Beta, Anthropic-Dangerous-Direct-Browser-Access, " "Idempotency-Key, X-MultiLLM-Image-QA, X-MultiLLM-Cascade, OpenAI-Beta, OpenAI-Organization, " diff --git a/routes/auto_routes.py b/routes/auto_routes.py index c96fc42a..34a29457 100644 --- a/routes/auto_routes.py +++ b/routes/auto_routes.py @@ -26,6 +26,9 @@ from services.resilience_service import ResilienceService from services.route_health import RouteHealth, ordering_settings from services.stream_preflight import preflight_chat_stream +from services.generation_deadline import ( + GenerationDeadlineExceeded, check_deadline, check_response_deadline, +) # Payment-required responses mean this candidate cannot serve the request with # its current credentials. Explicit auto routes may safely try the next @@ -161,6 +164,7 @@ def dispatch_auto_route( Each attempt's outcome and time to response feed route health, and a route set to health ordering tries its candidates in the order RouteHealth.order returns. """ + check_deadline() route = AutoRouteService.get_route(payload.get("model")) if route is None: raise APIError( @@ -177,10 +181,12 @@ def dispatch_auto_route( last_priority = 0 last_decision = "auto-failover" for position, candidate in enumerate(order.candidates): + check_deadline() priority = priorities[candidate] try: validate_candidate(candidate) - except (ModelCooldownExhausted, ModelCooldownCapacity): + check_deadline() + except (ModelCooldownExhausted, ModelCooldownCapacity, GenerationDeadlineExceeded): raise except (APIError, ValueError) as error: logger.info( @@ -202,6 +208,7 @@ def dispatch_auto_route( candidate, route_decision, ) + check_response_deadline(response) except AutoRouteCandidateUnavailable as error: logger.info( "Skipping auto route candidate %s before generation (%s)", @@ -226,6 +233,7 @@ def dispatch_auto_route( ): preflight = preflight_chat_stream(response) response = preflight.response + check_response_deadline(response) preflight_failed = preflight.outcome not in {"skipped", "validated"} preflight_outcome = preflight.outcome ok, reason = attempt_outcome(response) diff --git a/services/generation_deadline.py b/services/generation_deadline.py new file mode 100644 index 00000000..e468e38c --- /dev/null +++ b/services/generation_deadline.py @@ -0,0 +1,430 @@ +"""A request-local generation budget; expiry never grants replay permission.""" +from __future__ import annotations + +import json +import logging +import math +import os +import re +import threading +import time +from collections.abc import Callable, Mapping +from dataclasses import dataclass, field + +from flask import Response, current_app, g, has_request_context, request + +from error_handlers import APIError +from services.request_cancellation import RequestCancellation, bind_cancellation + +PUBLIC_HEADER = "X-MultiLLM-Deadline-Ms" +INTERNAL_HEADER = "X-MultiLLM-Internal-Deadline-Ms" +DEFAULT_MAX_MS = 300000 +logger = logging.getLogger(__name__) +_invalid_max_logged = False + + +class InvalidGenerationDeadline(APIError): + def __init__(self): + super().__init__("Invalid generation deadline", status_code=400) + + +class GenerationDeadlineExceeded(APIError): + code = "generation_deadline_exceeded" + + def __init__(self): + super().__init__("Generation deadline exceeded", status_code=504) + + +def error_payload(): + return {"error": {"message": "Generation deadline exceeded", + "type": "timeout_error", "code": GenerationDeadlineExceeded.code}} + + +def error_response(): + return Response(json.dumps(error_payload()), status=504, content_type="application/json") + + +def settlement_information(owner, confirmed_usage=None): + """Finalizers retain a handoff with no confirmed usage, including clean EOF.""" + return {"ambiguous": bool(owner.handed_off and confirmed_usage is None), + "usage_state": "unknown" if confirmed_usage is None else "known", + "usage": confirmed_usage, "replay_permission": False, + "cancellation_outcome": owner.outcome} + + +def error_event(protocol="chat"): + payload = error_payload() + if protocol == "responses": + payload = {**payload["error"], "type": "error"} + elif protocol == "anthropic": + payload = {"type": "error", "error": {**payload["error"], "type": "api_error"}} + return ("event: error\ndata: " + json.dumps(payload) + "\n\n").encode() + + +def _maximum(): + global _invalid_max_logged + raw = os.environ.get("GENERATION_DEADLINE_MAX_MS", "").strip() + if not raw: + return DEFAULT_MAX_MS + if re.fullmatch(r"[0-9]{1,6}", raw) and 1 <= int(raw) <= DEFAULT_MAX_MS: + return int(raw) + if not _invalid_max_logged: + logger.warning("Invalid GENERATION_DEADLINE_MAX_MS; generation deadlines disabled") + _invalid_max_logged = True + return None + + +def _milliseconds(value, maximum): + if not isinstance(value, str) or not re.fullmatch(r"[0-9]{1,6}", value) or not 1 <= int(value) <= maximum: + raise InvalidGenerationDeadline() + return int(value) + + +@dataclass +class Deadline: + expires_at: float + clock: Callable[[], float] = time.monotonic + _timer: threading.Timer | None = field(default=None, init=False, repr=False) + _cancellations: list[Callable[[], None]] = field(default_factory=list, init=False, repr=False) + _expired: bool = field(default=False, init=False, repr=False) + + def expire(self): + self._expired = True + for cancel in tuple(self._cancellations): + cancel() + + def remaining(self): + return max(0.0, self.expires_at - self.clock()) + + def remaining_ms(self): + # Round down: transport must never increase another machine's budget. + return max(0, math.floor(self.remaining() * 1000)) + + def check(self): + if self._expired or self.remaining() <= 0: + self.expire() + raise GenerationDeadlineExceeded() + + def arm(self, cancel): + if cancel not in self._cancellations: + self._cancellations.append(cancel) + if self._timer is None: + self._timer = threading.Timer(self.remaining(), self.expire) + self._timer.daemon = True + self._timer.start() + self.check() + + def stop(self): + if self._timer is not None: + self._timer.cancel() + + def sleep(self, seconds, sleep=time.sleep): + self.check() + if seconds >= self.remaining(): + self.expire() + raise GenerationDeadlineExceeded() + sleep(seconds) + self.check() + + +def deadline_from_headers(headers: Mapping, *, trusted_internal=False, limits_ms=(), clock=time.monotonic): + maximum = _maximum() + if maximum is None: + return None + normalized = {key.lower(): value for key, value in headers.items()} + public = normalized.get(PUBLIC_HEADER.lower()) + internal = normalized.get(INTERNAL_HEADER.lower()) if trusted_internal else None + if public is None and internal is None: + return None + budgets = [_milliseconds(value, maximum) for value in (public, internal) if value is not None] + budgets.extend(value for value in limits_ms if type(value) is int and value > 0) + return Deadline(clock() + min(budgets) / 1000, clock) + + +def forwarded_headers(headers, deadline): + result = {key: value for key, value in headers.items() if key.lower() != INTERNAL_HEADER.lower()} + if deadline is not None: + deadline.check() + remaining = deadline.remaining_ms() + if remaining < 1: + raise GenerationDeadlineExceeded() + result[INTERNAL_HEADER] = str(remaining) + # Public relative time must not restart when the receiving host converts it. + result = {key: value for key, value in result.items() if key.lower() != PUBLIC_HEADER.lower()} + return result + + +def current_deadline(): + return getattr(g, "generation_deadline", None) if has_request_context() else None + + +def check_deadline(): + deadline = current_deadline() + if deadline is not None: + deadline.check() + return deadline + + +def check_response_deadline(response): + try: + check_deadline() + except GenerationDeadlineExceeded: + response.close() + raise + + +def bounded_timeout(timeout): + deadline = check_deadline() + if deadline is None: + return timeout + remaining = deadline.remaining() + if isinstance(timeout, (tuple, list)): + return tuple(min(value, remaining / len(timeout)) for value in timeout) + return min(timeout, remaining) + + +def deadline_sleep(seconds, sleep=time.sleep): + deadline = current_deadline() + if deadline is None: + return sleep(seconds) + return deadline.sleep(seconds, sleep) + + +def retry_with_deadline(attempt, outcome, **kwargs): + from services.managed_dispatch import retry_managed_attempt + from services.retry_advice import parse_retry_advice, retry_advice_settings + deadline = current_deadline() + if deadline is None: + return retry_managed_attempt(attempt, outcome, **kwargs) + deadline.check() + if not outcome().replay_permission or kwargs["retry_count"] >= kwargs["max_retries"]: + return None + delay = kwargs["retry_delay"] * (kwargs["retry_count"] + 1) + settings = retry_advice_settings() + response = kwargs.get("response") + if settings.enabled and response is not None: + advice = parse_retry_advice(response.headers, now=time.time(), + status_code=response.status_code, max_seconds=settings.max_seconds) + if advice is not None: + delay = max(delay, advice.delay_seconds) + if delay >= deadline.remaining(): + if response is not None: + bind_cancellation(response).cancel() + raise GenerationDeadlineExceeded() + sleep = kwargs.get("sleep", time.sleep) + kwargs["sleep"] = lambda seconds: deadline.sleep(seconds, sleep) + result = retry_managed_attempt(attempt, outcome, **kwargs) + deadline.check() + return result + + +def own_upstream(response): + deadline = current_deadline() + if deadline is None: + return response + context = bind_cancellation(response) + owner = getattr(g, "gateway_cancellation", None) if has_request_context() else None + deadline.arm(owner.cancel if owner is not None else context.cancel) + deadline.arm(context.cancel) + for name in ("iter_content", "iter_lines"): + original = getattr(response, name, None) + if callable(original) and not getattr(response, "multillm_deadline_wrapped", False): + def guarded(*args, _original=original, **kwargs): + iterator = iter(_original(*args, **kwargs)) + while True: + deadline.check() + try: + value = next(iterator) + except StopIteration: + deadline.check() + return + deadline.check() + yield value + setattr(response, name, guarded) + response.multillm_deadline_wrapped = True + deadline.check() + return response + + +class DeadlineIterator: + """Check each read and suppress late completion bytes after cancellation.""" + def __init__(self, source, deadline, upstream, *, protocol="chat", sse=True): + self.source = iter(source) + self.deadline = deadline + self.context = bind_cancellation(upstream) + self.protocol = protocol + self.sse = sse + self.committed = False + self.closed = False + + def __iter__(self): + return self + + def __next__(self): + if self.closed: + raise StopIteration + try: + self.deadline.check() + value = next(self.source) + self.deadline.check() + self.committed = True + return value + except GenerationDeadlineExceeded: + self.close() + if self.committed and self.sse: + return error_event(self.protocol) + raise + except StopIteration: + try: + self.deadline.check() + except GenerationDeadlineExceeded: + self.close() + if self.committed and self.sse: + return error_event(self.protocol) + raise + self.deadline.stop() + self.closed = True + raise + except BaseException: + self.close() + raise + + def close(self): + if self.closed: + return + self.closed = True + self.deadline.stop() + self.context.cancel() + close = getattr(self.source, "close", None) + if close is not None: + close() + + +def generation_deadline_hook(): + """Mount after authentication/identity and before setup or admission work.""" + from services.request_accounting import classify + if classify() is None or getattr(g, "generation_deadline_initialized", False): + return None + g.generation_deadline_initialized = True + options = current_app.extensions.get("generation_deadline", {}) + try: + deadline = deadline_from_headers(request.headers, + trusted_internal=bool(options.get("verify_internal", lambda: False)()), + limits_ms=options.get("limits_ms", lambda: ())(), clock=options.get("clock", time.monotonic)) + except InvalidGenerationDeadline: + return Response(json.dumps({"error": {"code": "invalid_generation_deadline", + "message": "Invalid generation deadline"}}), status=400, content_type="application/json") + if deadline is not None: + g.generation_deadline = deadline + # Existing admission/cascade scheduling consumes the same local deadline. + g.cascade_deadline = min(getattr(g, "cascade_deadline", deadline.expires_at), deadline.expires_at) + g.gateway_generation_deadline = min(getattr(g, "gateway_generation_deadline", deadline.expires_at), deadline.expires_at) + owner = getattr(g, "gateway_cancellation", None) + if owner is None: + owner = g.gateway_cancellation = RequestCancellation() + deadline.arm(owner.cancel) + return None + + +def _useful_prefix(prefix): + text = prefix.decode("utf-8", errors="replace").replace("\r\n", "\n") + for event in text.split("\n\n")[:-1]: + data = "\n".join(line[5:].lstrip() for line in event.splitlines() if line.startswith("data:")) + if not data: + continue + try: + payload = json.loads(data) + except ValueError: + continue + if not isinstance(payload, dict): + continue + kind = payload.get("type", "") + if payload.get("error") or kind == "error" or isinstance(kind, str) and kind.endswith(("delta", "completed", "failed", "stop")): + return True + candidates = payload.get("candidates", []) + if isinstance(candidates, list) and any( + isinstance(candidate, dict) and isinstance(candidate.get("content"), dict) + and candidate["content"].get("parts") for candidate in candidates + ): + return True + return False + + +def _prime_response(response, deadline): + """Inspect a bounded protocol prefix before headers can be committed.""" + owner = bind_cancellation(response) + deadline.arm(owner.cancel) + source = iter(response.iter_encoded()) + prefix = [] + inspected = bytearray() + sse = response.mimetype == "text/event-stream" and response.status_code < 400 + useful = not sse + while len(inspected) < 65536: + deadline.check() + try: + chunk = next(source) + except StopIteration: + deadline.check() + break + deadline.check() + prefix.append(chunk) + inspected.extend(chunk[:65536 - len(inspected)]) + useful = not sse or _useful_prefix(inspected) + if useful: + break + if not useful: + response.close() + return Response(json.dumps({"error": {"type": "upstream_error", "code": "upstream_stream_invalid", + "message": "Upstream stream ended without a useful bounded prefix"}}), status=502, content_type="application/json") + def replay(): + try: + yield from prefix + prefix.clear() + yield from source + finally: + source.close() + response.close() + downstream = Response(replay(), status=response.status_code, headers=response.headers) + downstream.call_on_close(response.close) + return downstream + + +def deadline_response(response): + deadline = current_deadline() + if deadline is None: + return response + try: + deadline.check() + if not response.is_streamed: + deadline.stop() + return response + if response.mimetype == "text/event-stream" and request.path.endswith("/chat/completions"): + from services.stream_preflight import preflight_chat_stream + response = preflight_chat_stream(response).response + deadline.check() + if not response.is_streamed: + deadline.stop() + return response + else: + response = _prime_response(response, deadline) + if not response.is_streamed: + deadline.stop() + return response + protocol = "responses" if request.path.endswith("/responses") else "anthropic" if request.path.endswith("/messages") else "chat" + response.response = DeadlineIterator(response.response, deadline, response, + protocol=protocol, sse=response.mimetype == "text/event-stream") + deadline.arm(response.response.context.cancel) + response.call_on_close(response.response.close) + return response + except GenerationDeadlineExceeded: + response.close() + deadline.stop() + return error_response() + + +def register_generation_deadline(app, *, verify_internal=lambda: False, limits_ms=lambda: (), clock=time.monotonic): + """Register fixed callbacks; verification must authenticate the internal transport.""" + app.extensions["generation_deadline"] = {"verify_internal": verify_internal, "limits_ms": limits_ms, "clock": clock} + app.extensions.setdefault("gateway_after_authentication", []).append(generation_deadline_hook) + app.register_error_handler(GenerationDeadlineExceeded, lambda error: error_response()) + app.after_request(deadline_response) diff --git a/services/proxy_service.py b/services/proxy_service.py index 08373c0c..1c5bbaa7 100644 --- a/services/proxy_service.py +++ b/services/proxy_service.py @@ -15,7 +15,10 @@ from services.credential_pool import CredentialPool from services.resilience_service import ResilienceService from services.upstream_outcome import UpstreamOutcome, classify_upstream_outcome -from services.managed_dispatch import retry_managed_attempt +from services.generation_deadline import ( + GenerationDeadlineExceeded, bounded_timeout, check_deadline, own_upstream, + retry_with_deadline as retry_managed_attempt, +) from services.retry_advice import retry_advice_settings from services.transport_policy import RAW_PASSTHROUGH_PROVIDERS from services.secret_firewall import protect_body @@ -1219,7 +1222,7 @@ def _make_request_with_timeout( pass from services.admission_request import bounded_generation_timeout - timeout = bounded_generation_timeout(timeout) + timeout = bounded_timeout(bounded_generation_timeout(timeout)) response = session.request( method=method, @@ -1230,11 +1233,13 @@ def _make_request_with_timeout( timeout=timeout, allow_redirects=False, verify=True, - stream=is_streaming + stream=is_streaming or check_deadline() is not None ) + own_upstream(response) if not is_streaming: _ = response.content + check_deadline() logger.info( "Upstream response status=%s content_length=%s", response.status_code, @@ -1262,6 +1267,7 @@ def _make_base_request( """ Make a base request with retries and error handling """ + check_deadline() data = protect_body(data, headers, provider=api_provider) for key in list(headers): if key.lower() == "content-length" and data is not None: @@ -1304,7 +1310,7 @@ def next_attempt(count: int) -> requests.Response: ) from services.admission_request import bounded_generation_timeout - timeout = bounded_generation_timeout(timeout) + timeout = bounded_timeout(bounded_generation_timeout(timeout)) response = session.request( method=method, @@ -1316,6 +1322,7 @@ def next_attempt(count: int) -> requests.Response: timeout=timeout, allow_redirects=False, ) + own_upstream(response) # Native/raw providers expose protocol-specific streams and binary # bodies. Reading, normalizing, retrying, or substituting a local @@ -1359,6 +1366,7 @@ def next_attempt(count: int) -> requests.Response: try: # Let requests handle decompression automatically content = response.content + check_deadline() content_type = response.headers.get('content-type', '').lower() if not content or "json" not in content_type: cls._record_circuit_result(api_provider, response.status_code) @@ -1421,6 +1429,8 @@ def next_attempt(count: int) -> requests.Response: response._content = json.dumps(normalized_payload).encode('utf-8') response.headers['Content-Type'] = 'application/json' + except GenerationDeadlineExceeded: + raise except Exception as error: logger.error( "Response normalization failed provider=%s type=%s", @@ -1433,7 +1443,19 @@ def next_attempt(count: int) -> requests.Response: cls._record_circuit_result(api_provider, response.status_code) return response + except GenerationDeadlineExceeded: + if not raw_passthrough: + cls._record_circuit_result(api_provider, 504, + outcome=classify_upstream_outcome(cancelled=True)) + raise except requests.exceptions.RequestException as e: + try: + check_deadline() + except GenerationDeadlineExceeded: + if not raw_passthrough: + cls._record_circuit_result(api_provider, 504, + outcome=classify_upstream_outcome(cancelled=True)) + raise if raw_passthrough: provider_name = { **{pool.provider: pool.display_name for pool in CODEX_EVERYWHERE_POOLS}, diff --git a/tests/fixtures/secret_firewall_coverage.json b/tests/fixtures/secret_firewall_coverage.json index ee82f1ec..96e08791 100644 --- a/tests/fixtures/secret_firewall_coverage.json +++ b/tests/fixtures/secret_firewall_coverage.json @@ -233,6 +233,7 @@ "worker/fallback-page.mjs", "worker/gateway-extensions.mjs", "worker/gateway-lifecycle.mjs", + "worker/generation-deadline.mjs", "worker/gpt-image-moderation.mjs", "worker/health-schedule.mjs", "worker/intelligence-auth-d1.mjs", diff --git a/tests/test_generation_deadline.py b/tests/test_generation_deadline.py new file mode 100644 index 00000000..ba2fa9c0 --- /dev/null +++ b/tests/test_generation_deadline.py @@ -0,0 +1,277 @@ +"""Generation budgets with fake clocks and synthetic upstreams.""" +import json +from unittest.mock import Mock, patch + +import pytest +import requests +from flask import Flask, Response, g + +from services import generation_deadline as gd +from services.proxy_service import ProxyService +from services.resilience_service import ResilienceService + + +class Clock: + def __init__(self): + self.now = 100.0 + + def __call__(self): + return self.now + + +@pytest.fixture(autouse=True) +def isolated(monkeypatch): + monkeypatch.setenv("GENERATION_DEADLINE_MAX_MS", "") + monkeypatch.setenv("SECRET_SCAN_DEFAULT", "off") + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_ADVICE_ENABLED", "false") + ResilienceService.reset() + yield + ResilienceService.reset() + + +def upstream(status=200): + response = requests.Response() + response.status_code = status + response._content = b'{}' + response.headers = {"Content-Type": "application/json", "Retry-After": "20"} + response.close = Mock() + return response + + +def test_absent_is_identity_even_with_spoofed_internal_budget(): + assert gd.deadline_from_headers({gd.INTERNAL_HEADER: "1"}) is None + app = Flask(__name__) + gd.register_generation_deadline(app) + with app.test_request_context("/v1/chat/completions", method="POST"): + assert gd.current_deadline() is None + assert gd.bounded_timeout((5, 60)) == (5, 60) + + +@pytest.mark.parametrize("value", ["0", "-1", "1.5", "300001", "bad", ""]) +def test_invalid_header_is_rejected(value): + with pytest.raises(gd.InvalidGenerationDeadline): + gd.deadline_from_headers({gd.PUBLIC_HEADER: value}) + + +def test_one_clock_minimum_and_authenticated_transport(): + clock = Clock() + deadline = gd.deadline_from_headers( + {gd.PUBLIC_HEADER: "1000", gd.INTERNAL_HEADER: "100"}, clock=clock, + limits_ms=(500,), + ) + assert deadline.remaining_ms() == 500 + clock.now += .2 + assert 299 <= deadline.remaining_ms() <= 300 + forwarded = gd.forwarded_headers({gd.INTERNAL_HEADER: "99999"}, deadline) + assert int(forwarded[gd.INTERNAL_HEADER]) <= 300 + remote = gd.deadline_from_headers(forwarded, trusted_internal=True, clock=lambda: 900) + assert remote.expires_at < 901 + assert gd.forwarded_headers({gd.INTERNAL_HEADER: "1"}, None) == {} + + +def test_malformed_maximum_disables_without_logging_value(monkeypatch, caplog): + monkeypatch.setenv("GENERATION_DEADLINE_MAX_MS", "private-invalid") + assert gd.deadline_from_headers({gd.PUBLIC_HEADER: "100"}) is None + assert gd.deadline_from_headers({gd.PUBLIC_HEADER: "100"}) is None + assert "private-invalid" not in caplog.text + + +def test_setup_and_token_count_consume_transport_budget(): + clock = Clock() + app = Flask(__name__) + session = Mock() + session.request.return_value = upstream() + with app.test_request_context(): + g.generation_deadline = gd.Deadline(101, clock) + clock.now += .75 + with patch.object(ProxyService, "_get_provider_session", return_value=session): + result = ProxyService._make_base_request("POST", "https://example.invalid", {}, {}, b'{}', "openai") + assert result.status_code == 200 + assert sum(session.request.call_args.kwargs["timeout"]) <= .25 + clock.now = 102 + with patch.object(ProxyService, "_get_provider_session", return_value=session): + with pytest.raises(gd.GenerationDeadlineExceeded): + ProxyService._make_request_with_timeout("POST", "https://example.invalid", {}, {}, b'{}', (5, 60)) + assert session.request.call_count == 1 + + +@pytest.mark.parametrize("advice", ["false", "true"]) +def test_retry_wait_cannot_overrun_or_replay(monkeypatch, advice): + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_ADVICE_ENABLED", advice) + clock = Clock() + response = upstream(429) + session = Mock(request=Mock(return_value=response)) + with Flask(__name__).test_request_context(): + g.generation_deadline = gd.Deadline(100.5, clock) + with patch.object(ProxyService, "_get_provider_session", return_value=session), patch("services.proxy_service.time.sleep") as sleep: + with pytest.raises(gd.GenerationDeadlineExceeded): + ProxyService._make_base_request("GET", "https://example.invalid", {}, {}, None, "openai") + assert session.request.call_count == 1 + sleep.assert_not_called() + assert response.multillm_cancellation.outcome.usage_state == "unknown" + assert not response.multillm_cancellation.outcome.upstream.replay_permission + + +def test_midstream_error_is_real_and_cancellation_is_ambiguous(): + clock = Clock() + source = upstream() + def chunks(): + yield b'data: {"choices":[{"delta":{"content":"a"}}]}\n\n' + clock.now = 102 + yield b'data: [DONE]\n\n' + response = Response(chunks(), content_type="text/event-stream") + body = gd.DeadlineIterator(iter(response.response), gd.Deadline(101, clock), source) + assert b'"content"' in next(body) + error = next(body) + assert b"generation_deadline_exceeded" in error and b"[DONE]" not in error + assert list(body) == [] + assert source.multillm_cancellation.outcome.ambiguous + assert source.multillm_cancellation.outcome.usage is None + + +def test_preflight_expiry_is_504_through_registered_callback(): + clock = Clock() + app = Flask(__name__) + gd.register_generation_deadline(app, clock=clock) + @app.before_request + def authorized(): + return gd.generation_deadline_hook() + @app.post("/v1/chat/completions") + def generate(): + def chunks(): + yield b': heartbeat\n\n' + clock.now += 2 + yield b'data: {"choices":[{"delta":{"content":"late"}}]}\n\n' + return Response(chunks(), content_type="text/event-stream") + result = app.test_client().post("/v1/chat/completions", headers={gd.PUBLIC_HEADER: "1000"}) + assert result.status_code == 504 + assert result.json["error"]["code"] == "generation_deadline_exceeded" + + +def test_expiry_releases_half_open_probe_without_healing(): + clock = Clock() + ResilienceService._states["openai"] = {**ResilienceService._new_state(), "state": "half_open"} + def setup(*args, **kwargs): + clock.now = 102 + return Mock() + with Flask(__name__).test_request_context(): + g.generation_deadline = gd.Deadline(101, clock) + with patch.object(ProxyService, "_get_provider_session", side_effect=setup): + with pytest.raises(gd.GenerationDeadlineExceeded): + ProxyService._make_base_request("GET", "https://example.invalid", {}, {}, None, "openai") + state = ResilienceService.snapshot("openai") + assert state["half_open_in_flight"] == state["total_successes"] == 0 + + +def test_internal_budget_requires_verification_and_caps_admission(): + app = Flask(__name__) + gd.register_generation_deadline(app, verify_internal=lambda: True, limits_ms=lambda: (500,)) + with app.test_request_context("/v1/chat/completions", method="POST", headers={gd.INTERNAL_HEADER: "100"}): + gd.generation_deadline_hook() + deadline = gd.current_deadline() + assert deadline.remaining() <= .1 + assert g.cascade_deadline == g.gateway_generation_deadline == deadline.expires_at + deadline.stop() + + +def test_unknown_final_usage_keeps_ambiguous_settlement(): + from services.request_cancellation import CancellationContext + owner = CancellationContext(lambda: None) + owner.handoff() + owner.complete() + result = gd.settlement_information(owner) + assert result["ambiguous"] and result["usage_state"] == "unknown" + assert not result["replay_permission"] + + +def test_auto_route_expiry_stops_next_candidate_and_closes_response(): + from routes import auto_routes + from services.auto_route_service import AutoRoute + clock = Clock() + close = Mock() + calls = [] + route = AutoRoute("auto:synthetic", "synthetic", ["openai:first", "openai:second"]) + def dispatch(*args): + calls.append(args) + clock.now = 102 + response = Response("refusal", status=429) + response.call_on_close(close) + return response + with Flask(__name__).test_request_context(): + g.generation_deadline = gd.Deadline(101, clock) + with patch.object(auto_routes.AutoRouteService, "get_route", return_value=route): + with pytest.raises(gd.GenerationDeadlineExceeded): + auto_routes.dispatch_auto_route({"model": "auto:synthetic"}, validate_candidate=lambda model: None, dispatch_candidate=dispatch) + assert len(calls) == close.call_count == 1 + + +def test_registered_raw_defaults_preserve_binary_and_headers(): + app = Flask(__name__) + gd.register_generation_deadline(app) + @app.before_request + def authorized(): + return gd.generation_deadline_hook() + @app.post("/v1/chat/completions") + def generate(): + return Response(iter([b"\x00\xff", b"raw"]), content_type="application/octet-stream", headers={"X-Test": "same"}) + result = app.test_client().post("/v1/chat/completions", headers={gd.INTERNAL_HEADER: "1"}) + assert result.status_code == 200 and result.data == b"\x00\xffraw" + assert result.headers["X-Test"] == "same" + + +def test_active_deadline_does_not_retry_ambiguous_post_timeout(): + session = Mock(request=Mock(side_effect=requests.ReadTimeout("synthetic"))) + with Flask(__name__).test_request_context(): + g.generation_deadline = gd.Deadline(101, Clock()) + with patch.object(ProxyService, "_get_provider_session", return_value=session): + result = ProxyService._make_base_request("POST", "https://example.invalid", {}, {}, b'{}', "openai") + assert result.status_code == 502 and session.request.call_count == 1 + + +@pytest.mark.parametrize("path", ["/v1/responses", "/v1/messages"]) +def test_other_protocol_precommit_expiry_is_504(path): + clock = Clock() + app = Flask(__name__) + gd.register_generation_deadline(app, clock=clock) + @app.before_request + def authorized(): + return gd.generation_deadline_hook() + def generate(): + def chunks(): + yield b': heartbeat\n\n' + clock.now += 2 + yield b'data: {"type":"content_block_delta","delta":{"text":"late"}}\n\n' + return Response(chunks(), content_type="text/event-stream") + app.add_url_rule(path, view_func=generate, methods=["POST"]) + result = app.test_client().post(path, headers={gd.PUBLIC_HEADER: "1000"}) + assert result.status_code == 504 + + +@pytest.mark.parametrize("protocol", ["responses", "anthropic"]) +def test_protocol_error_event_contains_error_type(protocol): + clock = Clock() + def chunks(): + yield b'data: useful\n\n' + clock.now += 2 + yield b'data: [DONE]\n\n' + body = gd.DeadlineIterator(chunks(), gd.Deadline(101, clock), upstream(), protocol=protocol) + next(body) + event = next(body) + assert b'"type": "error"' in event + if protocol == "anthropic": + assert json.loads(event.split(b"data: ", 1)[1])["error"]["type"] == "api_error" + assert list(body) == [] + + +def test_prefix_bound_does_not_commit_heartbeats_as_success(): + app = Flask(__name__) + gd.register_generation_deadline(app) + @app.before_request + def authorized(): + return gd.generation_deadline_hook() + @app.post("/v1/responses") + def generate(): + return Response(iter([b': heartbeat\n\n' * 6000]), content_type="text/event-stream") + result = app.test_client().post("/v1/responses", headers={gd.PUBLIC_HEADER: "1000"}) + assert result.status_code == 502 + assert result.json["error"]["code"] == "upstream_stream_invalid" diff --git a/tests/test_generation_deadline_worker.mjs b/tests/test_generation_deadline_worker.mjs new file mode 100644 index 00000000..11b3695b --- /dev/null +++ b/tests/test_generation_deadline_worker.mjs @@ -0,0 +1,169 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { createGenerationDeadline, forwardedDeadlineHeaders, GenerationDeadlineExceeded, + withGenerationDeadline, generationDeadlineHook } from "../worker/generation-deadline.mjs"; +import { settlementInformation } from "../worker/generation-deadline.mjs"; +import { UpstreamCancellation } from "../worker/upstream-cancellation.mjs"; + +test("absent deadline preserves response and timeout policy", async () => { + const request = new Request("https://example.invalid", { headers: { "X-MultiLLM-Internal-Deadline-Ms": "1" } }); + assert.equal(createGenerationDeadline(request), null); + const response = new Response("raw"); + assert.equal(await withGenerationDeadline(request, {}, () => response), response); + assert.equal(forwardedDeadlineHeaders(request.headers, null).has("X-MultiLLM-Internal-Deadline-Ms"), false); +}); + +test("one local clock and trusted remaining budget use minimum limits", () => { + let now = 100; + const request = new Request("https://example.invalid", { headers: { + "X-MultiLLM-Deadline-Ms": "1000", "X-MultiLLM-Internal-Deadline-Ms": "1" } }); + const deadline = createGenerationDeadline(request, {}, { now: () => now, limitsMs: [500] }); + now += 200; + assert.equal(deadline.remainingMs(), 300); + const headers = forwardedDeadlineHeaders(request.headers, deadline); + const remote = createGenerationDeadline(new Request(request, { headers }), {}, { trustedInternal: true, now: () => 5000 }); + assert.equal(remote.remainingMs(), 300); + now = 1000; + assert.throws(() => deadline.check(), GenerationDeadlineExceeded); +}); + +test("invalid request rejects and malformed maximum disables", () => { + for (const value of ["", "0", "-1", "1.5", "bad", "300001"]) { + assert.throws(() => createGenerationDeadline(new Request("https://example.invalid", { + headers: { "X-MultiLLM-Deadline-Ms": value } })), /Invalid generation deadline/); + } + assert.equal(createGenerationDeadline(new Request("https://example.invalid", { + headers: { "X-MultiLLM-Deadline-Ms": "1000" } }), { GENERATION_DEADLINE_MAX_MS: "bad" }), null); +}); + +test("setup expiry returns 504 without provider replay", async () => { + let now = 0; + let calls = 0; + const request = new Request("https://example.invalid", { headers: { "X-MultiLLM-Deadline-Ms": "100" } }); + const result = await withGenerationDeadline(request, {}, () => { calls++; now = 101; return new Response("late"); }, { now: () => now }); + assert.equal(result.status, 504); + assert.equal((await result.json()).error.code, "generation_deadline_exceeded"); + assert.equal(calls, 1); +}); + +test("midstream expiry emits error without DONE and retains unknown usage", async () => { + let now = 0; + let reads = 0; + const request = new Request("https://example.invalid", { headers: { "X-MultiLLM-Deadline-Ms": "100" } }); + const owner = new UpstreamCancellation(); + const body = new ReadableStream({ pull(controller) { + if (++reads === 1) controller.enqueue(new TextEncoder().encode('data: {"choices":[{"delta":{"content":"a"}}]}\n\n')); + else { now = 101; controller.enqueue(new TextEncoder().encode("data: [DONE]\n\n")); } + } }, { highWaterMark: 0 }); + const result = await withGenerationDeadline(request, {}, () => new Response(body, { headers: { "Content-Type": "text/event-stream" } }), { now: () => now, owner }); + const content = await result.text(); + assert.match(content, /generation_deadline_exceeded/); + assert.doesNotMatch(content, /\[DONE\]/); + assert.equal(owner.outcome.ambiguous, true); + assert.equal(owner.outcome.usageState, "unknown"); + assert.equal(owner.outcome.replayPermission, false); +}); + +test("pending header/read aborts on one timer with ambiguous settlement", async () => { + const request = new Request("https://example.invalid", { headers: { "X-MultiLLM-Deadline-Ms": "10" } }); + const owner = new UpstreamCancellation(); + const result = await withGenerationDeadline(request, {}, () => new Promise(() => {}), { owner }); + assert.equal(result.status, 504); + assert.equal(owner.outcome.ambiguous, true); +}); + +test("native lifecycle hook consumes setup and never resets at dispatch", async () => { + let now = 0; + const request = new Request("https://example.invalid", { headers: { "X-MultiLLM-Deadline-Ms": "100" } }); + const hook = generationDeadlineHook(request, {}, { now: () => now }); + await hook.authorize({}); + now = 101; + assert.throws(() => hook.before_dispatch({}), GenerationDeadlineExceeded); + await hook.finalize({}); +}); + +test("unknown final usage stays ambiguous even at EOF", () => { + const owner = new UpstreamCancellation(); + owner.handoff(); + owner.complete(); + const settlement = settlementInformation(owner); + assert.equal(settlement.ambiguous, true); + assert.equal(settlement.usageState, "unknown"); + assert.equal(settlement.replayPermission, false); +}); + +test("caller cancellation aborts a pending call without hanging or replay", async () => { + const controller = new AbortController(); + const request = new Request("https://example.invalid", { signal: controller.signal, + headers: { "X-MultiLLM-Deadline-Ms": "1000" } }); + let calls = 0; + const result = withGenerationDeadline(request, {}, () => { calls++; controller.abort("client_cancelled"); return new Promise(() => {}); }); + await assert.rejects(result, { name: "AbortError" }); + assert.equal(calls, 1); +}); + +test("preflight heartbeats consume budget before useful output", async () => { + let now = 0; + let reads = 0; + const request = new Request("https://example.invalid", { headers: { "X-MultiLLM-Deadline-Ms": "100" } }); + const source = new ReadableStream({ pull(controller) { + if (++reads === 1) controller.enqueue(new TextEncoder().encode(": heartbeat\n\n")); + else { now = 101; controller.enqueue(new TextEncoder().encode('data: {"choices":[{"delta":{"content":"late"}}]}\n\n')); } + } }, { highWaterMark: 0 }); + const result = await withGenerationDeadline(request, {}, () => new Response(source, { headers: { "Content-Type": "text/event-stream" } }), { now: () => now }); + assert.equal(result.status, 504); + assert.equal((await result.json()).error.code, "generation_deadline_exceeded"); +}); + +test("allowed wait cannot exceed remaining generation time", async () => { + const request = new Request("https://example.invalid", { headers: { "X-MultiLLM-Deadline-Ms": "1000" } }); + const hook = generationDeadlineHook(request, {}, { now: () => 100 }); + hook.authorize({}); + let sleeps = 0; + await assert.rejects(hook.deadline.wait(2000, () => { sleeps++; }), GenerationDeadlineExceeded); + assert.equal(sleeps, 0); + hook.finalize({}); +}); + +for (const protocol of ["responses", "anthropic"]) { + test(`${protocol} expiry emits a protocol error event`, async () => { + let now = 0; + let reads = 0; + const request = new Request("https://example.invalid", { headers: { "X-MultiLLM-Deadline-Ms": "100" } }); + const source = new ReadableStream({ pull(controller) { + if (++reads === 1) controller.enqueue(new TextEncoder().encode('data: {"type":"content_block_delta","delta":{"text":"a"}}\n\n')); + else { now = 101; controller.enqueue(new TextEncoder().encode("data: [DONE]\n\n")); } + } }, { highWaterMark: 0 }); + const result = await withGenerationDeadline(request, {}, () => new Response(source, { headers: { "Content-Type": "text/event-stream" } }), + { now: () => now, protocol, preflight: false }); + const content = await result.text(); + assert.match(content, /"type":"error"/); + assert.doesNotMatch(content, /\[DONE\]/); + if (protocol === "anthropic") assert.match(content, /"type":"api_error"/); + }); +} + +test("late header response is cancelled after deadline return", async () => { + const request = new Request("https://example.invalid", { headers: { "X-MultiLLM-Deadline-Ms": "10" } }); + let resolveFetch; + let cancelled = 0; + const source = new ReadableStream({ cancel() { cancelled++; } }); + const result = await withGenerationDeadline(request, {}, () => new Promise(resolve => { resolveFetch = resolve; })); + assert.equal(result.status, 504); + resolveFetch(new Response(source)); + await new Promise(resolve => setTimeout(resolve, 0)); + assert.equal(cancelled, 1); +}); + +test("prefix bound rejects heartbeats before committing success", async () => { + const request = new Request("https://example.invalid", { headers: { "X-MultiLLM-Deadline-Ms": "1000" } }); + const result = await withGenerationDeadline(request, {}, () => new Response(": heartbeat\n\n".repeat(6000), { headers: { "Content-Type": "text/event-stream" } })); + assert.equal(result.status, 502); + assert.equal((await result.json()).error.code, "upstream_stream_invalid"); +}); + +test("DONE without useful output cannot pass preflight", async () => { + const request = new Request("https://example.invalid", { headers: { "X-MultiLLM-Deadline-Ms": "1000" } }); + const result = await withGenerationDeadline(request, {}, () => new Response("data: [DONE]\n\n", { headers: { "Content-Type": "text/event-stream" } })); + assert.equal(result.status, 502); +}); diff --git a/worker/container-env.mjs b/worker/container-env.mjs index 241cf668..ebc4c806 100644 --- a/worker/container-env.mjs +++ b/worker/container-env.mjs @@ -27,6 +27,7 @@ const DIRECT_ENV_KEYS = [ "INTELLIGENCE_NANOGPT_SUBSCRIPTION_API_KEY", "WORKBENCH_WORKER_URL", "MCP_CONTRACT_DIGESTS_ENABLED", + "GENERATION_DEADLINE_MAX_MS", "CONNECTION_PROFILES_DB_PATH", "CONFIG_REVISION_SYNC_ENABLED", "CONFIG_SYNC_TTL_SECONDS", diff --git a/worker/cors-policy.mjs b/worker/cors-policy.mjs index fbe48ac9..2324dfa2 100644 --- a/worker/cors-policy.mjs +++ b/worker/cors-policy.mjs @@ -1,6 +1,6 @@ export const CORS_ALLOWED_METHODS = "GET, POST, PUT, DELETE, PATCH, OPTIONS"; export const CORS_DEFAULT_HEADERS = - "X-MultiLLM-Tool-Repair, X-MultiLLM-Image-QA, X-MultiLLM-Cascade, X-MultiLLM-Conversion-Report, X-MultiLLM-Retention, Authorization, X-Api-Key, X-Goog-Api-Key, X-MultiLLM-Api-Key, X-Roleplay-Session-ID, Anthropic-Version, Anthropic-Beta, Anthropic-Dangerous-Direct-Browser-Access, Content-Type, Accept, Origin, X-Requested-With, OpenAI-Beta, OpenAI-Organization, OpenAI-Project, Idempotency-Key, Moderation, Moderation-Model, Redaction, X-Client-Request-ID, X-App-Name, X-Billing-Mode, X-BYOK-Provider, X-Encryption-Key, X-Encryption-Passphrase, X-Fal-Object-Lifecycle-Preference, X-PAYMENT, X-Prompt-Caching-Cut-After, X-Provider, X-Team-ID, X-Use-BYOK, x-x402"; + "X-MultiLLM-Tool-Repair, X-MultiLLM-Image-QA, X-MultiLLM-Cascade, X-MultiLLM-Conversion-Report, X-MultiLLM-Retention, X-MultiLLM-Deadline-Ms, Authorization, X-Api-Key, X-Goog-Api-Key, X-MultiLLM-Api-Key, X-Roleplay-Session-ID, Anthropic-Version, Anthropic-Beta, Anthropic-Dangerous-Direct-Browser-Access, Content-Type, Accept, Origin, X-Requested-With, OpenAI-Beta, OpenAI-Organization, OpenAI-Project, Idempotency-Key, Moderation, Moderation-Model, Redaction, X-Client-Request-ID, X-App-Name, X-Billing-Mode, X-BYOK-Provider, X-Encryption-Key, X-Encryption-Passphrase, X-Fal-Object-Lifecycle-Preference, X-PAYMENT, X-Prompt-Caching-Cut-After, X-Provider, X-Team-ID, X-Use-BYOK, x-x402"; export const CORS_EXPOSE_HEADERS = "X-MultiLLM-Secret-Scan, X-MultiLLM-Retention, X-MultiLLM-Roleplay-Recovery, " + "X-MultiLLM-Tool-Repair, X-MultiLLM-Image-QA, X-MultiLLM-Cascade, X-MultiLLM-Conversion-Fidelity, X-MultiLLM-Conversion-Fields, Retry-After, X-Request-ID, X-MultiLLM-RateLimit-Limit, X-MultiLLM-RateLimit-Remaining, X-MultiLLM-RateLimit-Reset, X-MultiLLM-Optimization, X-MultiLLM-Optimization-Mode, X-MultiLLM-Estimated-Input-Before, X-MultiLLM-Estimated-Input-After, X-MultiLLM-Image-Prompts-Compacted, X-MultiLLM-Messages-Summarized, X-MultiLLM-Optimization-Target-Met, X-MultiLLM-Summary, X-MultiLLM-Optimization-Cache-Hits, X-MultiLLM-Optimization-Cache-Misses, X-MultiLLM-Prompt-Cache, X-MultiLLM-Prompt-Cache-Mode, X-MultiLLM-Prompt-Cache-Estimated-Tokens, X-MultiLLM-Provider, X-MultiLLM-Model, X-MultiLLM-Credential-Attempts, X-MultiLLM-Route-Decision, X-MultiLLM-Circuit-State, X-MultiLLM-Auto-Route, X-MultiLLM-Auto-Selected-Model, X-MultiLLM-Auto-Attempts, X-MultiLLM-Auto-Selected-Priority, X-MultiLLM-Auto-Ordering, X-MultiLLM-Auto-Failover-Reasons, X-MultiLLM-Cache, X-MultiLLM-Transport-Failure, X-MultiLLM-Token-Count, X-MultiLLM-Latency-Ms, X-MultiLLM-Estimated-Cost-USD, X-MultiLLM-Cost-Basis, X-MultiLLM-Reasoning-Normalized, X-Roleplay-Session-ID, X-Roleplay-Trace-ID, X-Roleplay-Session-Source, X-Roleplay-Provider, X-Roleplay-Model, X-Roleplay-Selection, X-Roleplay-Memory, X-Roleplay-Estimated-Input-Tokens, X-Roleplay-Max-Output-Tokens, X-Roleplay-Fallback-Count, X-Roleplay-State-Cache, X-Roleplay-Credential-Check, Server-Timing, WWW-Authenticate, X-PAYMENT-RESPONSE, X-Poll-After, X-NanoGPT-Advisor-ID, X-NanoGPT-Data-Endpoint, X-NanoGPT-Direct-Endpoint, X-NanoGPT-Inline-Moderation-Cost-USD, X-NanoGPT-Inline-Moderation-Flagged, X-NanoGPT-Inline-Moderation-Model"; diff --git a/worker/generation-deadline.mjs b/worker/generation-deadline.mjs new file mode 100644 index 00000000..d501d6bf --- /dev/null +++ b/worker/generation-deadline.mjs @@ -0,0 +1,249 @@ +// One local generation budget across setup, retries, preflight and body reads. +import { UpstreamCancellation } from "./upstream-cancellation.mjs"; + +export const DEADLINE_HEADER = "X-MultiLLM-Deadline-Ms"; +export const INTERNAL_DEADLINE_HEADER = "X-MultiLLM-Internal-Deadline-Ms"; +const DEFAULT_MAX_MS = 300000; +let invalidMaximumLogged = false; + +export class GenerationDeadlineExceeded extends Error { + constructor() { super("Generation deadline exceeded"); this.status = 504; this.code = "generation_deadline_exceeded"; } + response() { return Response.json(errorPayload(), { status: 504 }); } +} + +export class InvalidGenerationDeadline extends Error { + constructor() { super("Invalid generation deadline"); this.status = 400; } + response() { return Response.json({ error: { code: "invalid_generation_deadline", message: this.message } }, { status: 400 }); } +} + +class StreamPrefixUnavailable extends Error { + response() { return Response.json({ error: { type: "upstream_error", code: "upstream_stream_invalid", + message: "Upstream stream ended without a useful bounded prefix" } }, { status: 502 }); } +} + +function errorPayload() { + return { error: { message: "Generation deadline exceeded", type: "timeout_error", code: "generation_deadline_exceeded" } }; +} + +export function settlementInformation(owner, confirmedUsage = null) { + return Object.freeze({ ambiguous: owner.handedOff && confirmedUsage === null, + usageState: confirmedUsage === null ? "unknown" : "known", usage: confirmedUsage, + replayPermission: false, cancellationOutcome: owner.outcome }); +} + +function errorEvent(protocol) { + let payload = errorPayload(); + if (protocol === "responses") payload = { ...payload.error, type: "error" }; + if (protocol === "anthropic") payload = { type: "error", error: { ...payload.error, type: "api_error" } }; + return new TextEncoder().encode(`event: error\ndata: ${JSON.stringify(payload)}\n\n`); +} + +function maximum(env) { + const raw = String(env.GENERATION_DEADLINE_MAX_MS ?? "").trim(); + if (!raw) return DEFAULT_MAX_MS; + if (/^[0-9]{1,6}$/.test(raw) && Number(raw) >= 1 && Number(raw) <= DEFAULT_MAX_MS) return Number(raw); + if (!invalidMaximumLogged) { console.warn("Invalid GENERATION_DEADLINE_MAX_MS; generation deadlines disabled"); invalidMaximumLogged = true; } + return null; +} + +function milliseconds(value, limit) { + if (!/^[0-9]{1,6}$/.test(value) || Number(value) < 1 || Number(value) > limit) throw new InvalidGenerationDeadline(); + return Number(value); +} + +export class GenerationDeadline { + constructor(expiresAt, now = () => performance.now()) { this.expiresAt = expiresAt; this.now = now; } + remainingMs() { return Math.max(0, Math.floor(this.expiresAt - this.now())); } + check() { + if (this.expired || this.expiresAt <= this.now()) { + this.expired = true; + void this.owner?.close("generation_deadline_exceeded"); + throw new GenerationDeadlineExceeded(); + } + } + start(owner) { + if (this.expiry) return; + this.owner = owner; + this.check(); + owner.throwIfAborted(); + this.abort = new Promise((resolve, reject) => { + this.abortListener = () => reject(this.expired ? new GenerationDeadlineExceeded() + : new DOMException("Request aborted", "AbortError")); + owner.controller.signal.addEventListener("abort", this.abortListener, { once: true }); + }); + void this.abort.catch(() => {}); + this.expiry = new Promise((resolve, reject) => { + this.timer = setTimeout(() => { + this.expired = true; + void owner.close("generation_deadline_exceeded"); + reject(new GenerationDeadlineExceeded()); + }, this.remainingMs()); + }); + // Lifecycle hooks can arm before the first asynchronous wait begins. + void this.expiry.catch(() => {}); + } + async run(action, owner = this.owner) { + this.start(owner); + this.check(); + const result = await Promise.race([Promise.resolve().then(action), this.expiry, this.abort]); + this.check(); + return result; + } + async wait(ms, sleep = duration => new Promise(resolve => setTimeout(resolve, duration))) { + this.check(); + if (!Number.isFinite(ms) || ms < 0 || ms >= this.remainingMs()) { + void this.owner?.close("generation_deadline_exceeded"); + throw new GenerationDeadlineExceeded(); + } + return this.run(() => sleep(ms)); + } + stop() { + clearTimeout(this.timer); + this.owner?.controller.signal.removeEventListener("abort", this.abortListener); + } +} + +export function createGenerationDeadline(request, env = {}, { trustedInternal = false, limitsMs = [], now = () => performance.now() } = {}) { + const limit = maximum(env); + if (limit === null) return null; + const publicMs = request.headers.get(DEADLINE_HEADER); + const internalMs = trustedInternal ? request.headers.get(INTERNAL_DEADLINE_HEADER) : null; + if (publicMs === null && internalMs === null) return null; + const budgets = [publicMs, internalMs].filter(value => value !== null).map(value => milliseconds(value, limit)); + budgets.push(...limitsMs.filter(value => Number.isInteger(value) && value > 0)); + return new GenerationDeadline(now() + Math.min(...budgets), now); +} + +export function forwardedDeadlineHeaders(original, deadline) { + const headers = new Headers(original); + headers.delete(INTERNAL_DEADLINE_HEADER); + if (deadline) { + deadline.check(); + const budget = deadline.remainingMs(); + if (budget < 1) throw new GenerationDeadlineExceeded(); + headers.delete(DEADLINE_HEADER); + headers.set(INTERNAL_DEADLINE_HEADER, String(budget)); + } + return headers; +} + +function usefulPrefix(bytes, protocol) { + const text = new TextDecoder().decode(bytes); + const events = text.replaceAll("\r\n", "\n").split("\n\n"); + events.pop(); + return events.some(event => { + const data = event.split("\n").filter(line => line.startsWith("data:")).map(line => line.slice(5).trimStart()).join("\n"); + if (!data) return false; + if (data === "[DONE]") return false; + try { + const payload = JSON.parse(data); + if (payload.error || payload.type === "error") return true; + if (protocol === "responses") return /(?:delta|completed|failed)$/.test(payload.type ?? ""); + if (protocol === "anthropic") return payload.type === "content_block_delta" || payload.type === "message_stop"; + return payload.choices?.some(choice => choice.finish_reason != null || Object.entries(choice.delta ?? {}).some(([key, value]) => + ["content", "reasoning_content", "reasoning", "thinking", "tool_calls", "function_call"].includes(key) && Boolean(value))); + } catch { return false; } + }); +} + +export async function deadlineResponse(response, deadline, owner, { protocol = "chat", preflight = true } = {}) { + deadline.check(); + if (!response.body) { deadline.stop(); owner.complete(); return response; } + const sse = response.headers.get("content-type")?.startsWith("text/event-stream"); + const inspectPrefix = sse && preflight && response.status < 400; + let useful = !inspectPrefix; + const reader = response.body.getReader(); + owner.reader = reader; + const read = () => deadline.run(() => reader.read(), owner); + let first; + const prefix = []; + let size = 0; + try { + do { + first = await read(); + if (first.done) break; + prefix.push(first.value); + size += first.value.byteLength; + if (inspectPrefix) { + const bytes = new Uint8Array(Math.min(size, 65536)); + let offset = 0; + for (const chunk of prefix) { + const inspected = chunk.subarray(0, bytes.length - offset); + bytes.set(inspected, offset); offset += inspected.byteLength; + } + useful = usefulPrefix(bytes, protocol); + if (useful) break; + } else break; + } while (size < 65536); + if (!useful) throw new StreamPrefixUnavailable(); + } catch (error) { + deadline.stop(); + await owner.close("interrupted"); + throw error; + } + const body = new ReadableStream({ + async pull(controller) { + try { + deadline.check(); + if (prefix.length) { controller.enqueue(prefix.shift()); return; } + const chunk = first.done ? first : await read(); + if (chunk.done) { controller.close(); deadline.stop(); owner.complete(); } + else controller.enqueue(chunk.value); + } catch (error) { + deadline.stop(); + await owner.close("interrupted"); + if (error instanceof GenerationDeadlineExceeded && sse) { + controller.enqueue(errorEvent(protocol)); + controller.close(); + } else controller.error(error); + } + }, + cancel(reason) { deadline.stop(); return owner.close(reason); }, + }, { highWaterMark: 0 }); + return new Response(body, { status: response.status, statusText: response.statusText, headers: response.headers }); +} + +export async function withGenerationDeadline(request, env, fetcher, options = {}) { + let deadline; + try { deadline = options.deadline ?? createGenerationDeadline(request, env, options); } + catch (error) { if (error instanceof InvalidGenerationDeadline) return error.response(); throw error; } + if (!deadline) return fetcher(request); + const owner = options.owner ?? new UpstreamCancellation({ signal: request.signal, onOutcome: options.onOutcome }); + // A sanitized external request must never carry an internal transport budget. + const headers = new Headers(request.headers); + headers.delete(INTERNAL_DEADLINE_HEADER); + headers.delete(DEADLINE_HEADER); + const outbound = new Request(request, { headers, signal: owner.controller.signal }); + try { + const response = await deadline.run(async () => { + owner.handoff(); + const upstream = await fetcher(outbound, deadline); + try { deadline.check(); owner.throwIfAborted(); } + catch (error) { void upstream.body?.cancel("generation_cancelled").catch(() => {}); throw error; } + return upstream; + }, owner); + return await deadlineResponse(response, deadline, owner, options); + } catch (error) { + deadline.stop(); + await owner.close("interrupted"); + if (error instanceof GenerationDeadlineExceeded) return error.response(); + if (error instanceof StreamPrefixUnavailable) return error.response(); + throw error; + } +} + +export function generationDeadlineHook(request, env, options = {}) { + // Construct before asynchronous context/setup work, after trusted authorization. + const deadline = createGenerationDeadline(request, env, options); + const owner = options.owner ?? new UpstreamCancellation({ signal: request.signal, onOutcome: options.onOutcome }); + return Object.freeze({ + enabled: () => deadline !== null, + deadline, owner, + authorize() { deadline?.start(owner); }, + admit() { deadline?.check(); }, + before_dispatch() { deadline?.check(); }, + observe() { deadline?.check(); }, + finalize() { deadline?.stop(); }, + fetch: fetcher => withGenerationDeadline(request, env, fetcher, { ...options, deadline, owner }), + }); +}