From 1ef4abfe80e79bacf21322014f15ce53fd0f9d71 Mon Sep 17 00:00:00 2001 From: Abhyudaya Baiju Tharakan Date: Fri, 9 Oct 2026 10:39:31 +0530 Subject: [PATCH] feat(routing): add opt-in bounded hedged auto requests --- docs/hedged-requests.md | 108 +++++++ routes/auto_routes.py | 9 + services/hedged_requests.py | 496 ++++++++++++++++++++++++++++++ services/managed_dispatch.py | 21 ++ tests/test_hedged_requests.py | 563 ++++++++++++++++++++++++++++++++++ worker/container-env.mjs | 2 + 6 files changed, 1199 insertions(+) create mode 100644 docs/hedged-requests.md create mode 100644 services/hedged_requests.py create mode 100644 tests/test_hedged_requests.py diff --git a/docs/hedged-requests.md b/docs/hedged-requests.md new file mode 100644 index 00000000..fcc2c0ad --- /dev/null +++ b/docs/hedged-requests.md @@ -0,0 +1,108 @@ +# Bounded hedged auto requests + +Hedging can race the first two candidates of a managed auto route for a pure, +non-streaming `POST /v1/chat/completions` request. The second candidate starts +after a delay if the first has not returned a useful, complete response. The +first validated response wins. There are at most two attempts in total. + +## Configuration + +Hedging is off by default. Disabling it keeps the existing auto-route response, +headers, accounting and storage behavior: + +```text +HEDGED_REQUESTS_ENABLED=false +HEDGED_REQUESTS_POLICY_JSON={} +``` + +Enable only routes whose providers support duplicate-safe generation or whose +pure-generation duplication contract has been reviewed. `idempotent_safe` is +an operator assertion; a request key alone does not establish provider safety. + +```text +HEDGED_REQUESTS_ENABLED=true +HEDGED_REQUESTS_POLICY_JSON={"auto:fast":{"enabled":true,"delay_ms":150,"max_duplicates":2,"idempotent_safe":true}} +MANAGED_IDEMPOTENCY_ENABLED=true +USAGE_RESERVATIONS_ENABLED=true +``` + +Configure the existing managed idempotency and reservation authorities, caller +budget caps, candidate pricing, and shared admission capacity as described in +[managed idempotency](managed-idempotency.md), +[usage reservations](usage-reservations.md) and +[admission leases](admission-leases.md). The Flask Container must receive both +`HEDGED_REQUESTS_ENABLED` and `HEDGED_REQUESTS_POLICY_JSON` when used behind the +Worker. This feature uses managed Flask dispatch; native Worker provider calls +and raw passthrough routes keep their existing behavior. + +Each policy may contain only `enabled`, `delay_ms`, `max_duplicates` and +`idempotent_safe`. Flags must be JSON booleans. The delay must be an integer +from 1 through 1000 milliseconds; its default is 150. `max_duplicates` must be +the integer 2, meaning the original plus one duplicate. Both flags default to +false. Unknown fields, duplicate fields, invalid route names or an invalid +entry disable the whole policy map. Empty settings use the defaults. Invalid +settings produce one warning per setting without logging its value. + +## Eligible requests + +An enabled route must have `idempotent_safe: true`. Send a stable +`Idempotency-Key` through managed idempotency, with usage reservations enabled, +known prices and enough caller budget for both conservative costs: + +```http +POST /v1/chat/completions +Content-Type: application/json +Idempotency-Key: generation-123 + +{"model":"auto:fast","messages":[{"role":"user","content":"Summarize this text."}],"max_tokens":256} +``` + +The request must specify a positive integer `max_tokens` or +`max_completion_tokens` so both output costs can be bounded. It must request +one completion, omit `tools`, `tool_choice`, `functions` and `function_call` +entirely, and omit streaming. Tool repair and context paging also keep the +existing single-call path. Without these requirements, a caller cap or an +active managed idempotency claim, the existing route loop handles the request. + +An otherwise eligible enabled policy with an unknown candidate price returns +HTTP 503 with `unpriced_reservation`. If both holds cannot fit the caller's +caps, the existing single-call path applies its normal budget policy. A durable +reservation or dispatch-boundary failure prevents submission and returns 503. + +## Timing, completion and cleanup + +Both conservative holds exist before the first submission. The duplicate starts +only after the configured delay from the first submission, while the shared +generation deadline has room and a second admission lease is immediately +available. Admission acquisition is attempted once; there is no queue or retry. +A denied lease, exhausted deadline window or completed first response suppresses +the duplicate and releases its unused hold. + +The winner must have HTTP 200, a valid complete JSON envelope, useful content in +every choice, and no tool or function calls. Validation buffers at most 1 MiB. +Empty, truncated or incomplete responses cannot win; a successful HTTP status +without valid completion becomes a 502 error if neither candidate succeeds. +Provider errors retain their normal error response. Caller cancellation and the +shared deadline stop both attempts through the existing cancellation transport. + +Each race uses at most two request-local threads. Cleanup joins the started +threads, closes cancelled transports through their existing cancellation owner, +and releases admission leases. A transport blocked before a closable response is +available must return within its configured timeout; Python threads cannot +forcibly interrupt arbitrary blocking provider code. No third candidate or new +provider retry is permitted after entering the race. + +## Cost and retention + +Every dispatched attempt has its own durable reservation and accounting row. +Measured usage settles both candidates, including the loser. An unknown outcome +retains its hold for the existing reservation review and reconciliation process; +cancelling a request does not prove that the provider avoided billing. A duplicate +that never dispatched releases its hold without recording provider usage. + +Managed idempotency owns the caller's response. A successful race can replay +only when the existing retention policy permits response content. Failed or +ambiguous races remain blocked under that policy rather than granting another +generation. Existing zero-content retention rules still apply. Hedging adds no +prompt or response logs, schema migration, public route or response header; the +existing auto-route headers identify the selected model and attempt count. diff --git a/routes/auto_routes.py b/routes/auto_routes.py index 7e423dad..f5f1bd6a 100644 --- a/routes/auto_routes.py +++ b/routes/auto_routes.py @@ -182,6 +182,15 @@ def dispatch_auto_route( if canary is not None: canary.observe(order.candidates) priorities = {candidate: index for index, candidate in enumerate(route.candidates)} + if fail_over is chat_fail_over: + from services.hedged_requests import dispatch_hedged_auto + hedged = dispatch_hedged_auto(payload, route.id, order.candidates, + validate=validate_candidate, dispatch=dispatch_candidate, + decision=lambda position: _route_decision(priorities[order.candidates[position]], position)) + if hedged is not None: + return _decorate_response(hedged.response, route, hedged.model, priorities[hedged.model], + hedged.attempts, route_decision=_route_decision(priorities[hedged.model], hedged.position), + ordering=order.mode, failures=hedged.failures, canary=canary) attempts = 0 failures: list[tuple[str, str]] = [] last_failure: Response | None = None diff --git a/services/hedged_requests.py b/services/hedged_requests.py new file mode 100644 index 00000000..d96440a2 --- /dev/null +++ b/services/hedged_requests.py @@ -0,0 +1,496 @@ +"""Delayed, two-attempt auto dispatch with separate durable usage ownership.""" +from __future__ import annotations + +import json +import logging +import math +import os +import re +import threading +import time +import uuid +from dataclasses import dataclass, replace + +from flask import Response, copy_current_request_context, current_app, g, has_request_context, request + +from error_handlers import APIError +from services import request_accounting as accounting, reservation_store +from services.budget_service import BudgetService, budgeted +from services.generation_deadline import Deadline, GenerationDeadlineExceeded, current_deadline +from services.idempotency_store import idempotency_enabled +from services.request_cancellation import CancellationContext, RequestCancellation, bind_cancellation + +logger = logging.getLogger(__name__) +_warned: set[str] = set() +_warning_lock = threading.Lock() +_FIELDS = {"enabled", "delay_ms", "max_duplicates", "idempotent_safe"} + + +def _warn(name): + with _warning_lock: + if name in _warned: + return + _warned.add(name) + logger.warning("Invalid %s; hedged requests disabled", name) + + +@dataclass(frozen=True) +class HedgePolicy: + enabled: bool = False + delay_ms: int = 150 + max_duplicates: int = 2 + idempotent_safe: bool = False + + +def _unique_object(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError("Duplicate field") + result[key] = value + return result + + +def route_policy(route_id): + flag = os.environ.get("HEDGED_REQUESTS_ENABLED", "").strip().lower() + if flag in {"", "0", "false", "no", "off"}: + return None + if flag not in {"1", "true", "yes", "on"}: + _warn("HEDGED_REQUESTS_ENABLED") + return None + try: + raw = os.environ.get("HEDGED_REQUESTS_POLICY_JSON", "").strip() or "{}" + if len(raw) > 65536: + raise ValueError("Policy too large") + document = json.loads(raw, object_pairs_hook=_unique_object) + if not isinstance(document, dict) or len(document) > 256: + raise ValueError("Invalid policy map") + policies = {} + for name, value in document.items(): + if (not re.fullmatch(r"auto:[A-Za-z0-9][A-Za-z0-9._:/+-]{0,255}", name) + or not isinstance(value, dict) or set(value) - _FIELDS): + raise ValueError("Invalid route policy") + policy = HedgePolicy(**value) + if (type(policy.enabled) is not bool or type(policy.idempotent_safe) is not bool + or type(policy.delay_ms) is not int or not 1 <= policy.delay_ms <= 1000 + or type(policy.max_duplicates) is not int or policy.max_duplicates != 2): + raise ValueError("Invalid bounds") + policies[name] = policy + policy = policies.get(route_id) + return policy if policy and policy.enabled and policy.idempotent_safe else None + except (ValueError, TypeError, RecursionError): + _warn("HEDGED_REQUESTS_POLICY_JSON") + return None + + +@dataclass(frozen=True) +class HedgeResult: + response: Response + model: str + position: int + attempts: int + failures: list[tuple[str, str]] + + +def _eligible(payload): + if not has_request_context() or request.method != "POST" or request.path != "/v1/chat/completions": + return False + if (payload.get("stream") or any(name in payload for name in ("tools", "tool_choice", "functions", "function_call")) + or not request.headers.get("Idempotency-Key") or not idempotency_enabled() + or getattr(g, "managed_idempotency_claim", None) is None + or not getattr(g, "managed_idempotency_handed_off", False) or not reservation_store.enabled() + or request.headers.get("X-MultiLLM-Tool-Repair")): + return False + # A missing output bound cannot conservatively reserve a provider generation. + output_limit = payload.get("max_completion_tokens", payload.get("max_tokens")) + if type(output_limit) is not int or output_limit <= 0 or payload.get("n", 1) != 1: + return False + from services.managed_turn import current_turn + turn = current_turn() + if turn is not None and turn.paging is not None: + return False + context = getattr(g, "usage_context", None) + user = getattr(g, "authenticated_user", None) or {} + return isinstance(context, accounting.UsageContext) and bool(context.reservation) and budgeted(user) + + +def _expires_at(models, clock): + deadline = current_deadline() + now = clock() + values = [deadline.expires_at] if deadline else [] + for name in ("cascade_deadline", "gateway_generation_deadline"): + value = getattr(g, name, None) + if type(value) in (int, float) and math.isfinite(value): + values.append(value) + timeouts = current_app.config.get("API_TIMEOUTS", {}) + for model in models: + timeout = timeouts.get(model.split(":", 1)[0], timeouts.get("default", (5, 60))) + seconds = sum(timeout) if isinstance(timeout, (tuple, list)) else timeout + values.append(now + min(300, max(0.001, float(seconds)))) + return min(values) + + +def _prepare_holds(models, payload): + outer = g.usage_context + output_tokens = payload.get("max_completion_tokens", payload.get("max_tokens")) + costs = [accounting.reservation_price([model], outer.input_tokens, output_tokens, 1) for model in models] + if any(cost is None for cost in costs): + raise APIError("Configure both candidate prices before hedging this route.", 503, + {"error": "unpriced_reservation"}) + try: + held = reservation_store.get_store().get(outer.reservation) + except reservation_store.ReservationError as error: + raise APIError("The durable usage reservation is unavailable.", 503, + {"error": error.code}) from error + if held["state"] != "reserved" or held["estimate_usd"] < costs[0]: + return None + decision = BudgetService.check_and_reserve(g.authenticated_user, costs[1]) + if not decision.allowed: + if decision.error == "budget_exceeded": + return None + raise APIError(decision.message, decision.status_code, {"error": decision.error}) + if decision.reservation is None: + return None + return [outer.reservation, decision.reservation] + + +def _useful_response(response): + from services.managed_turn import completed_envelope + from services.stream_preflight import _InvalidStream, _delta_useful + if response.status_code != 200 or response.is_streamed or response.mimetype != "application/json": + return False + raw = response.get_data() + if len(raw) > 1024 * 1024: + return False + try: + body = json.loads(raw) + if not completed_envelope(body): + return False + choices = body.get("choices") + return isinstance(choices, list) and bool(choices) and all( + _delta_useful(choice["message"]) + and not choice["message"].get("tool_calls") and not choice["message"].get("function_call") + for choice in choices) + except (ValueError, TypeError, KeyError, UnicodeError, RecursionError, _InvalidStream): + return False + + +class _Attempt: + def __init__(self, model, position, hold, outer, expires, clock, changed): + self.model, self.position = model, position + self.context = replace(outer, models=[model], selected=model, reservation=hold, + finished=False, ambiguous=False, output_tokens=outer.output_tokens) + self.owner = RequestCancellation() + self.deadline = Deadline(expires, clock) + self.stopped = threading.Event() + self.decision = threading.Event() + self.bridge = CancellationContext(self.cancel) + self.changed = changed + self.submissions = 0 + self.started = False + self.dispatched_at = None + self.accepted = False + self.response = None + self.error = None + self.usage = None + self.upstreams = [] + self.useful = False + self.reason = "interrupted" + self.lease = None + self.thread = None + + def cancel(self, error=None): + self.stopped.set() + self.owner.cancel() + with self.changed: + self.changed.notify_all() + + def before_submission(self): + if self.submissions or self.stopped.is_set(): + raise APIError("A hedged attempt cannot submit another generation.", 503, + {"error": "hedged_submission_exhausted"}) + self.deadline.check() + if self.lease is not None: + self.lease.check() + self._dispatch_started() + self.submissions += 1 + + def _dispatch_started(self): + try: + BudgetService.mark_dispatched(self.context.reservation) + except reservation_store.ReservationError as error: + raise APIError("The durable dispatch boundary is unavailable.", 503, + {"error": error.code}) from error + self.bridge.handoff() + with self.changed: + self.dispatched_at = self.deadline.clock() + self.changed.notify_all() + + def capture(self, response): + self.upstreams.append(response) + bind_cancellation(response) + self.capture_buffered_usage(response) + + def _buffer(self, response): + if not response.is_streamed or response.mimetype != "application/json": + return response + bind_cancellation(response) + parts, size = [], 0 + for chunk in response.iter_encoded(): + self.deadline.check() + if self.stopped.is_set(): + raise APIError("The caller cancelled the request.", 499) + size += len(chunk) + if size > 1024 * 1024: + response.close() + raise APIError("Hedged response exceeds the validation limit.", 502, + {"error": "upstream_response_invalid"}) + parts.append(chunk) + response.set_data(b"".join(parts)) + return response + + def _observe(self, response): + self.response = response + response = self._buffer(response) + self.useful = _useful_response(response) + self.reason = "ok" if self.useful else "invalid_response" if response.status_code < 400 else f"http_{response.status_code}" + self.usage = accounting._usage_from(accounting._json_body(response)) or self.usage + + def _settle(self): + for upstream in self.upstreams: + self.capture_buffered_usage(upstream) + # Cancellation is an uncertain outcome, not evidence of zero provider cost. + self.context.ambiguous = self.usage is None and (self.submissions > 0 or self.bridge.handed_off) + if not self.bridge.handed_off and not self.submissions: + BudgetService.settle(self.context.reservation, before_dispatch=True) + self.context.finished = True + return + status = self.response.status_code if self.response is not None else getattr(self.error, "status_code", 502) + accounting._record(self.context, status, self.usage, None) + + def capture_buffered_usage(self, upstream): + raw = getattr(upstream, "_content", None) + if isinstance(raw, bytes) and len(raw) <= 1024 * 1024: + try: + self.usage = accounting._usage_from(json.loads(raw)) or self.usage + except (ValueError, RecursionError): + pass + + def run(self, dispatch, body, decision, snapshot, turn, affinity, finished): + from services.managed_dispatch import isolated_managed_attempt + from services.prompt_cache_affinity import activate_scope + from services.route_health import RouteHealth + started = time.monotonic() + g.__dict__.update(snapshot) + g.usage_context = self.context + g.gateway_cancellation = self.owner + g.generation_deadline = self.deadline + g.gateway_generation_deadline = g.cascade_deadline = self.deadline.expires_at + # Worker teardown must not finalize the parent claim or release its lease. + # The secondary lease is checked and released by this attempt alone. + g.managed_idempotency_claim = None + g.gateway_admission_lease = None + g.hedged_attempt = self + with isolated_managed_attempt(turn) as local_turn, activate_scope(affinity): + try: + self.deadline.check() + if self.stopped.is_set(): + raise APIError("The caller cancelled the request.", 499) + if self.lease is not None: + self.lease.check() + # Managed dispatch signals the actual provider boundary. Injected + # candidate dispatchers without a managed turn own that boundary. + if local_turn is None: + self._dispatch_started() + self._observe(dispatch(body, self.model, decision)) + self.deadline.check() + self.useful = self.useful and not self.stopped.is_set() + except BaseException as error: + self.error, self.useful = error, False + self.reason = "interrupted" + finally: + try: + RouteHealth.record(self.model, ok=self.useful, latency_ms=(time.monotonic() - started) * 1000, + outcome=self.reason) + finally: + finished(self) + # Keep finalizers inside this attempt's request context, after selection. + self.decision.wait() + try: + for finalize in local_turn.finalizers if local_turn is not None else (): + finalize(self.accepted) + finally: + if self.accepted: + for context in tuple(self.owner.contexts): + context.complete() + else: + self.owner.cancel() + if self.response is not None: + self.response.close() + self.deadline.stop() + try: + self._settle() + finally: + if self.lease is not None: + self.lease.release() + + +class _Race: + def __init__(self, models, payload, holds, expires, clock): + self.clock, self.expires = clock, expires + self.changed = threading.Condition() + self.finished = [] + self.snapshot = dict(g.__dict__) + self.payload = payload + self.app = current_app._get_current_object() + self.parent_owner = getattr(g, "gateway_cancellation", None) + if self.parent_owner is None: + self.parent_owner = g.gateway_cancellation = RequestCancellation() + from services.managed_turn import current_turn + from services.prompt_cache_affinity import current_scope + self.turn, self.affinity = current_turn(), current_scope() + self.original_lease = getattr(g, "gateway_admission_lease", None) + self.attempts = [_Attempt(model, i, holds[i], g.usage_context, expires, clock, self.changed) + for i, model in enumerate(models)] + # The registrar remains the primary lease's sole release owner. + for attempt in self.attempts: + self.parent_owner.bind(attempt.bridge) + + def _finished(self, attempt): + with self.changed: + self.finished.append(attempt) + self.changed.notify_all() + + def _start(self, attempt, dispatch, decision): + @copy_current_request_context + def work(): + attempt.run(dispatch, {**self.payload, "model": attempt.model}, decision, + self.snapshot, self.turn, self.affinity, self._finished) + attempt.thread = threading.Thread(target=work, name=f"hedged-auto-{attempt.position}") + attempt.thread.start() + attempt.started = True + + def _admit_second(self): + from services.admission_leases import AdmissionError, AdmissionIdentity, principal_hash + from services.admission_leases import model_group + client = self.app.extensions.get("admission_client") + if client is None or self.clock() >= self.expires: + return False + lease = self.original_lease + identity = getattr(getattr(lease, "lease", lease), "identity", None) + group = identity.model_group if identity is not None else model_group(self.payload["model"]) + user = self.snapshot.get("authenticated_user") or {} + deadline_ms = math.floor((time.time() + self.expires - self.clock()) * 1000) + identity = AdmissionIdentity(principal_hash(user["username"]), group, uuid.uuid4().hex, deadline_ms) + second = self.attempts[1] + try: + second.lease = client.acquire(identity, on_lost=second.cancel) + if second.lease is None: + return False + second.lease.check() + return self.clock() < self.expires and not self.parent_owner.lost and not second.stopped.is_set() + except AdmissionError: + return False + + def _select(self, policy, dispatch, decision): + self._start(self.attempts[0], dispatch, decision(0)) + considered_second = False + while True: + with self.changed: + if self.parent_owner.lost: + raise APIError("The caller cancelled the request.", 499, {"error": "cancelled"}) + if self.clock() >= self.expires: + raise GenerationDeadlineExceeded() + winner = next((attempt for attempt in self.finished if attempt.useful), None) + if winner is not None: + return winner + started = self.attempts[0].dispatched_at + if started is None and self.attempts[0] in self.finished: + return self.attempts[0] + due = started + policy.delay_ms / 1000 if started is not None else None + if due is not None and due >= self.expires: + considered_second = True + if considered_second and len(self.finished) == sum(attempt.started for attempt in self.attempts): + return self.finished[-1] + remaining = self.expires - self.clock() + if not considered_second and due is not None and self.clock() >= due: + considered_second = True + else: + self.changed.wait(min(remaining, max(0, due - self.clock())) + if not considered_second and due is not None else remaining) + continue + # Shared admission decides immediately; never enqueue or retry acquisition. + if self._admit_second(): + with self.changed: + complete = any(attempt.useful for attempt in self.finished) + if not complete: + self._start(self.attempts[1], dispatch, decision(1)) + + def execute(self, policy, dispatch, decision): + selected = None + try: + if self.original_lease is not None: + self.original_lease.check() + selected = self._select(policy, dispatch, decision) + if self.original_lease is not None: + self.original_lease.check() + if self.parent_owner.lost: + raise APIError("The caller cancelled the request.", 499, {"error": "cancelled"}) + if self.clock() >= self.expires: + raise GenerationDeadlineExceeded() + selected.accepted = selected.useful + finally: + for attempt in self.attempts: + if not attempt.accepted: + attempt.bridge.cancel() + attempt.decision.set() + for attempt in self.attempts: + if attempt.thread is not None and attempt.started: + attempt.thread.join() + elif not attempt.started: + BudgetService.settle(attempt.context.reservation, before_dispatch=True) + if attempt.lease is not None: + attempt.lease.release() + if selected.error is not None: + raise selected.error + if selected.response is None: + raise APIError("Hedged candidates did not return a complete response.", 502, + {"error": "upstream_response_invalid"}) + if selected.response.status_code < 400 and not selected.useful: + selected.response = Response(json.dumps({"error": {"code": "upstream_response_invalid", + "message": "Hedged candidates did not return a useful complete response."}}), + status=502, content_type="application/json") + failures = [(attempt.model, attempt.reason) for attempt in self.attempts + if attempt.started and attempt is not selected and not attempt.useful] + return HedgeResult(selected.response, selected.model, selected.position, + sum(attempt.started for attempt in self.attempts), failures) + + +def dispatch_hedged_auto(payload, route_id, candidates, *, validate, dispatch, decision): + """None leaves the existing route loop, response and accounting untouched.""" + policy = route_policy(route_id) + if policy is None or not _eligible(payload) or len(candidates) < 2: + return None + from services import key_controls + from services.model_cooldown import ModelCooldownCapacity, ModelCooldownExhausted + models = list(candidates[:2]) + for model in models: + if not key_controls.model_allowed(g.authenticated_user, model): + return None + try: + validate(model) + except (ModelCooldownExhausted, ModelCooldownCapacity, GenerationDeadlineExceeded): + raise + except (APIError, ValueError): + return None + deadline = current_deadline() + clock = deadline.clock if deadline else time.monotonic + expires = _expires_at(models, clock) + if expires <= clock(): + raise GenerationDeadlineExceeded() + holds = _prepare_holds(models, payload) + if holds is None: + return None + # Per-attempt finalizers now own both holds; suppress the aggregate ledger row. + g.usage_context.finished = True + return _Race(models, payload, holds, expires, clock).execute(policy, dispatch, decision) diff --git a/services/managed_dispatch.py b/services/managed_dispatch.py index d48426a8..595ba941 100644 --- a/services/managed_dispatch.py +++ b/services/managed_dispatch.py @@ -3,6 +3,8 @@ import math import json import time +from contextlib import contextmanager +from dataclasses import replace from typing import Callable import requests @@ -17,10 +19,25 @@ Outcome = Callable[[], UpstreamOutcome] +@contextmanager +def isolated_managed_attempt(turn): + """Keep parallel attempt finalizers out of the parent managed turn.""" + from services.managed_turn import _turn + local = replace(turn, finalizers=[], pages=list(turn.pages), paged_candidates=dict(turn.paged_candidates)) if turn else None + token = _turn.set(local) + try: + yield local + finally: + _turn.reset(token) + + def execute_managed_attempt(send, model, credential): """Capture the actual dispatch credential; headers alone never bind affinity.""" from services.prompt_cache_affinity import current_scope from services.managed_turn import before_provider_submission + hedged = getattr(g, "hedged_attempt", None) if has_request_context() else None + if hedged is not None: + hedged.before_submission() before_provider_submission() scope = current_scope() try: @@ -29,6 +46,8 @@ def execute_managed_attempt(send, model, credential): if scope is not None: scope.observer(model, credential)(classify_upstream_outcome(transport_failure="interrupted")) raise + if hedged is not None: + hedged.capture(response) from services.pii_redaction import current_context context = current_context() if context is not None: @@ -219,6 +238,8 @@ def retry_managed_attempt( available; absent one, per-attempt timeouts remain the transport's policy. None means that the original response/error must finish normally. """ + if has_request_context() and getattr(g, "hedged_attempt", None) is not None: + return None classified = outcome() if not classified.replay_permission or retry_count >= max_retries: return None diff --git a/tests/test_hedged_requests.py b/tests/test_hedged_requests.py new file mode 100644 index 00000000..3e4a4ee0 --- /dev/null +++ b/tests/test_hedged_requests.py @@ -0,0 +1,563 @@ +"""Bounded auto races with synthetic transport, admission and durable holds.""" +import importlib +import json +import os +import threading +import time +from unittest.mock import patch + +import pytest +import requests +from flask import Flask, Response, g + +MODELS = ("mimo:first", "opencode:second", "mimo:third") +PAYLOAD = {"model": "auto:hedge", "messages": [{"role": "user", "content": "test"}], "max_tokens": 100} +USER = {"username": "caller", "daily_budget_usd": 5.0, "monthly_budget_usd": 10.0} + + +def envelope(text="ok", *, tokens=True, finish="stop"): + body = {"choices": [{"message": {"role": "assistant", "content": text}, "finish_reason": finish}]} + if tokens: + body["usage"] = {"prompt_tokens": 10, "completion_tokens": 20} + return body + + +class Lease: + def __init__(self, identity=None): + self.identity = identity + self.releases = 0 + self.closed = False + + def check(self): + assert not self.closed + + def release(self): + if not self.closed: + self.closed = True + self.releases += 1 + + +class Admission: + def __init__(self, *, denied=False): + self.denied = denied + self.leases = [] + self.calls = [] + + def acquire(self, identity, *, on_lost=None): + self.calls.append(identity) + if self.denied: + from services.admission_leases import AdmissionError + raise AdmissionError() + lease = Lease(identity) + self.leases.append(lease) + return lease + + +@pytest.fixture(autouse=True) +def isolated(monkeypatch, tmp_path): + global hedges, routes, accounting, reservations, budget + hedges = importlib.import_module("services.hedged_requests") + routes = importlib.import_module("routes.auto_routes") + accounting = importlib.import_module("services.request_accounting") + reservations = importlib.import_module("services.reservation_store") + budget = importlib.import_module("services.budget_service") + flags = {"HEDGED_REQUESTS_ENABLED": "true", "MANAGED_IDEMPOTENCY_ENABLED": "true", + "USAGE_RESERVATIONS_ENABLED": "true", "USAGE_HOLD_REVIEW_AFTER_SECONDS": "259200", + "INTELLIGENCE_STORAGE_BACKEND": "", "CONTROL_PLANE_DATABASE_URL": "", + "USAGE_LEDGER_ENABLED": "false", "PROMPT_CACHE_USAGE_BUCKETS_ENABLED": "false", + "PROMPT_CACHE_AFFINITY_ENABLED": "false", "CANARY_TRAFFIC_ENABLED": "false", + "ADMISSION_ENABLED": "false", "MULTILLM_STREAM_PREFLIGHT": "off", "NANOGPT_API_KEY": "", + "MODEL_REGISTRY_DB_PATH": str(tmp_path / "models.sqlite3"), "SESSION_TIER_MODE": "off"} + for name, value in flags.items(): + monkeypatch.setenv(name, value) + policy(monkeypatch) + monkeypatch.setenv("MODEL_PRICING_USD_PER_MILLION", json.dumps( + {model: {"input": 1000, "output": 1000} for model in MODELS})) + monkeypatch.setattr(hedges, "_warned", set()) + store = reservations.SqlReservationStore(tmp_path / "usage.sqlite3") + monkeypatch.setattr(reservations, "_store", store) + rows = [] + monkeypatch.setattr(accounting.usage_ledger.LEDGER, "record", rows.append) + monkeypatch.setattr(accounting.telemetry_export.EXPORTER, "submit", lambda row: None) + budget.BudgetService.reset() + routes.RouteHealth.reset() + yield store, rows + budget.BudgetService.reset() + routes.RouteHealth.reset() + + +def policy(monkeypatch, **changes): + config = {"enabled": True, "delay_ms": 15, "max_duplicates": 2, "idempotent_safe": True} + config.update(changes) + monkeypatch.setenv("HEDGED_REQUESTS_POLICY_JSON", json.dumps({"auto:hedge": config})) + + +def application(): + app = Flask(__name__) + app.config.update(TESTING=True, API_TIMEOUTS={"default": (0.5, 0.5)}) + from services.auto_route_service import AutoRoute + route = AutoRoute(id="auto:hedge", candidates=MODELS, updated_at="") + return app, route + + +def run(monkeypatch, send, *, body=None, headers=None, prepare=None, admission=None): + app, route = application() + app.extensions["admission_client"] = admission or Admission() + monkeypatch.setattr(routes.AutoRouteService, "get_route", lambda model: route) + payload = {**PAYLOAD, **(body or {})} + from services.admission_leases import AdmissionIdentity, principal_hash + with app.test_request_context("/v1/chat/completions", method="POST", json=payload, + headers={"Idempotency-Key": "stable", **(headers or {})}): + g.authenticated_user = dict(USER) + g.managed_idempotency_claim = object() + g.managed_idempotency_handed_off = True + g.gateway_generation_deadline = time.monotonic() + 1 + g.gateway_admission_lease = Lease(AdmissionIdentity(principal_hash("caller"), "auto:hedge", "original", int(time.time() * 1000) + 1000)) + assert accounting.begin() is None + context = g.usage_context + if prepare: + prepare() + response = routes.dispatch_auto_route_chat_completion(payload, validate_candidate=lambda model: None, + dispatch_candidate=send) + accounting.finish(response) + return response, context, app.extensions["admission_client"] + + +@pytest.mark.parametrize("flag", ["", "false", "0", "off", "malformed-private-value"]) +def test_disabled_exact_bytes_headers_accounting_storage(monkeypatch, isolated, caplog, flag): + store, rows = isolated + monkeypatch.setenv("HEDGED_REQUESTS_ENABLED", flag) + calls = [] + def send(body, model, decision): + calls.append((body, model, decision)) + return Response(json.dumps(envelope()), content_type="application/json", headers={"X-Provider": "same"}) + original, _, _ = run(monkeypatch, send) + counts = len(rows), store.summary("caller")["spent_today_usd"] + monkeypatch.setenv("HEDGED_REQUESTS_POLICY_JSON", "not-json-private-value") + second, _, _ = run(monkeypatch, send) + assert original.data == second.data and list(original.headers) == list(second.headers) + assert calls[0] == calls[1] and len(calls) == 2 + assert counts == (1, pytest.approx(0.03)) and len(rows) == 2 + assert store.summary("caller")["spent_today_usd"] == pytest.approx(0.06) + assert all(row["selected_model"] == MODELS[0] for row in rows) + assert "malformed-private-value" not in caplog.text and "not-json-private-value" not in caplog.text + assert sum("Invalid HEDGED" in record.message for record in caplog.records) == int(flag == "malformed-private-value") + + +@pytest.mark.parametrize("change", [{"extra": True}, {"delay_ms": 0}, {"delay_ms": 1001}, {"delay_ms": True}, + {"delay_ms": 1.5}, {"max_duplicates": 3}, {"max_duplicates": True}, {"max_duplicates": 2.0}, + {"enabled": "true"}, {"idempotent_safe": 1}]) +def test_invalid_policy_disables_whole_mapping_once(monkeypatch, caplog, change): + policy(monkeypatch, **change) + for _ in range(2): + assert hedges.route_policy("auto:hedge") is None + assert sum("Invalid HEDGED" in record.message for record in caplog.records) == 1 + + +@pytest.mark.parametrize("raw", ["[]", "null", "private-value", '{"direct:model":{}}', '{"auto:hedge":false}']) +def test_invalid_policy_object_and_route(monkeypatch, raw, caplog): + monkeypatch.setenv("HEDGED_REQUESTS_POLICY_JSON", raw) + assert hedges.route_policy("auto:hedge") is None + assert "private-value" not in caplog.text + + +def test_empty_policy_and_flags_use_defaults(monkeypatch): + monkeypatch.setenv("HEDGED_REQUESTS_POLICY_JSON", "") + assert hedges.route_policy("auto:hedge") is None + monkeypatch.setenv("HEDGED_REQUESTS_POLICY_JSON", '{"auto:hedge":{"enabled":true,"idempotent_safe":true}}') + settings = hedges.route_policy("auto:hedge") + assert settings.delay_ms == 150 and settings.max_duplicates == 2 + + +@pytest.mark.parametrize("body,prepare", [({"stream": True}, None), ({"tools": []}, None), + ({"tool_choice": None}, None), ({"functions": []}, None), ({"function_call": None}, None), + ({"max_tokens": None}, None), ({}, lambda: setattr(g, "managed_idempotency_claim", None)), + ({}, lambda: setattr(g, "managed_idempotency_handed_off", False)), + ({}, lambda: setattr(g, "authenticated_user", {"username": "unlimited"}))]) +def test_ineligible_requests_keep_single_path(monkeypatch, body, prepare): + calls = [] + response, _, admission = run(monkeypatch, lambda body, model, decision: ( + calls.append(model) or Response(json.dumps(envelope()), content_type="application/json")), body=body, prepare=prepare) + assert response.status_code == 200 and calls == [MODELS[0]] and not admission.calls + + +@pytest.mark.parametrize("flag", ["MANAGED_IDEMPOTENCY_ENABLED", "USAGE_RESERVATIONS_ENABLED"]) +def test_missing_enabled_dependency_keeps_single_path(monkeypatch, flag): + monkeypatch.setenv(flag, "false") + calls = [] + response, _, admission = run(monkeypatch, lambda body, model, decision: ( + calls.append(model) or Response(json.dumps(envelope()), content_type="application/json"))) + assert response.status_code == 200 and calls == [MODELS[0]] and not admission.calls + + +def test_unknown_price_rejected_only_for_otherwise_eligible_policy(monkeypatch): + monkeypatch.setenv("MODEL_PRICING_USD_PER_MILLION", json.dumps({MODELS[0]: {"input": 1000, "output": 1000}})) + app, route = application() + monkeypatch.setattr(routes.AutoRouteService, "get_route", lambda model: route) + with app.test_request_context("/v1/chat/completions", method="POST", json=PAYLOAD, headers={"Idempotency-Key": "stable"}): + g.authenticated_user = USER + g.managed_idempotency_claim = object() + g.managed_idempotency_handed_off = True + g.usage_context = accounting.UsageContext("chat", [PAYLOAD["model"]], None, USER, time.perf_counter(), time.time_ns(), input_tokens=10, output_tokens=100) + g.usage_context.reservation = budget.BudgetService.check_and_reserve(USER, 0.11).reservation + send = lambda *args: pytest.fail("Unpriced fanout") + with pytest.raises(routes.APIError) as raised: + routes.dispatch_auto_route_chat_completion(PAYLOAD, validate_candidate=lambda model: None, dispatch_candidate=send) + assert raised.value.status_code == 503 + policy(monkeypatch, idempotent_safe=False) + reply = routes.dispatch_auto_route_chat_completion(PAYLOAD, validate_candidate=lambda model: None, + dispatch_candidate=lambda *args: Response(json.dumps(envelope()), content_type="application/json")) + assert reply.status_code == 200 + + +def test_reserve_both_before_dispatch_fast_primary_releases_unused_hold(monkeypatch, isolated): + store, rows = isolated + def send(body, model, decision): + assert store.summary("caller")["reserved"] + store.summary("caller")["dispatched"] == 2 + return Response(json.dumps(envelope()), content_type="application/json") + response, outer, admission = run(monkeypatch, send) + assert response.headers["X-MultiLLM-Auto-Attempts"] == "1" and not admission.calls + assert outer.finished and len(rows) == 1 + assert store.summary("caller")["held_usd"] == 0 and store.summary("caller")["spent_today_usd"] == pytest.approx(0.03) + + +def test_secondary_wins_cancels_loser_once_retains_unknown_hold(monkeypatch, isolated): + store, rows = isolated + calls, closes = [], [] + from services.request_cancellation import bind_cancellation + def send(body, model, decision): + calls.append((model, time.monotonic())) + if model == MODELS[0]: + upstream = requests.Response() + upstream.status_code = 200 + upstream._content = b"" + owner = bind_cancellation(upstream, close_owner=lambda: closes.append(model)) + while not owner.closed: + time.sleep(0.001) + raise requests.ReadTimeout("synthetic interrupted provider") + return Response(json.dumps(envelope("winner")), content_type="application/json") + response, _, admission = run(monkeypatch, send) + assert response.get_json() == envelope("winner") and response.headers["X-MultiLLM-Auto-Selected-Model"] == MODELS[1] + assert len(calls) == 2 and calls[1][1] - calls[0][1] >= 0.015 + assert closes == [MODELS[0]] and admission.leases[0].releases == 1 + summary = store.summary("caller") + assert summary["unknown"] == 1 and summary["held_usd"] > 0 and summary["spent_today_usd"] == pytest.approx(0.03) + assert len(rows) == 2 and not any(thread.name.startswith("hedged-auto-") for thread in threading.enumerate()) + + +@pytest.mark.parametrize("first", [envelope("", finish="stop"), envelope("partial", finish="length"), + {"error": {"message": "provider failure"}}, {"choices": [{"message": {}, "finish_reason": "stop"}]}]) +def test_invalid_fast_response_cannot_win_or_start_third(monkeypatch, first): + calls = [] + def send(body, model, decision): + calls.append(model) + return Response(json.dumps(first if model == MODELS[0] else envelope("second")), content_type="application/json") + response, _, _ = run(monkeypatch, send) + assert calls == list(MODELS[:2]) and response.get_json() == envelope("second") + + +def test_both_measured_costs_survive_loser_close(monkeypatch, isolated): + store, rows = isolated + def send(body, model, decision): + return Response(json.dumps(envelope("bad", finish="length") if model == MODELS[0] else envelope()), content_type="application/json") + response, _, _ = run(monkeypatch, send) + assert response.status_code == 200 and len(rows) == 2 + summary = store.summary("caller") + assert summary["held_usd"] == 0 and summary["spent_today_usd"] == pytest.approx(0.06) + assert all(row["input_tokens"] == 10 and row["output_tokens"] == 20 for row in rows) + + +def test_ambiguous_failures_never_dispatch_third_or_add_retry_headers(monkeypatch): + calls = [] + def send(body, model, decision): + calls.append(model) + return Response('{"error":{"code":"interrupted"}}', status=503, content_type="application/json") + response, _, _ = run(monkeypatch, send) + assert calls == list(MODELS[:2]) and response.status_code == 503 + assert "Retry-After" not in response.headers and response.headers["X-MultiLLM-Auto-Attempts"] == "2" + + +def test_admission_denial_suppresses_second_and_releases_hold(monkeypatch, isolated): + calls = [] + def send(body, model, decision): + calls.append(model) + time.sleep(0.04) + return Response(json.dumps(envelope()), content_type="application/json") + response, _, admission = run(monkeypatch, send, admission=Admission(denied=True)) + assert calls == [MODELS[0]] and len(admission.calls) == 1 + assert response.headers["X-MultiLLM-Auto-Attempts"] == "1" and isolated[0].summary("caller")["held_usd"] == 0 + + +def test_insufficient_deadline_never_launches_duplicate(monkeypatch): + policy(monkeypatch, delay_ms=1000) + calls = [] + response, _, admission = run(monkeypatch, lambda body, model, decision: ( + calls.append(model) or Response(json.dumps(envelope()), content_type="application/json"))) + assert calls == [MODELS[0]] and not admission.calls and response.status_code == 200 + + +def test_caps_that_only_fit_original_keep_single_path(monkeypatch, isolated): + def prepare(): + g.authenticated_user["daily_budget_usd"] = 0.12 + calls = [] + response, _, admission = run(monkeypatch, lambda body, model, decision: ( + calls.append(model) or Response(json.dumps(envelope()), content_type="application/json")), prepare=prepare) + assert response.status_code == 200 and calls == [MODELS[0]] and not admission.calls + + +def test_managed_submission_guard_blocks_retry_and_additional_send(monkeypatch): + managed = importlib.import_module("services.managed_dispatch") + from services.upstream_outcome import classify_upstream_outcome + calls = [] + def send(body, model, decision): + def upstream(): + calls.append(model) + result = requests.Response() + result.status_code = 503 + result._content = b'{"error":{}}' + return result + managed.execute_managed_attempt(upstream, model, "synthetic-credential") + assert managed.retry_managed_attempt(lambda number: pytest.fail("extra retry"), + lambda: classify_upstream_outcome(429), retry_count=0, max_retries=2, retry_delay=0) is None + with pytest.raises(routes.APIError): + managed.execute_managed_attempt(upstream, model, "synthetic-credential") + return Response('{"error":{}}', status=503, content_type="application/json") + run(monkeypatch, send) + assert calls == list(MODELS[:2]) + + +def test_lazy_nonstreaming_json_is_validated_before_winning(monkeypatch, isolated): + def send(body, model, decision): + return Response(iter([json.dumps(envelope()).encode()]), content_type="application/json") + response, _, admission = run(monkeypatch, send) + assert response.status_code == 200 and response.get_json() == envelope() and not admission.calls + assert isolated[0].summary("caller")["spent_today_usd"] == pytest.approx(0.03) + + +def test_deadline_cancels_both_attempts_and_retains_both_holds(monkeypatch, isolated): + calls, closes = [], [] + from services.generation_deadline import Deadline, GenerationDeadlineExceeded + from services.request_cancellation import bind_cancellation + def send(body, model, decision): + calls.append(model) + upstream = requests.Response() + upstream.status_code = 200 + upstream._content = b"" + owner = bind_cancellation(upstream, close_owner=lambda: closes.append(model)) + while not owner.closed: + time.sleep(0.001) + raise requests.ReadTimeout("synthetic deadline") + with pytest.raises(GenerationDeadlineExceeded): + run(monkeypatch, send, prepare=lambda: setattr(g, "generation_deadline", Deadline(time.monotonic() + 0.04))) + assert calls == list(MODELS[:2]) and sorted(closes) == sorted(MODELS[:2]) + assert isolated[0].summary("caller")["unknown"] == 2 + assert not any(thread.name.startswith("hedged-auto-") for thread in threading.enumerate()) + + +def test_request_disconnect_cancels_both_without_retry(monkeypatch, isolated): + from services.request_cancellation import bind_cancellation + owner = [] + timer = [] + calls = [] + def prepare(): + from services.request_cancellation import RequestCancellation + g.gateway_cancellation = RequestCancellation() + owner.append(g.gateway_cancellation) + def send(body, model, decision): + calls.append(model) + upstream = requests.Response() + upstream.status_code = 200 + upstream._content = b"" + context = bind_cancellation(upstream, close_owner=lambda: None) + if model == MODELS[1]: + timer.append(threading.Timer(0.005, owner[0].cancel)) + timer[0].start() + while not context.closed: + time.sleep(0.001) + raise requests.ReadTimeout("synthetic disconnect") + try: + with pytest.raises(routes.APIError) as raised: + run(monkeypatch, send, prepare=prepare) + assert raised.value.status_code == 499 + finally: + for clock in timer: + clock.join() + assert calls == list(MODELS[:2]) and isolated[0].summary("caller")["unknown"] == 2 + + +def test_no_key_and_no_policy_never_race(monkeypatch): + calls = [] + response, _, admission = run(monkeypatch, lambda body, model, decision: ( + calls.append(model) or Response(json.dumps(envelope()), content_type="application/json")), headers={"Idempotency-Key": ""}) + assert response.status_code == 200 and calls == [MODELS[0]] and not admission.calls + + +def test_primary_can_win_after_second_starts_and_close_second_once(monkeypatch, isolated): + from services.request_cancellation import bind_cancellation + started = threading.Event() + closes = [] + def send(body, model, decision): + if model == MODELS[0]: + assert started.wait(0.5) + return Response(json.dumps(envelope()), content_type="application/json") + upstream = requests.Response() + upstream.status_code = 200 + upstream._content = b"" + owner = bind_cancellation(upstream, close_owner=lambda: closes.append(model)) + started.set() + while not owner.closed: + time.sleep(0.001) + raise requests.ReadTimeout("synthetic cancellation") + response, _, admission = run(monkeypatch, send) + assert response.headers["X-MultiLLM-Auto-Selected-Model"] == MODELS[0] and closes == [MODELS[1]] + assert admission.leases[0].releases == 1 and isolated[0].summary("caller")["unknown"] == 1 + + +def test_bad_unrelated_route_disables_policy_and_duplicate_fields_are_rejected(monkeypatch): + monkeypatch.setenv("HEDGED_REQUESTS_POLICY_JSON", json.dumps({"auto:hedge": { + "enabled": True, "idempotent_safe": True}, "auto:other": {"unexpected": True}})) + assert hedges.route_policy("auto:hedge") is None + monkeypatch.setenv("HEDGED_REQUESTS_POLICY_JSON", '{"auto:hedge":{"enabled":true,"enabled":false}}') + assert hedges.route_policy("auto:hedge") is None + + +def test_large_lazy_body_is_rejected_with_unknown_hold(monkeypatch, isolated): + def send(body, model, decision): + return Response(iter([b"x" * (1024 * 1024 + 1)]), content_type="application/json") + with pytest.raises(routes.APIError) as raised: + run(monkeypatch, send) + assert raised.value.status_code == 502 and isolated[0].summary("caller")["unknown"] == 2 + + +def test_primary_completion_during_admission_prevents_late_duplicate(monkeypatch, isolated): + primary = threading.Event() + class SlowAdmission(Admission): + def acquire(self, identity, *, on_lost=None): + lease = super().acquire(identity, on_lost=on_lost) + primary.set() + time.sleep(0.02) + return lease + calls = [] + def send(body, model, decision): + calls.append(model) + assert primary.wait(0.5) + return Response(json.dumps(envelope()), content_type="application/json") + response, _, admission = run(monkeypatch, send, admission=SlowAdmission()) + assert calls == [MODELS[0]] and response.status_code == 200 and admission.leases[0].releases == 1 + assert isolated[0].summary("caller")["held_usd"] == 0 + + +def test_delay_begins_at_provider_submission_after_managed_preparation(monkeypatch): + from services.managed_turn import ManagedTurn, _turn + from services.request_cancellation import bind_cancellation + managed = importlib.import_module("services.managed_dispatch") + submitted = [] + def send(body, model, decision): + if model == MODELS[0]: + time.sleep(0.035) + def upstream(): + submitted.append((model, time.monotonic())) + response = requests.Response() + response.status_code = 200 + response._content = json.dumps(envelope()).encode() + if model == MODELS[0]: + owner = bind_cancellation(response, close_owner=lambda: None) + while not owner.closed: + time.sleep(0.001) + raise requests.ReadTimeout("synthetic cancelled generation") + return response + raw = managed.execute_managed_attempt(upstream, model, "synthetic-credential") + return Response(raw.content, content_type="application/json") + token = _turn.set(ManagedTurn("chat")) + try: + response, _, _ = run(monkeypatch, send) + finally: + _turn.reset(token) + assert response.status_code == 200 and [model for model, at in submitted] == list(MODELS[:2]) + assert submitted[1][1] - submitted[0][1] >= 0.015 + + +def test_durable_dispatch_failure_stops_before_submission_and_releases_holds(monkeypatch, isolated): + calls = [] + def unavailable(reservation): + raise reservations.ReservationError() + monkeypatch.setattr(budget.BudgetService, "mark_dispatched", unavailable) + with pytest.raises(routes.APIError) as raised: + run(monkeypatch, lambda *args: calls.append(args)) + assert raised.value.status_code == 503 and not calls + assert isolated[0].summary("caller")["held_usd"] == 0 + assert not any(thread.name.startswith("hedged-auto-") for thread in threading.enumerate()) + + +def test_durable_hold_read_failure_is_controlled_before_dispatch(monkeypatch, isolated): + calls = [] + def prepare(): + def unavailable(identity): + raise reservations.ReservationError() + monkeypatch.setattr(isolated[0], "get", unavailable) + with pytest.raises(routes.APIError) as raised: + run(monkeypatch, lambda *args: calls.append(args), prepare=prepare) + assert raised.value.status_code == 503 and not calls + + +from tests.unified_api_test_case import UnifiedApiTestCase + + +class RegisteredHedgeTests(UnifiedApiTestCase): + def setUp(self): + with patch("config.load_runtime_env"): + super().setUp() + from services.auto_route_service import AutoRouteService + from services.idempotency_store import IdempotencyStore + from tests.test_managed_idempotency import Authority + self.authority = Authority() + self.app.extensions["managed_idempotency_store"] = IdempotencyStore(self.authority) + self.app.extensions["admission_client"] = Admission() + self.app.extensions["gateway_after_authentication"].append(lambda: g.authenticated_user.update(USER)) + self.models = ("mimo:mimo-v2.5", "opencode:glm-5.2", "mimo:mimo-v2.5-pro") + AutoRouteService.save_route("auto:hedge", list(self.models), self.app.config["API_BASE_URLS"]) + os.environ["MODEL_PRICING_USD_PER_MILLION"] = json.dumps( + {model: {"input": 1000, "output": 1000} for model in self.models}) + os.environ["CONTEXT_CACHE_SHARED_ENABLED"] = "false" + self.app.config["OUTPUT_SCHEMA_VALIDATION_ENABLED"] = False + self.headers = {"Authorization": "Bearer admin-test-key", "Idempotency-Key": "registered"} + + def test_registered_two_attempts_account_and_replay_without_new_dispatch(self): + from services.request_cancellation import bind_cancellation + sent, closes = [], [] + def send(**kwargs): + sent.append(kwargs["api_provider"]) + response = requests.Response() + response.status_code = 200 + response.headers["Content-Type"] = "application/json" + if kwargs["api_provider"] == "mimo": + response._content = b"" + owner = bind_cancellation(response, close_owner=lambda: closes.append("mimo")) + while not owner.closed: + time.sleep(0.001) + raise requests.ReadTimeout("synthetic interrupted loser") + response._content = json.dumps(envelope()).encode() + return response + with patch.object(self.app_module.ProxyService, "make_request", side_effect=send): + first = self.client.post("/v1/chat/completions", json=PAYLOAD, headers=self.headers) + replay = self.client.post("/v1/chat/completions", json=PAYLOAD, headers=self.headers) + assert first.status_code == replay.status_code == 200 + assert first.data == replay.data and replay.headers["X-MultiLLM-Idempotency"] == "replayed" + assert first.headers["X-MultiLLM-Auto-Selected-Model"] == self.models[1] + assert first.headers["X-MultiLLM-Auto-Attempts"] == "2" and sent == ["mimo", "opencode"] + assert closes == ["mimo"] + summary = reservations.get_store().summary("caller") + assert summary["unknown"] == 1 and summary["spent_today_usd"] == pytest.approx(0.03) + + def test_registered_ambiguous_race_blocks_replay_and_third_candidate(self): + def send(**kwargs): + response = requests.Response() + response.status_code = 503 + response.headers["Content-Type"] = "application/json" + response._content = b'{"error":{"code":"interrupted"}}' + return response + with patch.object(self.app_module.ProxyService, "make_request", side_effect=send) as upstream: + first = self.client.post("/v1/chat/completions", json=PAYLOAD, headers=self.headers) + retry = self.client.post("/v1/chat/completions", json=PAYLOAD, headers=self.headers) + assert first.status_code == 503 and retry.status_code == 409 and upstream.call_count == 2 + assert reservations.get_store().summary("caller")["unknown"] == 2 diff --git a/worker/container-env.mjs b/worker/container-env.mjs index a27d53fb..df278092 100644 --- a/worker/container-env.mjs +++ b/worker/container-env.mjs @@ -40,6 +40,8 @@ const DIRECT_ENV_KEYS = [ "CONNECTION_PROFILES_DB_PATH", "CONFIG_REVISION_SYNC_ENABLED", "CONFIG_SYNC_TTL_SECONDS", + "HEDGED_REQUESTS_ENABLED", + "HEDGED_REQUESTS_POLICY_JSON", "BANDIT_MODE", "BATCH_SPILLOVER_ENABLED", "CANARY_TRAFFIC_ENABLED",