From 6c672492c5ec90b7cf10c1871e0f19867e3cc5bf Mon Sep 17 00:00:00 2001 From: Abhyudaya Baiju Tharakan Date: Fri, 9 Oct 2026 10:18:54 +0530 Subject: [PATCH] feat(routing): add opt-in latency SLO admission --- docs/latency-slo.md | 110 +++++++ package.json | 2 +- services/gateway_extensions.py | 15 +- services/intelligence_policy.py | 7 +- services/latency_slo.py | 316 ++++++++++++++++++ services/route_health.py | 13 +- tests/fixtures/secret_firewall_coverage.json | 1 + tests/test_latency_slo.py | 317 +++++++++++++++++++ tests/test_latency_slo_worker.mjs | 135 ++++++++ tests/test_wave4_registrar.py | 2 +- tests/test_wave5_registrar.py | 2 +- worker/container-env.mjs | 4 + worker/latency-slo.mjs | 188 +++++++++++ 13 files changed, 1103 insertions(+), 9 deletions(-) create mode 100644 docs/latency-slo.md create mode 100644 services/latency_slo.py create mode 100644 tests/test_latency_slo.py create mode 100644 tests/test_latency_slo_worker.mjs create mode 100644 worker/latency-slo.mjs diff --git a/docs/latency-slo.md b/docs/latency-slo.md new file mode 100644 index 00000000..544260c5 --- /dev/null +++ b/docs/latency-slo.md @@ -0,0 +1,110 @@ +# Latency-SLO admission + +Latency admission predicts completion risk before provider handoff. It uses measured +time to first visible output and output throughput from completed observations. +It never sends generation probes, writes routing policy, changes an explicit model, +or grants permission to retry a request that may have generated output. + +## Configuration + +`LATENCY_SLO_MODE=off` is the default. Empty values also mean off. In this mode, +admission, routing, responses, headers and observation storage remain unchanged. +Malformed settings disable admission and produce one warning per setting without +including the value. Both runtimes use the same settings and prediction formula. + +To reject predicted misses on an exact managed route: + +```sh +LATENCY_SLO_MODE=reject +LATENCY_SLO_MIN_SAMPLES=20 +LATENCY_SLO_MAX_MODELS=128 +LATENCY_SLO_POLICY_JSON='{"routes":{"/v1/chat/completions":{"deadline_ms":5000,"require_coverage":false}}}' +``` + +To allow eligible alternatives for an automatic route: + +```sh +LATENCY_SLO_MODE=reroute +LATENCY_SLO_POLICY_JSON='{"routes":{"auto:intelligence":{"deadline_ms":5000,"require_coverage":true}},"keys":{"account-42":{"deadline_ms":3000}}}' +``` + +`routes` matches exact HTTP paths or exact automatic model identifiers. `keys` +matches an authenticated key/account identifier, never a caller-provided header +or API credential. Each entry requires integer `deadline_ms` between 1 and +300000; `require_coverage` is an optional boolean, default false. If several +entries match, the smallest deadline and any coverage requirement win. No match +means ordinary policy. The default policy is `{}`. + +Policies reject duplicate keys, unknown fields, invalid identifiers, more than +128 entries per scope, or more than 32768 UTF-8 bytes. The sample minimum is an +integer between 1 and 1000, default 20. The model cap is an integer between 1 and +512, default 128. Invalid numeric settings disable the feature. + +## Prediction and admission + +Each model retains at most 1000 paired observations from the last 15 minutes. +The oldest inactive models are evicted when the model cap is reached. Partial +measurements, nonpositive throughput, invalid timing, and future samples do not +provide coverage. Measurements are process/isolate-local and are lost on restart. +Stored routing-health averages and reviewed latency estimates do not count as +recent SLO samples. With admission disabled, no new SLO observations are stored. + +For each recent pair, predicted milliseconds are +`ttft_ms + requested_output_tokens * 1000 / tokens_per_second`. The prediction is +the empirical p95 using the nearest-rank method, rounded up. If both `max_tokens` +and `max_completion_tokens` are present, the smaller valid limit is used. The +default is 1024 output tokens. Intelligence selection uses its already validated +output limit. Limits must be integers from 1 to 131072; requests outside this +prediction bound are unknown and remain subject to ordinary request validation. +Admission never rewrites the payload to impose a prediction limit. + +The decision reports sample count, required sample count, coverage fraction, +latest observation age, window length, requested output, and predicted time. +Successful Flask decisions are request-local; errors include this content-free +report in `error.prediction`. No response header is added. + +Sparse or old evidence yields `prediction_unknown`. Ordinary policy applies +unless coverage is required, in which case admission returns HTTP 503 with +`latency_slo_unavailable`. A known miss in reject mode returns HTTP 503 with +`latency_slo_predicted_miss` before provider handoff. Both errors have +`retryable: false` and `Cache-Control: no-store`. + +Reroute mode changes only automatic selections, using alternatives already +eligible under the existing authorization, entitlement, privacy, capabilities, +context, billing and approved session-lane policy. Alternatives must have the +same reviewed quality tier as the original selection and a known prediction +within the deadline. Missing tier approval never authorizes rerouting. An +explicit model receives the reject behavior on a known miss; unknown predictions +pass through when coverage is optional. A reroute removes unsafe alternatives +from that selection rather than adding models or retry attempts. + +Generation deadlines remain the hard runtime limit. Admission uses the remaining +runtime budget when present and never extends, resets, or cancels that deadline. +A passing prediction can still time out or fail. Predictions are neither billing +facts nor a guarantee of provider latency, and rejected admission creates no +provider usage or fabricated successful completion. + +## Runtime interfaces + +Flask registers `latency_slo_request_hook` immediately after the generation +deadline hook and before the idempotency claim. Managed chat, messages, Responses +and intelligence requests are covered; provider passthrough is unchanged. +`latency_slo_candidate_policy` is an explicit read-only collaborator returning an +ordered list after eligibility and approved lane selection. An unresolved auto +selection has unknown coverage at this early boundary. Intelligence selection +and automatic route-health ordering enforce the policy again on their actual +candidate lists before dispatch. Existing provider validation still applies. + +Native Worker routes call `prepareLatencySLO` after verified authentication and +deadline setup, with canonical provider/model IDs, the verified key identifier, +and already eligible approved candidates. The returned rejection must be handled +before dispatch, and rerouted candidates must remain restricted at dispatch. +`recordLatencyObservation` consumes the existing telemetry finalization event. +Only successful, measured, non-cache observations with positive observed output +and time after first output contribute throughput. Forwarded or passthrough +traffic must not call this native admission interface twice. + +Observation windows contain model IDs and numeric timings only. They store no +prompts, completions, credentials, identity, queue size, prices or usage ledgers. +Content retention controls remain independent. No database table or migration is +required. diff --git a/package.json b/package.json index a006629c..0657ff16 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "lint": "ruff check . --select E9,F63,F7,F82", "typecheck": "mypy --ignore-missing-imports --follow-imports=skip --disable-error-code import-untyped --disable-error-code var-annotated --disable-error-code arg-type --disable-error-code assignment --disable-error-code operator app.py routes services providers streaming", "test": "python -m pytest -q", - "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_managed_idempotency_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_gateway_alerts_worker.mjs tests/test_session_tiers_worker.mjs tests/test_usage_reservations_worker.mjs tests/test_shared_generation_cache_worker.mjs tests/test_deferred_tools_worker.mjs tests/test_generation_deadline_worker.mjs tests/test_wave4_worker.mjs tests/test_responses_state_worker.mjs tests/test_gateway_batches_worker.mjs tests/test_semantic_generation_cache_worker.mjs tests/test_canary_traffic_worker.mjs tests/test_context_pages_worker.mjs tests/test_observability_exports_worker.mjs tests/test_prompt_injection_worker.mjs tests/test_pii_rehydration_worker.mjs tests/test_wave5_worker.mjs", + "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_managed_idempotency_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_gateway_alerts_worker.mjs tests/test_session_tiers_worker.mjs tests/test_usage_reservations_worker.mjs tests/test_shared_generation_cache_worker.mjs tests/test_deferred_tools_worker.mjs tests/test_generation_deadline_worker.mjs tests/test_wave4_worker.mjs tests/test_responses_state_worker.mjs tests/test_gateway_batches_worker.mjs tests/test_semantic_generation_cache_worker.mjs tests/test_canary_traffic_worker.mjs tests/test_context_pages_worker.mjs tests/test_observability_exports_worker.mjs tests/test_prompt_injection_worker.mjs tests/test_pii_rehydration_worker.mjs tests/test_wave5_worker.mjs tests/test_latency_slo_worker.mjs", "db:migrate": "python scripts/validate_sqlite_schema.py --apply", "db:validate": "python scripts/validate_sqlite_schema.py", "security:secrets": "python scripts/check_static_secrets.py", diff --git a/services/gateway_extensions.py b/services/gateway_extensions.py index 9b62728d..dbc6fdf2 100644 --- a/services/gateway_extensions.py +++ b/services/gateway_extensions.py @@ -8,7 +8,7 @@ AUTHENTICATED_HOOK_ORDER = ( "request_policy_hook", "prompt_injection_request_hook", "spillover_hook", "pii_request_hook", - "responses_state_hook", "generation_deadline_hook", "idempotency_request_hook", "admit", + "responses_state_hook", "generation_deadline_hook", "latency_slo_request_hook", "idempotency_request_hook", "admit", ) @@ -93,7 +93,7 @@ def gateway_callbacks(*, csrf=None): from middleware.rate_limit_headers import register_rate_limit_headers from services.pii_redaction import register_pii_redaction return (register_retention, register_injection_decision, register_batch_spillover, register_pii_redaction, - partial(register_hosted_responses, csrf=csrf), register_deadline, register_managed_idempotency, + partial(register_hosted_responses, csrf=csrf), register_deadline, register_latency_slo, register_managed_idempotency, register_admission, register_cooldown_errors, register_rate_limit_headers) @@ -125,6 +125,17 @@ def register_deadline(app): register_generation_deadline(app, verify_internal=verified_internal_transport, limits_ms=generation_limits_ms) +def latency_slo_candidate_policy(payload): + """Read-only collaborator returns candidates after eligibility and approved lane selection.""" + callback = current_app.extensions.get("latency_slo_candidate_policy") + return callback(payload) if callback is not None else None + + +def register_latency_slo(app): + from services.latency_slo import register_latency_slo as register + register(app, is_managed=managed_generation_request, candidates=latency_slo_candidate_policy) + + def managed_policy_revision(payload): from services.managed_turn import idempotency_policy_revision return idempotency_policy_revision(payload) diff --git a/services/intelligence_policy.py b/services/intelligence_policy.py index 07a46b4f..89e07c4a 100644 --- a/services/intelligence_policy.py +++ b/services/intelligence_policy.py @@ -254,8 +254,11 @@ def rank(item): def apply_session_tier(choices): if session_tier is not None and not request.explicit: - return session_tier.select(choices) - return choices + choices = session_tier.select(choices) + from services.latency_slo import selection_candidates + return selection_candidates(choices, route="auto:intelligence", + output_tokens=request.output_tokens, auto=not request.explicit, + lane_selected=session_tier is not None) ranked = sorted(candidates, key=rank) if request.explicit or request.profile == "balanced": diff --git a/services/latency_slo.py b/services/latency_slo.py new file mode 100644 index 00000000..2c298fd5 --- /dev/null +++ b/services/latency_slo.py @@ -0,0 +1,316 @@ +"""Opt-in admission from bounded, paired measurements; no probes or policy writes.""" +from __future__ import annotations + +import json +import logging +import math +import os +import re +import threading +import time +from collections import OrderedDict, deque +from dataclasses import dataclass, field + +from flask import current_app, g, has_request_context, jsonify, request + +from services.intelligence_contract import GatewayError + +ENV_KEYS = ("LATENCY_SLO_MODE", "LATENCY_SLO_MIN_SAMPLES", "LATENCY_SLO_MAX_MODELS", "LATENCY_SLO_POLICY_JSON") +WINDOW_SECONDS = 900 +MAX_SAMPLES = 1000 +MAX_OUTPUT_TOKENS = 131072 +DEFAULT_OUTPUT_TOKENS = 1024 +logger = logging.getLogger(__name__) +_warned: set[str] = set() +_warning_lock = threading.Lock() +_IDENTIFIER = re.compile(r"[A-Za-z0-9._:/@+-]{1,256}\Z") + + +def _warn_once(setting): + with _warning_lock: + if setting not in _warned: + _warned.add(setting) + logger.warning("Invalid %s; latency SLO admission disabled", setting) + + +def _integer(value, minimum, maximum): + return type(value) is int and minimum <= value <= maximum + + +def _number(value, minimum=0, maximum=1e13): + return type(value) in (int, float) and math.isfinite(value) and minimum <= value <= maximum + + +def _unique_object(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError("Duplicate policy key") + result[key] = value + return result + + +@dataclass(frozen=True) +class Rule: + deadline_ms: int + require_coverage: bool = False + + +@dataclass(frozen=True) +class Settings: + mode: str = "off" + min_samples: int = 20 + max_models: int = 128 + rules: dict = field(default_factory=dict) + + def rule_for(self, *, route="", key_id="", model=""): + selected = [self.rules.get("routes", {}).get(name) for name in (route, model)] + selected.append(self.rules.get("keys", {}).get(key_id)) + selected = [rule for rule in selected if rule is not None] + if not selected: + return None + return Rule(min(rule.deadline_ms for rule in selected), any(rule.require_coverage for rule in selected)) + + +def _parse_policy(raw): + if len(raw.encode()) > 32768: + raise ValueError("Policy too large") + config = json.loads(raw, object_pairs_hook=_unique_object) + if not isinstance(config, dict) or set(config) - {"routes", "keys"}: + raise ValueError("Invalid policy") + rules = {} + for scope in ("routes", "keys"): + entries = config.get(scope, {}) + if not isinstance(entries, dict) or len(entries) > 128: + raise ValueError("Invalid scope") + rules[scope] = {} + for name, rule in entries.items(): + if (not _IDENTIFIER.fullmatch(name) or not isinstance(rule, dict) + or set(rule) - {"deadline_ms", "require_coverage"} + or not _integer(rule.get("deadline_ms"), 1, 300000) + or type(rule.get("require_coverage", False)) is not bool): + raise ValueError("Invalid rule") + rules[scope][name] = Rule(rule["deadline_ms"], rule.get("require_coverage", False)) + return rules + + +def load_settings(env=None): + env = os.environ if env is None else env + mode = str(env.get("LATENCY_SLO_MODE", "")).strip().lower() or "off" + if mode == "off": + return Settings() + if mode not in {"reject", "reroute"}: + _warn_once("LATENCY_SLO_MODE") + return Settings() + limits = {} + for name, default, maximum in (("LATENCY_SLO_MIN_SAMPLES", 20, 1000), ("LATENCY_SLO_MAX_MODELS", 128, 512)): + raw = str(env.get(name, "")).strip() or str(default) + if not re.fullmatch(r"[0-9]{1,4}", raw) or not 1 <= int(raw) <= maximum: + _warn_once(name) + return Settings() + limits[name] = int(raw) + try: + rules = _parse_policy(str(env.get("LATENCY_SLO_POLICY_JSON", "")).strip() or "{}") + except (ValueError, TypeError, RecursionError): + _warn_once("LATENCY_SLO_POLICY_JSON") + return Settings() + return Settings(mode, limits["LATENCY_SLO_MIN_SAMPLES"], limits["LATENCY_SLO_MAX_MODELS"], rules) + + +def requested_output(payload, default=DEFAULT_OUTPUT_TOKENS): + values = [payload[name] for name in ("max_tokens", "max_completion_tokens") if name in payload] + values = values or [default] + return min(values) if all(_integer(value, 1, MAX_OUTPUT_TOKENS) for value in values) else None + + +class ObservationWindow: + """Process-local FIFO model windows; reads neither mutate nor persist measurements.""" + def __init__(self, max_models=128): + if not _integer(max_models, 1, 512): + raise ValueError("Invalid model cap") + self.max_models = max_models + self._models: OrderedDict[str, deque] = OrderedDict() + self._lock = threading.RLock() + + def clear(self): + with self._lock: + self._models.clear() + + def record(self, model, *, ttft_ms, tokens_per_second, now=None, max_models=None): + timestamp = time.time() if now is None else now + if (not isinstance(model, str) or not _IDENTIFIER.fullmatch(model) + or not _number(timestamp) or not _number(ttft_ms, maximum=3600000) + or not _number(tokens_per_second, minimum=1e-9, maximum=1000000)): + return False + with self._lock: + cap = self.max_models if max_models is None else max_models + if not _integer(cap, 1, 512): + return False + for name in list(self._models): + recent = [sample for sample in self._models[name] if sample[0] >= timestamp - WINDOW_SECONDS] + if recent: + self._models[name] = deque(recent, maxlen=MAX_SAMPLES) + else: + del self._models[name] + samples = self._models.setdefault(model, deque(maxlen=MAX_SAMPLES)) + samples.append((timestamp, ttft_ms, tokens_per_second)) + self._models.move_to_end(model) + while len(self._models) > cap: + self._models.popitem(last=False) + return True + + def predict(self, model, output_tokens, *, min_samples=20, now=None): + timestamp = time.time() if now is None else now + with self._lock: + samples = tuple(self._models.get(model, ())) + recent = [sample for sample in samples if 0 <= timestamp - sample[0] <= WINDOW_SECONDS] + age = max(0, round((timestamp - max(sample[0] for sample in samples)) * 1000)) if samples else None + prediction = {"status": "prediction_unknown", "predicted_ms": None, "samples": len(recent), + "min_samples": min_samples, "coverage": min(1, len(recent) / min_samples), + "observation_age_ms": age, "window_ms": WINDOW_SECONDS * 1000, + "output_tokens": output_tokens} + if len(recent) >= min_samples and _integer(output_tokens, 1, MAX_OUTPUT_TOKENS): + times = sorted(ttft + output_tokens * 1000 / rate for _, ttft, rate in recent) + prediction.update(status="prediction_known", predicted_ms=math.ceil(times[math.ceil(.95 * len(times)) - 1])) + return prediction + + +observations = ObservationWindow() + + +def record_observation(model, *, ttft_ms, tokens_per_second, now=None): + settings = load_settings() + if settings.mode == "off": + return False + return observations.record(model, ttft_ms=ttft_ms, tokens_per_second=tokens_per_second, + now=now, max_models=settings.max_models) + + +class LatencySLOError(GatewayError): + def __init__(self, code, prediction): + message = ("Recent latency coverage is unavailable" if code == "latency_slo_unavailable" + else "Predicted completion exceeds the latency deadline") + super().__init__(code, message, 503) + self.prediction = prediction + + def envelope(self): + payload = super().envelope() + payload["error"]["prediction"] = self.prediction + return payload + + +@dataclass(frozen=True) +class Decision: + candidates: list + action: str + prediction: dict | None + + +def admit_candidates(candidates, *, settings, rule, output_tokens, auto, observations=observations, now=None): + """Caller supplies an ordered eligible list, already restricted to approved lanes.""" + if settings.mode == "off" or rule is None: + return Decision(candidates, "pass", None) + primary = candidates[0] if candidates else {} + prediction = observations.predict(primary.get("model"), output_tokens, min_samples=settings.min_samples, now=now) + if prediction["status"] == "prediction_unknown": + if rule.require_coverage: + raise LatencySLOError("latency_slo_unavailable", prediction) + return Decision(candidates, "pass", prediction) + if prediction["predicted_ms"] <= rule.deadline_ms: + return Decision(candidates, "pass", prediction) + tier = primary.get("quality_tier") + if settings.mode == "reroute" and auto and _integer(tier, 0, 100): + safe = [] + for candidate in candidates[1:]: + if type(candidate.get("quality_tier")) is not int or candidate["quality_tier"] != tier: + continue + alternate = observations.predict(candidate.get("model"), output_tokens, min_samples=settings.min_samples, now=now) + if alternate["status"] == "prediction_known" and alternate["predicted_ms"] <= rule.deadline_ms: + safe.append(candidate) + if safe: + return Decision(safe, "reroute", observations.predict(safe[0]["model"], output_tokens, + min_samples=settings.min_samples, now=now)) + raise LatencySLOError("latency_slo_predicted_miss", prediction) + + +def selection_candidates(candidates, *, route, output_tokens, auto, lane_selected=False): + """Enforce after eligibility, including callers outside Flask.""" + context = getattr(g, "latency_slo_context", None) if has_request_context() else None + if has_request_context() and context is None: + return candidates + if context is not None and auto and not lane_selected: + from services.managed_turn import current_turn + turn = current_turn() + if turn is not None and turn.tier_metadata is not None: + # Lane acquisition requires the original approved model to remain eligible. + return candidates + if context is None: + settings = load_settings() + rule = settings.rule_for(route=route) + else: + settings, rule, _ = context + from services.generation_deadline import check_deadline + deadline = check_deadline() + if deadline is not None: + rule = Rule(min(rule.deadline_ms, deadline.remaining_ms()), rule.require_coverage) + decision = admit_candidates(candidates, settings=settings, rule=rule, + output_tokens=output_tokens, auto=auto, + now=current_app.extensions["latency_slo"]["clock"]() if context is not None else None) + if context is not None: + g.latency_slo_decision = decision + return decision.candidates + + +def order_auto_candidates(route, candidates): + """Generic routes have no tier evidence unless the authenticated callback supplies it.""" + if not has_request_context() or not getattr(g, "latency_slo_context", None): + return candidates + metadata = {item["model"]: item for item in getattr(g, "latency_slo_eligible", ())} + decision = getattr(g, "latency_slo_decision", None) + allowed = {item["model"] for item in decision.candidates} if decision is not None and decision.action == "reroute" else None + eligible = [metadata.get(model, {"model": model}) for model in candidates if allowed is None or model in allowed] + _, _, output = g.latency_slo_context + selected = selection_candidates(eligible, route=route, output_tokens=output, auto=True) + return tuple(item["model"] for item in selected) + + +def latency_slo_request_hook(): + options = current_app.extensions["latency_slo"] + settings = load_settings() + if settings.mode == "off" or not options["is_managed"]() or not getattr(g, "authenticated_user", None): + return None + body = request.get_json(silent=True) + if not isinstance(body, dict) or not isinstance(body.get("model"), str): + return None + model = body["model"] + user = g.authenticated_user + rule = settings.rule_for(route=request.path, model=model, + key_id=str(user.get("api_key_id") or user.get("id") or user.get("username") or "")) + if rule is None: + return None + from services.generation_deadline import check_deadline + deadline = check_deadline() + if deadline is not None: + rule = Rule(min(rule.deadline_ms, deadline.remaining_ms()), rule.require_coverage) + output = requested_output(body) + auto = model.startswith("auto:") + eligible = options["candidates"](body) if auto else [{"model": model}] + g.latency_slo_context = (settings, rule, output) + g.latency_slo_eligible = eligible or [] + try: + g.latency_slo_decision = admit_candidates(eligible or [], settings=settings, rule=rule, + output_tokens=output, auto=auto, now=options["clock"]()) + except LatencySLOError as error: + return latency_slo_error_response(error) + return None + + +def latency_slo_error_response(error): + return jsonify(error.envelope()), 503, {"Cache-Control": "no-store"} + + +def register_latency_slo(app, *, is_managed, candidates=lambda body: None, clock=time.time): + from services.gateway_extensions import register_authenticated_hook + app.extensions["latency_slo"] = {"is_managed": is_managed, "candidates": candidates, "clock": clock} + register_authenticated_hook(app, latency_slo_request_hook) + app.register_error_handler(LatencySLOError, latency_slo_error_response) diff --git a/services/route_health.py b/services/route_health.py index 1f078ff4..93ada721 100644 --- a/services/route_health.py +++ b/services/route_health.py @@ -197,6 +197,8 @@ class RouteHealth: @classmethod def reset(cls) -> None: + from services.latency_slo import observations + observations.clear() with cls._lock: cls._entries = OrderedDict() cls._dirty = set() @@ -266,6 +268,8 @@ def record_speed(cls, candidate: str, *, ttft_ms: float | None = None, tokens_per_second: float | None = None, now: float | None = None) -> None: """A streamed generation's time to first visible output and its visible output rate.""" current_time = time.time() if now is None else now + from services.latency_slo import record_observation + record_observation(candidate, ttft_ms=ttft_ms, tokens_per_second=tokens_per_second, now=current_time) observed = { "ewma_ttft_ms": ttft_ms if _number(ttft_ms, maximum=3_600_000) else None, "ewma_tps": tokens_per_second if _number(tokens_per_second, maximum=1_000_000) else None, @@ -360,7 +364,7 @@ def order(cls, route_id: str, candidates: Iterable[str], *, configured = tuple(candidates) settings = ordering_settings() if settings.mode_for(route_id) != "health" or len(configured) < 2: - return RouteOrder(configured, "priority") + return cls._latency_order(route_id, RouteOrder(configured, "priority")) current_time = time.time() if now is None else now costs = cls._cost_penalties(configured) if settings.cost_weight else {} with cls._lock: @@ -400,7 +404,12 @@ def order(cls, route_id: str, candidates: Iterable[str], *, ordered.remove(probe) ordered.insert(0, probe) mode = "health-probe" - return RouteOrder(tuple(ordered), mode) + return cls._latency_order(route_id, RouteOrder(tuple(ordered), mode)) + + @staticmethod + def _latency_order(route_id: str, order: RouteOrder) -> RouteOrder: + from services.latency_slo import order_auto_candidates + return RouteOrder(order_auto_candidates(route_id, order.candidates), order.mode) # Public summaries --------------------------------------------------------- diff --git a/tests/fixtures/secret_firewall_coverage.json b/tests/fixtures/secret_firewall_coverage.json index 37f37f77..dfbd80e2 100644 --- a/tests/fixtures/secret_firewall_coverage.json +++ b/tests/fixtures/secret_firewall_coverage.json @@ -316,6 +316,7 @@ "worker/knowledge/skills-store.mjs", "worker/knowledge/skills-validation.mjs", "worker/knowledge/skills.mjs", + "worker/latency-slo.mjs", "worker/log.mjs", "worker/managed-state-dispatch.mjs", "worker/media-files.mjs", diff --git a/tests/test_latency_slo.py b/tests/test_latency_slo.py new file mode 100644 index 00000000..8f9205ab --- /dev/null +++ b/tests/test_latency_slo.py @@ -0,0 +1,317 @@ +"""Content-free predictions and pre-dispatch admission with synthetic observations.""" +import copy +import json +from unittest.mock import Mock + +import pytest +from flask import Flask, Response, g + +from services import latency_slo as slo +from services import gateway_extensions as extensions +from services import generation_deadline as gd +from services import intelligence_policy as ip +from services.intelligence_contract import ChatRequest +from services.route_health import RouteHealth + +NOW = 1800000000.0 +A, B, C = "openai:slow", "openai:fast", "openai:other" + + +def environment(mode="reject", required=False, **changes): + return {"LATENCY_SLO_MODE": mode, "LATENCY_SLO_POLICY_JSON": json.dumps({ + "routes": {"/v1/chat/completions": {"deadline_ms": 5000, "require_coverage": required}, + "auto:intelligence": {"deadline_ms": 5000, "require_coverage": required}}}), **changes} + + +@pytest.fixture(autouse=True) +def isolated(monkeypatch): + for name in slo.ENV_KEYS: + monkeypatch.setenv(name, "") + monkeypatch.setenv("GENERATION_DEADLINE_MAX_MS", "") + monkeypatch.setenv("SECRET_SCAN_DEFAULT", "off") + slo._warned.clear() + RouteHealth.reset() + yield + RouteHealth.reset() + + +def measured(store, model=A, ttft=1000, rate=10, count=20, now=NOW): + for _ in range(count): + store.record(model, ttft_ms=ttft, tokens_per_second=rate, now=now) + + +def choice(model=A, tier=1, **changes): + return {"model": model, "quality_tier": tier, "enabled": True, "entitled": True, + "privacy_allowed": True, "billing": "free", "capabilities": [], + "context_window": 32768, "max_output_tokens": 4096, **changes} + + +def decide(store, candidates=None, mode="reject", required=False, output=100, auto=True): + settings = slo.load_settings(environment(mode, required)) + rule = settings.rule_for(route="auto:intelligence") + return slo.admit_candidates(candidates or [choice()], settings=settings, rule=rule, + output_tokens=output, auto=auto, observations=store, now=NOW) + + +def test_sparse_stale_and_invalid_pairs_are_unknown(): + store = slo.ObservationWindow() + measured(store, count=19) + prediction = store.predict(A, 100, min_samples=20, now=NOW) + assert prediction["status"] == "prediction_unknown" + assert prediction["samples"] == 19 and prediction["coverage"] == .95 + store.record(A, ttft_ms=None, tokens_per_second=10, now=NOW) + store.record(A, ttft_ms=100, tokens_per_second=0, now=NOW) + assert store.predict(A, 100, min_samples=20, now=NOW)["samples"] == 19 + measured(store, now=NOW - 901) + assert store.predict(A, 100, min_samples=20, now=NOW + 901)["samples"] == 0 + + +def test_empirical_p95_and_output_bound(): + store = slo.ObservationWindow() + measured(store, count=18, ttft=100, rate=100) + store.record(A, ttft_ms=2000, tokens_per_second=100, now=NOW - 1) + store.record(A, ttft_ms=9000, tokens_per_second=100, now=NOW - 1) + small = store.predict(A, 100, min_samples=20, now=NOW) + large = store.predict(A, 1000, min_samples=20, now=NOW) + assert small["predicted_ms"] == 3000 and large["predicted_ms"] == 12000 + assert small["observation_age_ms"] == 0 and small["coverage"] == 1 + assert slo.requested_output({}) == 1024 + assert slo.requested_output({"max_tokens": 100, "max_completion_tokens": 50}) == 50 + for value in [True, 0, -1, "100", 131073]: + assert slo.requested_output({"max_tokens": value}) is None + + +def test_window_and_model_caps_concurrent_and_read_only(): + from concurrent.futures import ThreadPoolExecutor + store = slo.ObservationWindow(max_models=2) + with ThreadPoolExecutor(max_workers=4) as pool: + list(pool.map(lambda _: store.record(A, ttft_ms=1, tokens_per_second=10, now=NOW), range(1100))) + assert store.predict(A, 1, min_samples=20, now=NOW)["samples"] == 1000 + measured(store, B) + measured(store, C) + assert store.predict(A, 1, min_samples=20, now=NOW)["samples"] == 0 + assert len(store._models) == 2 + before = copy.deepcopy(store._models) + store.predict(C, 1, min_samples=20, now=NOW) + assert store._models == before + + +def test_unknown_policy_and_explicit_model_preserved(): + store = slo.ObservationWindow() + assert decide(store).prediction["status"] == "prediction_unknown" + with pytest.raises(slo.LatencySLOError) as error: + decide(store, required=True) + assert error.value.code == "latency_slo_unavailable" + measured(store) + measured(store, B, rate=100) + with pytest.raises(slo.LatencySLOError) as error: + decide(store, [choice(), choice(B)], mode="reroute", auto=False) + assert error.value.code == "latency_slo_predicted_miss" + + +def test_reroute_only_already_eligible_same_approved_tier(): + store = slo.ObservationWindow() + measured(store) + measured(store, B, rate=100) + measured(store, C, rate=100) + candidates = [choice(), choice(C, tier=2), choice(B)] + original = copy.deepcopy(candidates) + decision = decide(store, candidates, mode="reroute") + assert decision.candidates == [choice(B)] and decision.action == "reroute" + assert candidates == original + with pytest.raises(slo.LatencySLOError): + decide(store, candidates[:2], mode="reroute") + assert decide(store, output=10).action == "pass" + + +@pytest.mark.parametrize("change", [ + {"LATENCY_SLO_MODE": "private-bad"}, {"LATENCY_SLO_MIN_SAMPLES": "0"}, + {"LATENCY_SLO_MAX_MODELS": "513"}, {"LATENCY_SLO_POLICY_JSON": "[]"}, + {"LATENCY_SLO_POLICY_JSON": '{"routes":{"x":{"deadline_ms":true}}}'}, + {"LATENCY_SLO_POLICY_JSON": '{"routes":{"x":{"deadline_ms":10,"extra":1}}}'}, + {"LATENCY_SLO_POLICY_JSON": '{"routes":{},"routes":{}}'}, + {"LATENCY_SLO_POLICY_JSON": '{"keys":{"x":{"deadline_ms":1,"require_coverage":"yes"}}}'}, +]) +def test_invalid_settings_disable_once_without_values(change, caplog): + for _ in range(2): + assert slo.load_settings(environment(**change)).mode == "off" + assert len(caplog.records) == 1 + assert "private-bad" not in caplog.text + + +def test_empty_defaults_and_tightest_route_key_rule(): + assert slo.load_settings({name: "" for name in slo.ENV_KEYS}).mode == "off" + settings = slo.load_settings(environment(LATENCY_SLO_POLICY_JSON=json.dumps({ + "routes": {"auto:intelligence": {"deadline_ms": 5000}}, + "keys": {"caller": {"deadline_ms": 1000, "require_coverage": True}}}))) + rule = settings.rule_for(route="auto:intelligence", key_id="caller") + assert rule.deadline_ms == 1000 and rule.require_coverage + assert settings.rule_for(route="other", key_id="other") is None + + +def test_route_health_records_paired_measurements_only_when_on(monkeypatch): + RouteHealth.record_speed(A, ttft_ms=100, tokens_per_second=100, now=NOW) + assert slo.observations.predict(A, 100, min_samples=1, now=NOW)["samples"] == 0 + for name, value in environment().items(): + monkeypatch.setenv(name, value) + for _ in range(20): + RouteHealth.record_speed(A, ttft_ms=100, tokens_per_second=100, now=NOW) + assert slo.observations.predict(A, 100, min_samples=20, now=NOW)["predicted_ms"] == 1100 + assert "latency_slo" not in RouteHealth.snapshot(A) + assert all("latency_slo" not in row["state"] for row in RouteHealth.dirty_rows(100)) + + +def registered_app(monkeypatch, env, candidates=None, clock=lambda: NOW): + for name, value in env.items(): + monkeypatch.setenv(name, value) + app = Flask(__name__) + gd.register_generation_deadline(app, clock=clock) + slo.register_latency_slo(app, is_managed=extensions.managed_generation_request, + candidates=lambda body: candidates, clock=clock) + claims, handoffs = [], [] + def idempotency_request_hook(): + claims.append(True) + extensions.register_authenticated_hook(app, idempotency_request_hook) + @app.before_request + def authenticated(): + g.authenticated_user = {"id": "caller", "username": "caller"} + return extensions.after_authentication() + @app.post("/v1/chat/completions") + @app.post("/openai/v1/chat/completions") + def generate(): + handoffs.append(True) + return Response(b'{"untouched":true}', headers={"X-Test": "unchanged"}, content_type="application/json") + return app, claims, handoffs + + +def test_registered_rejection_before_claim_or_handoff_and_fixed_order(monkeypatch): + measured(slo.observations) + app, claims, handoffs = registered_app(monkeypatch, environment()) + response = app.test_client().post("/v1/chat/completions", json={"model": A, "max_tokens": 100}) + assert response.status_code == 503 and response.json["error"]["code"] == "latency_slo_predicted_miss" + assert response.json["error"]["prediction"]["samples"] == 20 + assert not claims and not handoffs + names = [hook.__name__ for hook in app.extensions["gateway_after_authentication"]] + assert names == ["generation_deadline_hook", "latency_slo_request_hook", "idempotency_request_hook"] + assert extensions.AUTHENTICATED_HOOK_ORDER.index("latency_slo_request_hook") == extensions.AUTHENTICATED_HOOK_ORDER.index("generation_deadline_hook") + 1 + assert extensions.register_latency_slo in extensions.gateway_callbacks() + + +def test_default_and_raw_responses_unchanged(monkeypatch): + app, _, handoffs = registered_app(monkeypatch, environment(mode="off")) + client = app.test_client() + body = {"model": A, "max_tokens": 100} + baseline = client.post("/v1/chat/completions", json=body) + measured(slo.observations) + monkeypatch.setenv("LATENCY_SLO_MODE", "reject") + raw = client.post("/openai/v1/chat/completions", json=body) + assert raw.data == baseline.data and raw.headers == baseline.headers + monkeypatch.setenv("LATENCY_SLO_MODE", "") + disabled = client.post("/v1/chat/completions", json=body) + assert disabled.data == baseline.data and disabled.headers == baseline.headers + assert len(handoffs) == 3 + + +def test_registered_unknown_required_and_auto_fake_collaborator(monkeypatch): + app, claims, handoffs = registered_app(monkeypatch, environment(required=True), [choice(), choice(B)]) + client = app.test_client() + response = client.post("/v1/chat/completions", json={"model": "auto:intelligence", "max_tokens": 100}) + assert response.status_code == 503 and response.json["error"]["code"] == "latency_slo_unavailable" + assert not claims and not handoffs + measured(slo.observations) + measured(slo.observations, B, rate=100) + monkeypatch.setenv("LATENCY_SLO_MODE", "reroute") + assert client.post("/v1/chat/completions", json={"model": "auto:intelligence", "max_tokens": 100}).status_code == 200 + + +def test_prediction_does_not_extend_runtime_deadline(monkeypatch): + clock = Mock(return_value=NOW) + app, _, _ = registered_app(monkeypatch, environment(), clock=clock) + measured(slo.observations, ttft=1, rate=10000) + with app.test_request_context("/v1/chat/completions", method="POST", json={"model": A, "max_tokens": 100}, headers={gd.PUBLIC_HEADER: "1000"}): + g.authenticated_user = {"id": "caller"} + assert extensions.after_authentication() is None + deadline = gd.current_deadline() + deadline.stop() + assert deadline.expires_at == NOW + 1 + clock.return_value = NOW + 2 + with pytest.raises(gd.GenerationDeadlineExceeded): + gd.check_deadline() + + +def test_intelligence_filters_after_eligibility_without_policy_write(monkeypatch): + configured = ip.validate_policy({"candidates": [choice(), choice(B), choice(C, entitled=False)]}) + before = copy.deepcopy(configured) + parsed = ChatRequest.parse({"model": "auto:intelligence", "messages": [{"role": "user", "content": "synthetic"}], "max_tokens": 100}, configured) + monkeypatch.setattr(ip, "eligible", lambda candidate, *args: candidate["entitled"]) + for name, value in environment(mode="reroute").items(): + monkeypatch.setenv(name, value) + measured(slo.observations, now=slo.time.time()) + measured(slo.observations, B, rate=100, now=slo.time.time()) + assert ip.select_candidates(configured, parsed, {"API_BASE_URLS": {}}) == [choice(B)] + assert configured == before + + +def test_generic_auto_order_respects_pre_admission_restriction(monkeypatch): + measured(slo.observations) + measured(slo.observations, B, rate=100) + app, _, _ = registered_app(monkeypatch, environment(mode="reroute"), [choice(), choice(B), choice(C, 2)]) + with app.test_request_context("/v1/chat/completions", method="POST", json={"model": "auto:test", "max_tokens": 100}): + g.authenticated_user = {"id": "caller"} + assert extensions.after_authentication() is None + assert RouteHealth.order("auto:test", (A, B, C), now=NOW).candidates == (B,) + + +def test_missing_approved_tier_never_authorizes_generic_reroute(monkeypatch): + measured(slo.observations) + measured(slo.observations, B, rate=100) + app, _, _ = registered_app(monkeypatch, environment(mode="reroute")) + with app.test_request_context("/v1/chat/completions", method="POST", json={"model": "auto:test", "max_tokens": 100}): + g.authenticated_user = {"id": "caller"} + assert extensions.after_authentication() is None + with pytest.raises(slo.LatencySLOError) as error: + RouteHealth.order("auto:test", (A, B), now=NOW) + assert error.value.code == "latency_slo_predicted_miss" + + +def test_session_tier_filter_precedes_slo_filter(monkeypatch): + configured = ip.validate_policy({"candidates": [choice(), choice(B, 2), choice(C, 1)]}) + parsed = ChatRequest.parse({"model": "auto:intelligence", "messages": [{"role": "user", "content": "synthetic"}], "max_tokens": 100}, configured) + monkeypatch.setattr(ip, "eligible", lambda *args: True) + for name, value in environment(mode="reroute").items(): + monkeypatch.setenv(name, value) + measured(slo.observations, now=slo.time.time()) + measured(slo.observations, B, rate=100, now=slo.time.time()) + measured(slo.observations, C, rate=100, now=slo.time.time()) + tier = Mock(select=lambda choices: [candidate for candidate in choices if candidate["quality_tier"] == 1]) + assert ip.select_candidates(configured, parsed, {"API_BASE_URLS": {}}, session_tier=tier) == [choice(C)] + + +def test_unknown_output_cannot_underestimate_large_request(): + store = slo.ObservationWindow() + measured(store, rate=1000000) + with pytest.raises(slo.LatencySLOError) as error: + decide(store, required=True, output=None) + assert error.value.code == "latency_slo_unavailable" + + +def test_lane_acquisition_keeps_original_approval_before_filtering(monkeypatch): + from services import managed_turn + configured = ip.validate_policy({"candidates": [choice(), choice(B, 2), choice(C, 1)]}) + body = {"model": "auto:intelligence", "messages": [{"role": "user", "content": "synthetic"}], "max_tokens": 100} + parsed = ChatRequest.parse(body, configured) + monkeypatch.setattr(ip, "eligible", lambda *args: True) + app, _, _ = registered_app(monkeypatch, environment(mode="reroute")) + measured(slo.observations) + measured(slo.observations, B, rate=100) + measured(slo.observations, C, rate=100) + with app.test_request_context("/v1/chat/completions", method="POST", json=body): + g.authenticated_user = {"id": "caller"} + assert extensions.after_authentication() is None + token = managed_turn._turn.set(managed_turn.ManagedTurn("chat", tier_metadata={"approved_model": A})) + try: + assert ip.select_candidates(configured, parsed, {}) == configured["candidates"] + lane = Mock(select=lambda candidates: [candidate for candidate in candidates if candidate["quality_tier"] == 1]) + assert ip.select_candidates(configured, parsed, {}, session_tier=lane) == [choice(C)] + finally: + managed_turn._turn.reset(token) diff --git a/tests/test_latency_slo_worker.mjs b/tests/test_latency_slo_worker.mjs new file mode 100644 index 00000000..42b9758a --- /dev/null +++ b/tests/test_latency_slo_worker.mjs @@ -0,0 +1,135 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { ObservationWindow, latencySLOSettings, requestedOutput, admitCandidates, + LatencySLOError, prepareLatencySLO, recordLatencyObservation } from "../worker/latency-slo.mjs"; + +const now = 1800000000000, a = "openai:slow", b = "openai:fast", c = "openai:other"; +const env = (mode = "reject", required = false, changes = {}) => ({ LATENCY_SLO_MODE: mode, + LATENCY_SLO_POLICY_JSON: JSON.stringify({ routes: { "/v1/chat/completions": { deadline_ms: 5000, require_coverage: required } } }), ...changes }); +const choices = [{ model: a, quality_tier: 1 }, { model: b, quality_tier: 1 }, { model: c, quality_tier: 2 }]; +function measured(store, model = a, rate = 10, count = 20, at = now) { + for (let i = 0; i < count; i++) store.record(model, { ttft_ms: 1000, tokens_per_second: rate, now: at }); +} +function decision(store, candidates = choices, mode = "reject", required = false, auto = true, outputTokens = 100) { + const settings = latencySLOSettings(env(mode, required)); + return admitCandidates(candidates, { settings, rule: settings.ruleFor({ route: "/v1/chat/completions" }), observations: store, now, auto, outputTokens }); +} + +test("sparse, expired, unpaired and zero rate are unknown", () => { + const store = new ObservationWindow(); measured(store, a, 10, 19); + store.record(a, { ttft_ms: null, tokens_per_second: 10, now }); + store.record(a, { ttft_ms: 10, tokens_per_second: 0, now }); + assert.equal(store.predict(a, 100, { minSamples: 20, now }).samples, 19); + assert.equal(store.predict(a, 100, { minSamples: 20, now }).coverage, .95); + assert.equal(store.predict(a, 100, { minSamples: 20, now: now + 900001 }).samples, 0); + assert.equal(decision(store).prediction.status, "prediction_unknown"); + assert.throws(() => decision(store, choices, "reject", true), error => error.code === "latency_slo_unavailable"); +}); + +test("empirical p95 uses measured pairs and bounded requested output", () => { + const store = new ObservationWindow(); + for (let i = 0; i < 18; i++) store.record(a, { ttft_ms: 100, tokens_per_second: 100, now }); + store.record(a, { ttft_ms: 2000, tokens_per_second: 100, now }); + store.record(a, { ttft_ms: 9000, tokens_per_second: 100, now }); + assert.equal(store.predict(a, 100, { minSamples: 20, now }).predicted_ms, 3000); + assert.equal(store.predict(a, 1000, { minSamples: 20, now }).predicted_ms, 12000); + assert.equal(requestedOutput({}), 1024); + assert.equal(requestedOutput({ max_tokens: 100, max_completion_tokens: 50 }), 50); + for (const value of [true, 0, -1, "100", 131073]) assert.equal(requestedOutput({ max_tokens: value }), null); +}); + +test("observation and model bounds", () => { + const store = new ObservationWindow({ maxModels: 2 }); measured(store, a, 10, 1100); + assert.equal(store.predict(a, 100, { minSamples: 20, now }).samples, 1000); + measured(store, b); measured(store, c); + assert.equal(store.models.size, 2); + assert.equal(store.predict(a, 100, { minSamples: 20, now }).samples, 0); +}); + +test("reroute selects only supplied same-tier safe candidates and preserves explicit model", () => { + const store = new ObservationWindow(); measured(store); measured(store, b, 100); measured(store, c, 100); + const before = structuredClone(choices); + assert.deepEqual(decision(store, choices, "reroute").candidates, [choices[1]]); + assert.deepEqual(choices, before); + assert.throws(() => decision(store, [choices[0], choices[2]], "reroute"), LatencySLOError); + assert.throws(() => decision(store, choices, "reroute", false, false), error => error.code === "latency_slo_predicted_miss"); + assert.equal(decision(store, choices, "reject", false, true, 10).action, "pass"); +}); + +test("invalid configuration disables once with no setting values", () => { + const warnings = []; + for (const changes of [{ LATENCY_SLO_MODE: "private-invalid" }, { LATENCY_SLO_MIN_SAMPLES: "0" }, + { LATENCY_SLO_MAX_MODELS: "513" }, { LATENCY_SLO_POLICY_JSON: "[]" }, + { LATENCY_SLO_POLICY_JSON: '{"routes":{"x":{"deadline_ms":true}}}' }, + { LATENCY_SLO_POLICY_JSON: '{"routes":{"x":{"deadline_ms":10,"extra":1}}}' }]) { + for (let i = 0; i < 2; i++) assert.equal(latencySLOSettings(env("reject", false, changes), text => warnings.push(text)).mode, "off"); + } + assert.equal(warnings.length, 4); + assert.ok(!warnings.join().includes("private-invalid")); +}); + +test("tightest route and verified key opt-in; empty means defaults", () => { + assert.equal(latencySLOSettings({ LATENCY_SLO_MODE: "", LATENCY_SLO_MIN_SAMPLES: "" }).mode, "off"); + const settings = latencySLOSettings(env("reject", false, { LATENCY_SLO_POLICY_JSON: JSON.stringify({ + routes: { "auto:test": { deadline_ms: 5000 } }, keys: { caller: { deadline_ms: 1000, require_coverage: true } } }) })); + assert.deepEqual(settings.ruleFor({ route: "auto:test", keyId: "caller" }), { deadline_ms: 1000, require_coverage: true }); + assert.equal(settings.ruleFor({ route: "other" }), null); +}); + +test("native callback rejects before handoff, reports coverage and never rewrites request", async () => { + const store = new ObservationWindow(); measured(store); + const request = new Request("https://example.invalid/v1/chat/completions", { method: "POST", body: JSON.stringify({ model: a, max_tokens: 100 }) }); + let handoffs = 0; + const decision = await prepareLatencySLO(request, env(), { authenticated: true, model: a, keyId: "caller", observations: store, now }); + if (!decision.response) handoffs++; + assert.equal(handoffs, 0); + assert.equal(decision.response.status, 503); + const payload = await decision.response.json(); + assert.equal(payload.error.code, "latency_slo_predicted_miss"); + assert.equal(payload.error.prediction.observation_age_ms, 0); + assert.equal(await request.text(), JSON.stringify({ model: a, max_tokens: 100 })); +}); + +test("off, unverified authority and passthrough do not read request or call collaborators", async () => { + const request = { clone() { throw new Error("must not read"); } }; + assert.equal(await prepareLatencySLO(request, env("off"), { authenticated: true }), null); + assert.equal(await prepareLatencySLO(request, env(), { authenticated: false }), null); + assert.equal(await prepareLatencySLO(request, env(), { authenticated: true, passthrough: true }), null); +}); + +test("native observation adapter requires measured completed non-cache output", () => { + const store = new ObservationWindow(); + const event = { provider: "openai", model: "slow", ttft_ms: 1000, duration_ms: 2000, + output_tokens: 100, outcome: "success", usage_basis: "measured", cost_basis: "usage" }; + assert.equal(recordLatencyObservation(event, env("off"), { observations: store, now }), false); + for (const changes of [{ usage_basis: "estimated" }, { outcome: "unknown" }, { cost_basis: "cache" }, { output_tokens: null }]) { + assert.equal(recordLatencyObservation({ ...event, ...changes }, env(), { observations: store, now }), false); + } + for (let i = 0; i < 20; i++) assert.equal(recordLatencyObservation(event, env(), { observations: store, now }), true); + assert.equal(store.predict(a, 100, { minSamples: 20, now }).predicted_ms, 2000); +}); + +test("duplicate, null and invalid policy scopes disable admission", () => { + for (const policy of ['{"routes":{},"routes":{}}', '{"routes":null}', '{"keys":[]}', + '{"routes":{"x":{"deadline_ms":10,"deadline_ms":20}}}', '{"routes":{"x":{"deadline_ms":10,"require_coverage":null}}}']) { + assert.equal(latencySLOSettings(env("reject", false, { LATENCY_SLO_POLICY_JSON: policy }), () => {}).mode, "off"); + } +}); + +test("remaining generation deadline is checked and never extended", async () => { + const store = new ObservationWindow(); measured(store, a, 100); + const request = new Request("https://example.invalid/v1/chat/completions", { method: "POST", body: JSON.stringify({ model: a, max_tokens: 100 }) }); + let checked = 0; + const deadline = { check() { checked++; }, remainingMs() { return 500; } }; + const result = await prepareLatencySLO(request, env(), { authenticated: true, model: a, observations: store, now, deadline }); + assert.equal(result.response.status, 503); assert.equal(checked, 1); assert.equal(deadline.remainingMs(), 500); + const expired = new Error("generation_deadline_exceeded"); + await assert.rejects(() => prepareLatencySLO(request, env(), { authenticated: true, model: a, deadline: { check() { throw expired; } } }), error => error === expired); +}); + +test("unmatched policy and malformed payload preserve ordinary validation", async () => { + const request = new Request("https://example.invalid/v1/messages", { method: "POST", body: "invalid" }); + assert.equal(await prepareLatencySLO(request, env(), { authenticated: true }), null); + const invalid = new Request("https://example.invalid/v1/chat/completions", { method: "POST", body: "invalid" }); + assert.equal(await prepareLatencySLO(invalid, env(), { authenticated: true }), null); +}); diff --git a/tests/test_wave4_registrar.py b/tests/test_wave4_registrar.py index 9663de05..a4d819d0 100644 --- a/tests/test_wave4_registrar.py +++ b/tests/test_wave4_registrar.py @@ -111,7 +111,7 @@ def test_named_hook_order_and_replay_bypass_all_work(harness): enable() assert [hook.__name__ for hook in app.extensions["gateway_after_authentication"]] == [ "request_policy_hook", "prompt_injection_request_hook", "spillover_hook", "pii_request_hook", - "responses_state_hook", "generation_deadline_hook", "idempotency_request_hook", "admit"] + "responses_state_hook", "generation_deadline_hook", "latency_slo_request_hook", "idempotency_request_hook", "admit"] client = app.test_client() headers = {"Idempotency-Key": "one"} first = client.post("/v1/chat/completions", json=BODY, headers=headers) diff --git a/tests/test_wave5_registrar.py b/tests/test_wave5_registrar.py index 9fc55a3e..0cc94023 100644 --- a/tests/test_wave5_registrar.py +++ b/tests/test_wave5_registrar.py @@ -12,7 +12,7 @@ from tests import unified_api_test_case as fixtures ORDER = ["request_policy_hook", "prompt_injection_request_hook", "spillover_hook", "pii_request_hook", - "responses_state_hook", "generation_deadline_hook", "idempotency_request_hook", "admit"] + "responses_state_hook", "generation_deadline_hook", "latency_slo_request_hook", "idempotency_request_hook", "admit"] FLAGS = ("HOSTED_RESPONSES_ENABLED", "GATEWAY_BATCHES_ENABLED", "BATCH_SPILLOVER_ENABLED", "CONTEXT_PAGING_ENABLED", "SEMANTIC_CACHE_ENABLED", "CANARY_TRAFFIC_ENABLED", "BANDIT_MODE", "PII_REDACTION_ENABLED", "PROMPT_INJECTION_MODE", "POOL_RESET_SCHEDULING_ENABLED", diff --git a/worker/container-env.mjs b/worker/container-env.mjs index a27d53fb..a73141a4 100644 --- a/worker/container-env.mjs +++ b/worker/container-env.mjs @@ -40,6 +40,10 @@ const DIRECT_ENV_KEYS = [ "CONNECTION_PROFILES_DB_PATH", "CONFIG_REVISION_SYNC_ENABLED", "CONFIG_SYNC_TTL_SECONDS", + "LATENCY_SLO_MODE", + "LATENCY_SLO_MIN_SAMPLES", + "LATENCY_SLO_MAX_MODELS", + "LATENCY_SLO_POLICY_JSON", "BANDIT_MODE", "BATCH_SPILLOVER_ENABLED", "CANARY_TRAFFIC_ENABLED", diff --git a/worker/latency-slo.mjs b/worker/latency-slo.mjs new file mode 100644 index 00000000..20f2c189 --- /dev/null +++ b/worker/latency-slo.mjs @@ -0,0 +1,188 @@ +/** Bounded measured latency admission. Candidate eligibility remains the caller's authority. */ +const WINDOW_MS = 900000, MAX_SAMPLES = 1000, MAX_OUTPUT = 131072; +const IDENTIFIER = /^[A-Za-z0-9._:/@+-]{1,256}$/; +const warned = new Set(); +const integer = (value, min, max) => Number.isInteger(value) && value >= min && value <= max; +const number = (value, min = 0, max = 1e13) => typeof value === "number" && Number.isFinite(value) && value >= min && value <= max; +const object = value => value !== null && typeof value === "object" && !Array.isArray(value); + +function warnOnce(name, warn) { + if (!warned.has(name)) { warned.add(name); warn(`Invalid ${name}; latency SLO admission disabled`); } +} + +function uniqueJSON(raw) { + const value = JSON.parse(raw), stack = []; + // JSON.parse validates syntax; this pass rejects duplicate object keys before normalization. + for (const token of raw.match(/"(?:\\.|[^"\\])*"|[{}\[\]:,]/g) ?? []) { + if (token === "{" || token === "[") stack.push({ object: token === "{", keys: new Set(), key: token === "{" }); + else if (token === "}" || token === "]") stack.pop(); + else if (token === ",") { if (stack.at(-1)?.object) stack.at(-1).key = true; } + else if (token === ":") { if (stack.at(-1)) stack.at(-1).key = false; } + else if (token.startsWith('"') && stack.at(-1)?.object && stack.at(-1).key) { + const key = JSON.parse(token), current = stack.at(-1); + if (current.keys.has(key)) throw new Error("Duplicate policy key"); + current.keys.add(key); current.key = false; + } + } + return value; +} + +function parsePolicy(raw) { + if (new TextEncoder().encode(raw).length > 32768) throw new Error("Policy too large"); + const config = uniqueJSON(raw); + if (!object(config) || Object.keys(config).some(key => !["routes", "keys"].includes(key))) throw new Error("Invalid policy"); + const rules = {}; + for (const scope of ["routes", "keys"]) { + const entries = Object.hasOwn(config, scope) ? config[scope] : {}; + if (!object(entries) || Object.keys(entries).length > 128) throw new Error("Invalid scope"); + rules[scope] = new Map(); + for (const [name, rule] of Object.entries(entries)) { + if (!IDENTIFIER.test(name) || !object(rule) || Object.keys(rule).some(key => !["deadline_ms", "require_coverage"].includes(key)) + || !integer(rule.deadline_ms, 1, 300000) || "require_coverage" in rule && typeof rule.require_coverage !== "boolean") throw new Error("Invalid rule"); + rules[scope].set(name, { deadline_ms: rule.deadline_ms, require_coverage: rule.require_coverage ?? false }); + } + } + return rules; +} + +function settingsObject(mode = "off", minSamples = 20, maxModels = 128, rules = {}) { + return { mode, minSamples, maxModels, ruleFor({ route = "", keyId = "", model = "" } = {}) { + const selected = [rules.routes?.get(route), rules.routes?.get(model), rules.keys?.get(keyId)].filter(Boolean); + return selected.length ? { deadline_ms: Math.min(...selected.map(rule => rule.deadline_ms)), + require_coverage: selected.some(rule => rule.require_coverage) } : null; + } }; +} + +export function latencySLOSettings(env = {}, warn = text => console.warn(text)) { + const mode = String(env.LATENCY_SLO_MODE ?? "").trim().toLowerCase() || "off"; + if (mode === "off") return settingsObject(); + if (!["reject", "reroute"].includes(mode)) { warnOnce("LATENCY_SLO_MODE", warn); return settingsObject(); } + const limits = []; + for (const [name, fallback, maximum] of [["LATENCY_SLO_MIN_SAMPLES", 20, 1000], ["LATENCY_SLO_MAX_MODELS", 128, 512]]) { + const raw = String(env[name] ?? "").trim() || String(fallback); + if (!/^[0-9]{1,4}$/.test(raw) || !integer(Number(raw), 1, maximum)) { warnOnce(name, warn); return settingsObject(); } + limits.push(Number(raw)); + } + try { return settingsObject(mode, ...limits, parsePolicy(String(env.LATENCY_SLO_POLICY_JSON ?? "").trim() || "{}")); } + catch { warnOnce("LATENCY_SLO_POLICY_JSON", warn); return settingsObject(); } +} + +export function requestedOutput(payload, fallback = 1024) { + const values = ["max_tokens", "max_completion_tokens"].filter(name => Object.hasOwn(payload, name)).map(name => payload[name]); + if (!values.length) values.push(fallback); + return values.every(value => integer(value, 1, MAX_OUTPUT)) ? Math.min(...values) : null; +} + +export class ObservationWindow { + constructor({ maxModels = 128 } = {}) { + if (!integer(maxModels, 1, 512)) throw new Error("Invalid model cap"); + this.maxModels = maxModels; this.models = new Map(); + } + record(model, { ttft_ms, tokens_per_second, now = Date.now(), maxModels = this.maxModels }) { + if (typeof model !== "string" || !IDENTIFIER.test(model) || !number(now) + || !number(ttft_ms, 0, 3600000) || !number(tokens_per_second, 1e-9, 1000000) || !integer(maxModels, 1, 512)) return false; + for (const [name, samples] of this.models) { + const recent = samples.filter(sample => sample[0] >= now - WINDOW_MS); + if (recent.length) this.models.set(name, recent); else this.models.delete(name); + } + const samples = this.models.get(model) ?? []; + samples.push([now, ttft_ms, tokens_per_second]); + if (samples.length > MAX_SAMPLES) samples.splice(0, samples.length - MAX_SAMPLES); + this.models.delete(model); this.models.set(model, samples); + while (this.models.size > maxModels) this.models.delete(this.models.keys().next().value); + return true; + } + predict(model, outputTokens, { minSamples = 20, now = Date.now() } = {}) { + const samples = this.models.get(model) ?? [], recent = samples.filter(sample => now - sample[0] >= 0 && now - sample[0] <= WINDOW_MS); + const age = samples.length ? Math.max(0, Math.round(now - Math.max(...samples.map(sample => sample[0])))) : null; + const prediction = { status: "prediction_unknown", predicted_ms: null, samples: recent.length, + min_samples: minSamples, coverage: Math.min(1, recent.length / minSamples), observation_age_ms: age, + window_ms: WINDOW_MS, output_tokens: outputTokens }; + if (recent.length >= minSamples && integer(outputTokens, 1, MAX_OUTPUT)) { + const times = recent.map(([, ttft, rate]) => ttft + outputTokens * 1000 / rate).sort((a, b) => a - b); + prediction.status = "prediction_known"; prediction.predicted_ms = Math.ceil(times[Math.ceil(.95 * times.length) - 1]); + } + return prediction; + } +} + +export const observations = new ObservationWindow(); + +export class LatencySLOError extends Error { + constructor(code, prediction) { + super(code === "latency_slo_unavailable" ? "Recent latency coverage is unavailable" : "Predicted completion exceeds the latency deadline"); + this.code = code; this.prediction = prediction; + } + response() { return Response.json({ error: { code: this.code, message: this.message, retryable: false, prediction: this.prediction } }, + { status: 503, headers: { "Cache-Control": "no-store" } }); } +} + +export function admitCandidates(candidates, { settings, rule, outputTokens, auto, observations: store = observations, now = Date.now() }) { + if (settings.mode === "off" || !rule) return { candidates, action: "pass", prediction: null }; + const primary = candidates[0] ?? {}, prediction = store.predict(primary.model, outputTokens, { minSamples: settings.minSamples, now }); + if (prediction.status === "prediction_unknown") { + if (rule.require_coverage) throw new LatencySLOError("latency_slo_unavailable", prediction); + return { candidates, action: "pass", prediction }; + } + if (prediction.predicted_ms <= rule.deadline_ms) return { candidates, action: "pass", prediction }; + if (settings.mode === "reroute" && auto && integer(primary.quality_tier, 0, 100)) { + const safe = candidates.slice(1).filter(candidate => { + if (!integer(candidate.quality_tier, 0, 100) || candidate.quality_tier !== primary.quality_tier) return false; + const predicted = store.predict(candidate.model, outputTokens, { minSamples: settings.minSamples, now }); + return predicted.status === "prediction_known" && predicted.predicted_ms <= rule.deadline_ms; + }); + if (safe.length) return { candidates: safe, action: "reroute", prediction: store.predict(safe[0].model, outputTokens, { minSamples: settings.minSamples, now }) }; + } + throw new LatencySLOError("latency_slo_predicted_miss", prediction); +} + +async function boundedPayload(request) { + if (!request.body) return null; + const reader = request.clone().body.getReader(), parts = []; let size = 0; + try { + while (true) { + const { value, done } = await reader.read(); if (done) break; + size += value.byteLength; if (size > 1048576) return null; + parts.push(value); + } + const bytes = new Uint8Array(size); let offset = 0; + for (const part of parts) { bytes.set(part, offset); offset += part.length; } + const payload = JSON.parse(new TextDecoder().decode(bytes)); + return object(payload) ? payload : null; + } catch { return null; } + finally { void reader.cancel().catch(() => {}); } +} + +export async function prepareLatencySLO(request, env, authority = {}) { + const settings = latencySLOSettings(env); + if (settings.mode === "off" || authority.authenticated !== true || authority.passthrough) return null; + const route = authority.route ?? new URL(request.url).pathname; + const rule = settings.ruleFor({ route, keyId: authority.keyId, model: authority.routeId }); + if (!rule) return null; + authority.deadline?.check(); + const payload = authority.payload ?? await boundedPayload(request); + if (!payload) return null; + const model = authority.model ?? payload.model; + if (typeof model !== "string" || !IDENTIFIER.test(model)) return null; + const auto = model.startsWith("auto:"), candidates = auto ? authority.candidates ?? [] : [{ model }]; + const remaining = authority.deadline?.remainingMs(); + const effective = remaining === undefined ? rule : { ...rule, deadline_ms: Math.min(rule.deadline_ms, remaining) }; + try { + return admitCandidates(candidates, { settings, rule: effective, outputTokens: requestedOutput(payload), auto, + observations: authority.observations ?? observations, now: authority.now ?? Date.now() }); + } catch (error) { + if (!(error instanceof LatencySLOError)) throw error; + return { response: error.response(), action: "reject", prediction: error.prediction }; + } +} + +export function recordLatencyObservation(event, env, { observations: store = observations, now = Date.now() } = {}) { + const settings = latencySLOSettings(env); + if (settings.mode === "off" || event.outcome !== "success" || event.usage_basis !== "measured" + || event.cost_basis === "cache" || event.cacheServed || !integer(event.output_tokens, 1, 2147483647) + || !number(event.ttft_ms, 0, 3600000) || !number(event.duration_ms) || event.duration_ms <= event.ttft_ms + || typeof event.provider !== "string" || typeof event.model !== "string") return false; + const model = event.model.startsWith(event.provider + ":") ? event.model : `${event.provider}:${event.model}`; + return store.record(model, { ttft_ms: event.ttft_ms, tokens_per_second: event.output_tokens * 1000 / (event.duration_ms - event.ttft_ms), + now, maxModels: settings.maxModels }); +}