From 22d205ca74b916d2196a1131629b92d0bb3878eb Mon Sep 17 00:00:00 2001 From: Abhyudaya Baiju Tharakan Date: Fri, 9 Oct 2026 05:51:26 +0530 Subject: [PATCH] feat(routing): add opt-in outcome-bound prompt cache affinity Prefer the credential that actually completed a reusable prefix on authenticated managed automatic routes behind PROMPT_CACHE_AFFINITY_ENABLED (default off). Bindings come only from classified successful attempts, are hash-only and bounded by TTL and LRU, and never override cooldown, grants, budgets or fallback eligibility. --- docs/prompt-cache-affinity.md | 60 +++++ routes/unified.py | 13 +- services/credential_pool.py | 11 +- services/managed_dispatch.py | 174 ++++++++++++++ services/opencode_session.py | 19 ++ services/prompt_cache_affinity.py | 283 +++++++++++++++++++++++ tests/test_prompt_cache_affinity.py | 340 ++++++++++++++++++++++++++++ worker/container-env.mjs | 2 + 8 files changed, 897 insertions(+), 5 deletions(-) create mode 100644 docs/prompt-cache-affinity.md create mode 100644 services/prompt_cache_affinity.py create mode 100644 tests/test_prompt_cache_affinity.py diff --git a/docs/prompt-cache-affinity.md b/docs/prompt-cache-affinity.md new file mode 100644 index 00000000..8503ae28 --- /dev/null +++ b/docs/prompt-cache-affinity.md @@ -0,0 +1,60 @@ +# Outcome-bound prompt cache affinity + +Prompt cache affinity is disabled by default. Enable it for authenticated managed +automatic text routes with: + +```sh +PROMPT_CACHE_AFFINITY_ENABLED=true +PROMPT_CACHE_AFFINITY_TTL_SECONDS=900 +``` + +Disable it with `PROMPT_CACHE_AFFINITY_ENABLED=false`. An empty value uses the +default. TTL defaults to 900 seconds and accepts integers from 1 to 86400. +Malformed settings disable affinity and produce one warning naming the setting +without its value. Disabling affinity preserves request and response bytes, +headers, credential order, and retry behavior. + +The managed Chat Completions route and automatic aliases translated through +Responses or Messages observe the credential that actually served an attempt. +A successful complete response binds that provider/model and credential. Stream +headers and early content do not bind: the observer waits for terminal protocol +evidence and normal exhaustion. Failed, incomplete, interrupted, and cancelled +responses evict only the matching binding. Affinity adds no retry permission. +Explicit models, raw proxy paths, and native roleplay keep their existing behavior. +This integration covers stored ordered automatic aliases. The intelligence +gateway uses a separate transport and does not consume this request scope. + +Credential preference only considers keys that the pool already allows. Model +cooldown, credential rest, grants, budgets, context capability, and fallback +eligibility remain authoritative. Ordered automatic routes retain their configured +provider order. A tier-aware scheduler can call `AffinityScope.order_candidates` +with its already eligible ordered models and approved tier map. Preference can +only move a successful candidate within the first eligible tier. Missing tier +evidence retains the original order. This interface does not authorize candidates +or retries, and it never crosses a fallback tier during tool loops. + +Bindings are isolated by authenticated tenant/principal, session, role lane, +automatic route, reusable opening prefix, tool schema, and policy revision. +Changing route configuration or the supplied policy revision invalidates an old +binding. Credential removal and cooldown invalidate its key preference. Explicit +session IDs do not replace prefix checks; subsequent turns and tool results keep +the opening identity. The final observer captures an opaque credential fingerprint. +No prompt, response, principal, session ID, or credential plaintext is persisted +in the affinity map or logged by this feature. + +Storage is a process-local LRU capped at 10000 entries. TTL is monotonic and is +not extended by reads. Expired entries are removed on access and insertion. A +restart loses bindings; multiple Containers have independent maps. Cross-instance +affinity is best effort, and there is no D1 migration or distributed consistency +claim. The bounded stream parser ignores oversized lines and cannot bind a JSON +body larger than 1 MiB without other completion evidence. Unknown completion +evidence declines to bind; it never fabricates a success body or changes the +original error returned to the client. + +The observer consumes provider cache-read usage through the existing prompt cache +usage buckets. Counts remain measured when the provider reports them. Read cost +and rebuilding the same tokens at ordinary input prices are explicitly named +estimates derived from configured catalog prices; absent counts or prices remain +unknown. These observations do not prove savings on every request, do not change +budget accounting, and never update routing policy automatically. Provider cache +retention and billing still follow that provider's contract. diff --git a/routes/unified.py b/routes/unified.py index 1c20b653..55dee4ed 100644 --- a/routes/unified.py +++ b/routes/unified.py @@ -89,6 +89,8 @@ from services.rate_limit_service import RateLimitService from services.reasoning_policy import apply_glm_5_reasoning_policy from services.transport_policy import RAW_PASSTHROUGH_PROVIDERS +from services.prompt_cache_affinity import affinity_auto_request, affinity_candidate +from services.managed_dispatch import execute_managed_attempt, observe_managed_response logger = logging.getLogger(__name__) @@ -342,7 +344,7 @@ def send_request(token: str): request_kwargs["force_raw_passthrough"] = True if request_timeout is not None: request_kwargs["timeout_override"] = bounded_timeout(request_timeout) - return proxy_service_cls.make_request(**request_kwargs) + return execute_managed_attempt(lambda: proxy_service_cls.make_request(**request_kwargs), f"{provider}:{provider_model}", token) return _request_with_provider_token_rotation( app, @@ -413,6 +415,7 @@ def _dispatch_bridged_chat( target=CHAT, stream=bool(payload.get("stream")), ) + observe_managed_response(downstream, response, stream=bool(payload.get("stream"))) return _add_credential_attempt_headers( _add_prompt_cache_headers(downstream, cache_decision), provider, @@ -420,6 +423,7 @@ def _dispatch_bridged_chat( ) +@affinity_candidate @with_chat_tool_repair def _dispatch_unified_chat_candidate( app, @@ -556,14 +560,14 @@ def send_request(token: str): } if request_timeout is not None: request_kwargs["timeout_override"] = bounded_timeout(request_timeout) - return send_configured_unified_provider_request( + return execute_managed_attempt(lambda: send_configured_unified_provider_request( proxy_service_cls, request_kwargs, provider=provider, runtime_config=app.config, upstream_path=upstream_path, request_headers=headers_source, - ) + ), f"{provider}:{provider_model}", token) response, credential_attempts = _request_with_provider_token_rotation( app, @@ -599,6 +603,7 @@ def send_request(token: str): ) if isinstance(response, Response): + observe_managed_response(response, response, stream=bool(payload.get("stream"))) return _add_credential_attempt_headers( _add_prompt_cache_headers( _add_adaptive_context_headers(response, adaptive_result), @@ -617,6 +622,7 @@ def send_request(token: str): headers=copy_raw_provider_response_headers(response.headers), ) mark_transport_failure(downstream_response, response) + observe_managed_response(downstream_response, response, stream=bool(payload.get("stream"))) return _add_credential_attempt_headers( _add_prompt_cache_headers( _add_adaptive_context_headers( @@ -643,6 +649,7 @@ def send_request(token: str): @sample_chat_dispatch +@affinity_auto_request def dispatch_unified_chat_completion( app, auth_service_cls, diff --git a/services/credential_pool.py b/services/credential_pool.py index e8a017e8..574bb38b 100644 --- a/services/credential_pool.py +++ b/services/credential_pool.py @@ -117,7 +117,7 @@ def select( if config.enabled and (model or quota_bucket): with cls._lock: legacy = {key: cls._resting.get((provider, key), 0) for key in keys} - return model_cooldown.select( + selected = model_cooldown.select( provider, keys, model=model, @@ -126,7 +126,14 @@ def select( now=now, legacy_rest_until=legacy, ) - return next(iter(cls.available(provider, keys, now=now) or keys), None) + from services.prompt_cache_affinity import prefer_credential + preferred = prefer_credential(provider, model, cls.available( + provider, keys, model=model, quota_bucket=quota_bucket, now=now, + )) + return preferred or selected + available = cls.available(provider, keys, now=now) + from services.prompt_cache_affinity import prefer_credential + return prefer_credential(provider, model, available) or next(iter(available or keys), None) @classmethod def record( diff --git a/services/managed_dispatch.py b/services/managed_dispatch.py index 99911a54..e9f198eb 100644 --- a/services/managed_dispatch.py +++ b/services/managed_dispatch.py @@ -1,6 +1,7 @@ """Managed attempt scheduling, separate from raw dispatch and normalization.""" import math +import json import time from typing import Callable @@ -10,11 +11,184 @@ from services.retry_advice import parse_retry_advice, retry_advice_settings from services.upstream_outcome import UpstreamOutcome from services.upstream_transport import close_retry_response +from services.upstream_outcome import classify_upstream_outcome Attempt = Callable[[int], requests.Response] Outcome = Callable[[], UpstreamOutcome] +def execute_managed_attempt(send, model, credential): + """Capture the actual dispatch credential; headers alone never bind affinity.""" + from services.prompt_cache_affinity import current_scope + scope = current_scope() + if scope is None: + return send() + try: + response = send() + except BaseException: + scope.observer(model, credential)(classify_upstream_outcome(transport_failure="interrupted")) + raise + return capture_managed_attempt(response, model, credential) + + +def capture_managed_attempt(response, model, credential): + from services.prompt_cache_affinity import current_scope + scope = current_scope() + if scope is not None: + observer = scope.observer(model, credential) + if not 200 <= response.status_code < 300: + observer(classify_upstream_outcome(response.status_code)) + else: + response.multillm_affinity_observer = observer + return response + + +def _completed_body(body): + if not isinstance(body, dict) or body.get("error"): + return False + if body.get("type") in {"error", "response.failed", "response.incomplete"}: + return False + if body.get("status") in {"failed", "incomplete", "cancelled", "in_progress"}: + return False + choices = body.get("choices") + if isinstance(choices, list) and choices: + return all(isinstance(choice, dict) and choice.get("finish_reason") is not None for choice in choices) + return body.get("status") == "completed" or ( + body.get("type") == "message" and body.get("stop_reason") is not None) + + +class ManagedOutcomeIterator: + """Observe final bytes without changing them or granting another dispatch.""" + + def __init__(self, source, callback, *, event_stream, context=None): + from services.prompt_cache_cost import CacheStreamObserver + self.source = iter(source) + self.callback = callback + self.event_stream = event_stream + self.context = context + self.usage = CacheStreamObserver(65536) + self.buffer = b"" + self.oversized = False + self.complete = False + self.failed = False + self.closed = False + + def __iter__(self): + return self + + def _event(self, line): + if line.startswith(b"event:"): + self.failed |= line[6:].strip() in {b"error", b"response.failed", b"response.incomplete"} + if not line.startswith(b"data:"): + return + data = line[5:].strip() + if data == b"[DONE]": + self.complete = True + return + try: + body = json.loads(data) + except (ValueError, RecursionError): + return + if not isinstance(body, dict): + return + self.failed |= bool(body.get("error")) or body.get("type") in { + "error", "response.failed", "response.incomplete"} + self.complete |= _completed_body(body) or body.get("type") in {"message_stop", "response.completed"} + + def _feed(self, chunk): + self.usage.feed(chunk) + if not self.event_stream: + if len(self.buffer) + len(chunk) <= 1024 * 1024 and not self.oversized: + self.buffer += chunk + else: + self.buffer, self.oversized = b"", True + return + for index, part in enumerate(chunk.split(b"\n")): + if index: + if not self.oversized: + self._event(self.buffer.strip()) + self.buffer, self.oversized = b"", False + if len(self.buffer) + len(part) <= 65536 and not self.oversized: + self.buffer += part + else: + self.buffer, self.oversized = b"", True + + def _finish(self, outcome): + if self.closed: + return + self.closed = True + owned = getattr(self.context, "outcome", None) + if owned is not None and owned.reason != "complete": + outcome = owned.upstream + self.callback(outcome, self.usage.finish()) + + def _exhausted(self): + if self.event_stream and not self.oversized: + self._event(self.buffer.strip()) + if not self.event_stream and not self.oversized: + try: + body = json.loads(self.buffer) + self.complete = _completed_body(body) + from services.prompt_cache_cost import CacheObservation + self.usage.observation = CacheObservation.from_body(body) + except (ValueError, RecursionError): + self.complete = False + outcome = classify_upstream_outcome(200) if self.complete and not self.failed else classify_upstream_outcome( + transport_failure="interrupted") + self._finish(outcome) + + def __next__(self): + if self.closed: + raise StopIteration + try: + chunk = next(self.source) + except StopIteration: + self._exhausted() + raise + except BaseException: + self._finish(classify_upstream_outcome(transport_failure="interrupted")) + raise + self._feed(chunk.encode("utf-8") if isinstance(chunk, str) else chunk) + return chunk + + def close(self): + # The existing Flask owner closes the body while reporting upstream EOF. + owned = getattr(self.context, "outcome", None) + if owned is not None and owned.reason == "complete": + self._exhausted() + else: + self._finish(classify_upstream_outcome(cancelled=True)) + close = getattr(self.source, "close", None) + if close is not None: + close() + + +def observe_managed_response(downstream, upstream, *, stream=False): + """Deliver classified final outcomes through the managed-dispatch callback.""" + callback = getattr(upstream, "multillm_affinity_observer", None) + if callback is None: + return downstream + if not 200 <= downstream.status_code < 300: + callback(classify_upstream_outcome(downstream.status_code)) + return downstream + event_stream = stream or downstream.mimetype == "text/event-stream" + if event_stream or downstream.is_streamed: + observer = ManagedOutcomeIterator(downstream.response, callback, event_stream=event_stream, + context=getattr(upstream, "multillm_cancellation", None)) + downstream.response = observer + downstream.call_on_close(observer.close) + else: + from services.prompt_cache_cost import CacheObservation + try: + body = json.loads(downstream.get_data()) + except (ValueError, RecursionError): + body = None + classified = classify_upstream_outcome(200) if _completed_body(body) else classify_upstream_outcome( + transport_failure="interrupted") + callback(classified, CacheObservation.from_body(body)) + return downstream + + def retry_managed_attempt( attempt: Attempt, outcome: Outcome, diff --git a/services/opencode_session.py b/services/opencode_session.py index 2702547d..024a8be7 100644 --- a/services/opencode_session.py +++ b/services/opencode_session.py @@ -83,3 +83,22 @@ def with_opencode_request_session( session = f"multillm_request_{uuid.uuid4()}" result["X-Opencode-Session"] = session return result + + +def reusable_prefix_digest(payload) -> str | None: + """Hash the opening and tool schema separately from a caller's session ID.""" + if not isinstance(payload, dict): + return None + opening_payload = {key: payload[key] for key in ("messages", "input", "system", "instructions") if key in payload} + anchor = _conversation_anchor(opening_payload) + if anchor is None: + return None + try: + encoded = json.dumps([anchor, payload.get("tools"), payload.get("tool_choice")], + sort_keys=True, ensure_ascii=False, separators=(",", ":"), + allow_nan=False).encode("utf-8") + except (ValueError, TypeError, UnicodeError, RecursionError): + return None + if len(encoded) > MAX_SESSION_BODY_BYTES: + return None + return hashlib.sha256(encoded).hexdigest() diff --git a/services/prompt_cache_affinity.py b/services/prompt_cache_affinity.py new file mode 100644 index 00000000..77315da8 --- /dev/null +++ b/services/prompt_cache_affinity.py @@ -0,0 +1,283 @@ +"""Bounded outcome affinity; eligibility and fallback remain caller-owned.""" +from __future__ import annotations + +import hashlib +import json +import logging +import os +import threading +import time +from collections import OrderedDict +from contextlib import contextmanager +from contextvars import ContextVar +from dataclasses import dataclass, replace +from functools import wraps +from typing import Callable + +from flask import current_app, g, has_request_context, request + +from services.opencode_session import reusable_prefix_digest +from services.upstream_outcome import UpstreamOutcome + +logger = logging.getLogger(__name__) +MAX_ENTRIES = 10_000 +_warned: set[str] = set() +_active: ContextVar[AffinityScope | None] = ContextVar("prompt_cache_affinity", default=None) +_candidate: ContextVar[str | None] = ContextVar("prompt_cache_candidate", default=None) + + +def _digest(value) -> str: + return hashlib.sha256(json.dumps(value, sort_keys=True, ensure_ascii=False, + separators=(",", ":"), allow_nan=False).encode()).hexdigest() + + +def _warn(name): + if name not in _warned: + _warned.add(name) + logger.warning("Invalid %s; prompt cache affinity disabled", name) + + +def settings() -> tuple[bool, int]: + flag = os.environ.get("PROMPT_CACHE_AFFINITY_ENABLED", "").strip().lower() + if flag in {"", "false", "0", "off", "no"}: + return False, 900 + if flag not in {"true", "1", "on", "yes"}: + _warn("PROMPT_CACHE_AFFINITY_ENABLED") + return False, 900 + raw = os.environ.get("PROMPT_CACHE_AFFINITY_TTL_SECONDS", "").strip() + if raw and (not raw.isascii() or not raw.isdigit() or len(raw) > 5 or not 1 <= int(raw) <= 86400): + _warn("PROMPT_CACHE_AFFINITY_TTL_SECONDS") + return False, 900 + return True, int(raw or "900") + + +@dataclass(frozen=True) +class Binding: + revision: str + model: str + credential: str + expires: float + cache_read_tokens: int | None = None + estimated_cache_read_cost_microusd: float | None = None + estimated_rebuild_cost_microusd: float | None = None + + +def cache_cost_evidence(model, usage): + """Measured cache-read counts; catalogue-derived costs remain estimates.""" + from services.prompt_cache_cost import CacheObservation, price_buckets + if not isinstance(usage, CacheObservation) or usage.basis != "provider": + return None, None, None + if usage.cache_read is None: + return None, None, None + actual = price_buckets(model, usage)["cache_read_input_cost_microusd"] + rebuild = replace(usage, input_tokens=usage.cache_read, output_tokens=0, + raw_input=usage.cache_read, cache_read=0, cache_write=0, source="openai") + estimate = price_buckets(model, rebuild)["ordinary_input_cost_microusd"] + return usage.cache_read, actual, estimate + + +class PromptCacheAffinity: + """Process-local, hash-only LRU. Concurrent finalization is serialized.""" + + def __init__(self, *, clock: Callable[[], float] = time.monotonic, max_entries=MAX_ENTRIES): + self.clock = clock + self.max_entries = min(MAX_ENTRIES, max(1, max_entries)) + self._entries: OrderedDict[str, Binding] = OrderedDict() + self._lock = threading.Lock() + + def _purge(self): + now = self.clock() + for key, value in tuple(self._entries.items()): + if value.expires <= now: + self._entries.pop(key, None) + + def lookup(self, key, revision): + with self._lock: + found = self._entries.get(key) + if found is None: + return None + if found.expires <= self.clock() or found.revision != revision: + self._entries.pop(key, None) + return None + self._entries.move_to_end(key) + return found + + def bind(self, key, revision, model, credential, ttl, evidence=(None, None, None)): + with self._lock: + self._purge() + self._entries[key] = Binding(revision, model, credential, self.clock() + ttl, *evidence) + self._entries.move_to_end(key) + while len(self._entries) > self.max_entries: + self._entries.popitem(last=False) + + def evict(self, key, binding): + with self._lock: + if self._entries.get(key) == binding: + self._entries.pop(key, None) + + def entries(self): + with self._lock: + self._purge() + return tuple(self._entries.items()) + + def __len__(self): + return len(self.entries()) + + +store = PromptCacheAffinity() + + +@dataclass(frozen=True) +class AffinityScope: + storage: PromptCacheAffinity + identity: str + revision: str + ttl: int + + def binding(self): + enabled, ttl = settings() + return self.storage.lookup(self.identity, self.revision) if enabled and ttl == self.ttl else None + + def record(self, model: str, credential: str, outcome: UpstreamOutcome, *, usage=None): + self.observer(model, credential)(outcome, usage) + + def observer(self, model: str, credential: str): + # Capture only an opaque credential fingerprint across lazy streaming. + fingerprint = _digest(credential) + def observed(outcome, usage=None): + enabled, ttl = settings() + if not enabled or ttl != self.ttl: + return + if outcome.provider_health == "success" and outcome.credential_health == "accepted": + self.storage.bind(self.identity, self.revision, model, fingerprint, ttl, + cache_cost_evidence(model, usage)) + else: + found = self.binding() + if found is not None and found.model == model and found.credential == fingerprint: + self.storage.evict(self.identity, found) + return observed + + def preferred_key(self, model: str, eligible: list[str]): + found = self.binding() + if found is None or found.model != model: + return None + for key in eligible: + if _digest(key) == found.credential: + return key + self.storage.evict(self.identity, found) + return None + + def preferred_model(self, eligible: list[str], tiers: dict): + found = self.binding() + if found is None: + return None + if found.model not in eligible: + self.storage.evict(self.identity, found) + return None + # Missing tier evidence preserves ordered fallback semantics. + if not eligible or eligible[0] not in tiers or found.model not in tiers: + return None + return found.model if tiers[found.model] == tiers[eligible[0]] else None + + def order_candidates(self, eligible: list[str], tiers: dict) -> list[str]: + preferred = self.preferred_model(eligible, tiers) + return [preferred, *(model for model in eligible if model != preferred)] if preferred else list(eligible) + + +def make_scope(*, principal, session, role, payload, revision, route) -> AffinityScope | None: + enabled, ttl = settings() + if not enabled or not principal or not role: + return None + prefix = reusable_prefix_digest(payload) + if prefix is None: + return None + try: + identity = _digest([principal, session, role, route, prefix]) + policy = _digest([revision, ttl]) + except (ValueError, TypeError, RecursionError, UnicodeError): + return None + return AffinityScope(store, identity, policy, ttl) + + +def request_scope(payload, config): + """Only authenticated, managed automatic text routes create a scope.""" + if not settings()[0] or not has_request_context(): + return None + if request.path not in {"/v1/chat/completions", "/v1/responses", "/v1/messages"}: + return None + model = payload.get("model") + if not isinstance(model, str) or not model.startswith("auto:"): + return None + user = getattr(g, "authenticated_user", None) + if not isinstance(user, dict) or not (user.get("id") or user.get("username")): + return None + principal = [user.get("tenant_id"), user.get("id") or user.get("username")] + headers = {key.lower(): value for key, value in request.headers.items()} + metadata = payload.get("metadata") + metadata = metadata if isinstance(metadata, dict) else {} + session = next((value for value in (headers.get("x-opencode-session"), headers.get("session-id"), + headers.get("thread-id"), payload.get("session_id"), payload.get("conversation_id"), + metadata.get("session_id"), metadata.get("conversation_id"), user.get("session_id")) + if isinstance(value, str) and value.strip()), "prefix") + role = [user.get("role") or ("admin" if user.get("is_admin") else "user"), + getattr(g, "prompt_cache_affinity_lane", "default")] + from services.auto_route_service import AutoRouteService + route = AutoRouteService.get_route(model) + if route is None: + return None + sync = current_app.extensions.get("config_revision_sync") + revisions = {name: state["revision"] for name, state in sync.status()["domains"].items()} if sync else None + revision = [route.candidates, route.updated_at, getattr(g, "gateway_policy_revision", None), + revisions, + user.get("scopes"), user.get("allowed_models"), config.get("PROMPT_CACHE_ENABLED"), + config.get("PROMPT_CACHE_MIN_TOKENS"), config.get("NANOGPT_SUBSCRIPTION_ONLY")] + return make_scope(principal=principal, session=session, role=role, payload=payload, + revision=revision, route=model) + + +@contextmanager +def activate_scope(scope): + token = _active.set(scope) + try: + yield + finally: + _active.reset(token) + + +@contextmanager +def candidate_scope(model): + token = _candidate.set(model) + try: + yield + finally: + _candidate.reset(token) + + +def current_scope(): + return _active.get() + + +def affinity_auto_request(dispatch): + @wraps(dispatch) + def wrapped(app, auth, metrics, proxy, payload, **kwargs): + if current_scope() is not None: + return dispatch(app, auth, metrics, proxy, payload, **kwargs) + with activate_scope(request_scope(payload, app.config)): + return dispatch(app, auth, metrics, proxy, payload, **kwargs) + return wrapped + + +def affinity_candidate(dispatch): + @wraps(dispatch) + def wrapped(app, auth, metrics, proxy, payload, **kwargs): + with candidate_scope(payload.get("model")): + return dispatch(app, auth, metrics, proxy, payload, **kwargs) + return wrapped + + +def prefer_credential(provider, model, eligible): + scope = current_scope() + candidate = _candidate.get() + if scope is None or candidate is None or candidate.split(":", 1)[0] != provider: + return None + return scope.preferred_key(candidate, eligible) diff --git a/tests/test_prompt_cache_affinity.py b/tests/test_prompt_cache_affinity.py new file mode 100644 index 00000000..070bfc10 --- /dev/null +++ b/tests/test_prompt_cache_affinity.py @@ -0,0 +1,340 @@ +"""Outcome affinity with synthetic credentials and deterministic storage.""" +import json +import importlib +from concurrent.futures import ThreadPoolExecutor +from unittest.mock import patch + +import pytest +from flask import Flask, Response, g + +from services import prompt_cache_affinity as affinity +from services import managed_dispatch +from services.credential_pool import CredentialPool +from services.upstream_outcome import classify_upstream_outcome + +MODEL = "ce-gpt-plus:synthetic" +OTHER = "ce-gpt-pro:synthetic" +PAYLOAD = {"model": "auto:synthetic", "messages": [ + {"role": "system", "content": "private reusable prefix"}, + {"role": "user", "content": "private first turn"}]} + + +@pytest.fixture(autouse=True) +def isolated(monkeypatch, tmp_path): + global affinity, managed_dispatch, CredentialPool + affinity = importlib.import_module("services.prompt_cache_affinity") + managed_dispatch = importlib.import_module("services.managed_dispatch") + CredentialPool = importlib.import_module("services.credential_pool").CredentialPool + monkeypatch.setenv("PROMPT_CACHE_AFFINITY_ENABLED", "true") + monkeypatch.setenv("PROMPT_CACHE_AFFINITY_TTL_SECONDS", "900") + monkeypatch.setenv("MODEL_COOLDOWN_ENABLED", "false") + monkeypatch.setenv("SECRET_SCAN_DEFAULT", "off") + monkeypatch.setenv("AUTH_DB_PATH", str(tmp_path / "auth.sqlite3")) + monkeypatch.setenv("RATE_LIMIT_DB_PATH", str(tmp_path / "limits.sqlite3")) + monkeypatch.setenv("MODEL_REGISTRY_DB_PATH", str(tmp_path / "models.sqlite3")) + monkeypatch.setenv("CODEX_EVERYWHERE_API_KEY_GPT_PLUS", "synthetic-first") + monkeypatch.setattr(CredentialPool, "keys", staticmethod(lambda provider: ["synthetic-first", "synthetic-second"])) + CredentialPool.reset() + monkeypatch.setattr(affinity, "store", affinity.PromptCacheAffinity()) + + +def scope(**changes): + args = dict(principal="alice", session="conversation", role="coding", payload=PAYLOAD, + revision="revision-1", route="auto:synthetic") + args.update(changes) + return affinity.make_scope(**args) + + +def bind(s, model=MODEL, key="synthetic-second", **kwargs): + s.record(model, key, classify_upstream_outcome(200), **kwargs) + + +def test_failure_never_binds_and_failure_evicts_only_matching_candidate(): + s = scope() + s.record(MODEL, "synthetic-second", classify_upstream_outcome(503)) + assert s.preferred_model([MODEL, OTHER], {MODEL: "same", OTHER: "same"}) is None + bind(s, OTHER) + s.record(MODEL, "synthetic-second", classify_upstream_outcome(429)) + assert s.preferred_model([MODEL, OTHER], {MODEL: "same", OTHER: "same"}) == OTHER + s.record(OTHER, "synthetic-second", classify_upstream_outcome(cancelled=True)) + assert s.preferred_model([MODEL, OTHER], {MODEL: "same", OTHER: "same"}) is None + + +def test_actual_successful_fallback_and_approved_tier_order(): + s = scope() + bind(s, OTHER) + assert s.order_candidates([MODEL, OTHER], {MODEL: "same", OTHER: "same"}) == [OTHER, MODEL] + assert s.order_candidates([MODEL, OTHER], {MODEL: "primary", OTHER: "fallback"}) == [MODEL, OTHER] + assert s.order_candidates([MODEL, OTHER], {}) == [MODEL, OTHER] + assert s.order_candidates([MODEL], {MODEL: "same"}) == [MODEL] + assert s.order_candidates([MODEL, OTHER], {MODEL: "same", OTHER: "same"}) == [MODEL, OTHER] + + +@pytest.mark.parametrize("changes", [dict(principal="bob"), dict(role="review"), + dict(session="other"), dict(revision="revision-2"), dict(route="auto:other"), + dict(payload={**PAYLOAD, "tools": [{"type": "function", "function": {"name": "other"}}]}), + dict(payload={**PAYLOAD, "messages": [{"role": "user", "content": "different"}]})]) +def test_principal_session_role_revision_prefix_separation(changes): + bind(scope()) + assert scope(**changes).preferred_key(MODEL, ["synthetic-second"]) is None + + +def test_tool_loop_keeps_prefix_but_respects_tier(): + s = scope() + bind(s, OTHER) + continued = {**PAYLOAD, "messages": PAYLOAD["messages"] + [ + {"role": "assistant", "tool_calls": [{"id": "call"}]}, + {"role": "tool", "content": "result", "tool_call_id": "call"}]} + later = scope(payload=continued) + assert later.preferred_key(OTHER, ["synthetic-second"]) == "synthetic-second" + assert later.order_candidates([MODEL, OTHER], {MODEL: 1, OTHER: 2}) == [MODEL, OTHER] + + +def test_expiry_bounded_hash_only_storage(monkeypatch): + clock = [10.0] + storage = affinity.PromptCacheAffinity(clock=lambda: clock[0], max_entries=2) + monkeypatch.setattr(affinity, "store", storage) + for user in ("alice", "bob", "charlie"): + bind(scope(principal=user)) + assert len(storage) == 2 + assert scope().preferred_key(MODEL, ["synthetic-second"]) is None + state = repr(storage.entries()) + for secret in ("private", "synthetic-second", "alice", "conversation"): + assert secret not in state + clock[0] += 900 + assert scope(principal="charlie").preferred_key(MODEL, ["synthetic-second"]) is None + assert len(storage) == 0 + + +def test_cooldown_and_removed_key_invalidation(): + s = scope() + bind(s) + with affinity.activate_scope(s), affinity.candidate_scope(MODEL): + assert CredentialPool.select("ce-gpt-plus", now=10) == "synthetic-second" + CredentialPool.record("ce-gpt-plus", "synthetic-second", 429, now=10) + assert CredentialPool.select("ce-gpt-plus", now=11) == "synthetic-first" + assert s.preferred_key(MODEL, ["synthetic-second"]) is None + + +@pytest.mark.parametrize("flag,ttl", [("", ""), ("false", "900"), ("bad-secret", "900"), + ("true", "bad-secret"), ("true", "0")]) +def test_disabled_invalid_unchanged(flag, ttl, monkeypatch, caplog): + monkeypatch.setenv("PROMPT_CACHE_AFFINITY_ENABLED", flag) + monkeypatch.setenv("PROMPT_CACHE_AFFINITY_TTL_SECONDS", ttl) + assert scope() is None + assert CredentialPool.select("ce-gpt-plus") == "synthetic-first" + assert "bad-secret" not in caplog.text + + +def observed_response(s, body, *, status=200, stream=False, model=MODEL): + with affinity.activate_scope(s), affinity.candidate_scope(model): + upstream = Response(body, status=status, + mimetype="text/event-stream" if stream else "application/json") + managed_dispatch.capture_managed_attempt(upstream, model, "synthetic-second") + return managed_dispatch.observe_managed_response(upstream, upstream, stream=stream) + + +def test_nonstream_success_requires_classified_completed_response(): + s = scope() + response = observed_response(s, json.dumps({"choices": [{"message": {"content": "answer"}, "finish_reason": "stop"}]})) + assert response.status_code == 200 + assert s.preferred_key(MODEL, ["synthetic-second"]) == "synthetic-second" + + +@pytest.mark.parametrize("body", ['{"error":{"message":"secret"}}', 'not json', '{}']) +def test_200_error_or_unknown_body_does_not_bind(body): + s = scope() + observed_response(s, body) + assert s.preferred_key(MODEL, ["synthetic-second"]) is None + + +def test_stream_header_and_first_delta_never_bind_and_wire_is_unchanged(): + s = scope() + chunks = [b'data: {"choices":[{"delta":{"content":"text"}}]}\n\n', + b'data: {"choices":[{"finish_reason":"stop"}]}\n\n', b'data: [DONE]\n\n'] + response = observed_response(s, iter(chunks), stream=True) + assert s.preferred_key(MODEL, ["synthetic-second"]) is None + iterator = iter(response.response) + assert next(iterator) == chunks[0] + assert s.preferred_key(MODEL, ["synthetic-second"]) is None + assert list(iterator) == chunks[1:] + assert s.preferred_key(MODEL, ["synthetic-second"]) == "synthetic-second" + + +@pytest.mark.parametrize("event", [b'data: {"error":{"message":"failure"}}\n\n', b'data: {}\n\n']) +def test_error_or_incomplete_stream_evicts(event): + s = scope() + bind(s) + response = observed_response(s, iter([event]), stream=True) + assert list(response.response) == [event] + assert s.preferred_key(MODEL, ["synthetic-second"]) is None + + +def test_abort_before_first_read_does_not_bind(): + s = scope() + response = observed_response(s, iter([b'data: [DONE]\n\n']), stream=True) + response.close() + assert s.preferred_key(MODEL, ["synthetic-second"]) is None + + +def test_cancellation_after_terminal_event_is_not_success(): + from services.request_cancellation import bind_cancellation + s = scope() + with affinity.activate_scope(s), affinity.candidate_scope(MODEL): + response = Response(iter([b'data: [DONE]\n\n']), mimetype="text/event-stream") + context = bind_cancellation(response) + managed_dispatch.capture_managed_attempt(response, MODEL, "synthetic-second") + managed_dispatch.observe_managed_response(response, response, stream=True) + iterator = iter(response.response) + assert next(iterator) == b'data: [DONE]\n\n' + context.cancel() + assert list(iterator) == [] + assert s.preferred_key(MODEL, ["synthetic-second"]) is None + + +def test_disabled_observation_keeps_response_and_iterator_identity(monkeypatch): + monkeypatch.setenv("PROMPT_CACHE_AFFINITY_ENABLED", "false") + source = iter([b'data: [DONE]\n\n']) + response = Response(source, mimetype="text/event-stream") + assert managed_dispatch.capture_managed_attempt(response, MODEL, "synthetic-second") is response + assert managed_dispatch.observe_managed_response(response, response, stream=True) is response + assert response.response is source + assert not hasattr(response, "multillm_affinity_observer") + + +def test_ttl_change_and_warning_once(monkeypatch, caplog): + s = scope() + bind(s) + monkeypatch.setenv("PROMPT_CACHE_AFFINITY_TTL_SECONDS", "10") + assert s.preferred_key(MODEL, ["synthetic-second"]) is None + assert scope().preferred_key(MODEL, ["synthetic-second"]) is None + affinity._warned.clear() + monkeypatch.setenv("PROMPT_CACHE_AFFINITY_ENABLED", "invalid-private-value") + for _ in range(3): + assert affinity.settings()[0] is False + assert len(caplog.records) == 1 + assert "invalid-private-value" not in caplog.text + + +def test_concurrent_outcomes_are_bounded(monkeypatch): + storage = affinity.PromptCacheAffinity(max_entries=8) + monkeypatch.setattr(affinity, "store", storage) + with ThreadPoolExecutor(max_workers=4) as executor: + list(executor.map(lambda i: bind(scope(principal=str(i))), range(100))) + assert len(storage) == 8 + + +def test_authenticated_auto_scope_excludes_explicit_raw_and_unauthenticated(monkeypatch): + from services.auto_route_service import AutoRouteService, AutoRoute + monkeypatch.setattr(AutoRouteService, "get_route", lambda model: AutoRoute("auto:synthetic", (MODEL, OTHER), "revision-1")) + app = Flask(__name__) + with app.test_request_context("/v1/chat/completions", method="POST", headers={"Session-Id": "test"}): + assert affinity.request_scope(PAYLOAD, app.config) is None + g.authenticated_user = {"username": "alice", "role": "coding"} + assert affinity.request_scope(PAYLOAD, app.config) is not None + assert affinity.request_scope({**PAYLOAD, "model": MODEL}, app.config) is None + with app.test_request_context("/proxy/provider", method="POST"): + g.authenticated_user = {"username": "alice"} + assert affinity.request_scope(PAYLOAD, app.config) is None + + +def test_cache_cost_evidence_preserves_unknowns_and_labels_estimates(monkeypatch): + from services.prompt_cache_cost import CacheObservation + monkeypatch.setenv("MODEL_PRICING_USD_PER_MILLION", json.dumps({MODEL: {"input": 2, "cache_read": 0.2, "output": 4}})) + monkeypatch.setenv("PROMPT_CACHE_PRICE_METADATA_JSON", "") + measured = CacheObservation.from_body({"usage": {"prompt_tokens": 100, "completion_tokens": 5, + "prompt_tokens_details": {"cached_tokens": 80}}}) + assert affinity.cache_cost_evidence(MODEL, measured) == (80, 16.0, 160.0) + bind(scope(), usage=measured) + entry = affinity.store.entries()[0][1] + assert entry.cache_read_tokens == 80 + assert entry.estimated_rebuild_cost_microusd == 160.0 + assert affinity.cache_cost_evidence(MODEL, None) == (None, None, None) + monkeypatch.setenv("MODEL_PRICING_USD_PER_MILLION", "{}") + assert affinity.cache_cost_evidence(MODEL, measured) == (80, None, None) + + +def test_model_scoped_cooldown_cannot_be_overridden(monkeypatch): + from services.model_cooldown import model_cooldown, ModelCooldownExhausted + monkeypatch.setenv("MODEL_COOLDOWN_ENABLED", "true") + monkeypatch.setenv("MODEL_COOLDOWN_MAX_SECONDS", "3600") + model_cooldown.reset() + s = scope() + bind(s) + with affinity.activate_scope(s), affinity.candidate_scope(MODEL): + CredentialPool.record("ce-gpt-plus", "synthetic-second", 429, model="synthetic", now=10) + assert CredentialPool.select("ce-gpt-plus", model="synthetic", now=11) == "synthetic-first" + CredentialPool.record("ce-gpt-plus", "synthetic-first", 429, model="synthetic", now=10) + with pytest.raises(ModelCooldownExhausted): + CredentialPool.select("ce-gpt-plus", model="synthetic", now=11) + model_cooldown.reset() + + +def test_stream_exception_evicts_and_propagates(): + s = scope() + bind(s) + def broken(): + yield b'data: {"choices":[{"delta":{"content":"text"}}]}\n\n' + raise OSError("synthetic interrupted socket") + response = observed_response(s, broken(), stream=True) + with pytest.raises(OSError): + list(response.response) + assert s.preferred_key(MODEL, ["synthetic-second"]) is None + response.close() + + +@pytest.mark.parametrize("path,body", [ + ("/v1/chat/completions", {"messages": PAYLOAD["messages"]}), + ("/v1/responses", {"input": "private first turn"}), + ("/v1/messages", {"messages": PAYLOAD["messages"], "max_tokens": 128})]) +@pytest.mark.parametrize("flag", ["false", "true"]) +@pytest.mark.parametrize("fallback", [False, True]) +def test_registered_managed_alias_preserves_bytes_and_prefers_completed_key(monkeypatch, path, body, flag, fallback): + monkeypatch.setenv("PROMPT_CACHE_AFFINITY_ENABLED", flag) + monkeypatch.setenv("ADMIN_API_KEY", "admin-test-key") + monkeypatch.setenv("FLASK_SECRET_KEY", "test-secret") + monkeypatch.setenv("JWT_SECRET", "jwt-test-secret") + with patch("env_loader.load_runtime_env"), patch("config.load_runtime_env"): + app_module = importlib.import_module("app") + with patch.object(app_module, "load_runtime_env"), patch("config.load_runtime_env"): + app = app_module.create_app() + app.config.update(WTF_CSRF_ENABLED=False, IMAGE_RELAY_CATALOG_AUTO_REFRESH=False, + PROMPT_CACHE_ENABLED=False, NANOGPT_SPEED_ROUTING="") + from services.auto_route_service import AutoRouteService, AutoRoute + monkeypatch.setattr(AutoRouteService, "get_route", lambda model: AutoRoute("auto:synthetic", (MODEL, OTHER), "revision-1")) + monkeypatch.setattr(app_module.AuthService, "verify_api_key", lambda *args: {"username": "alice", "is_admin": True}) + tokens = [] + payloads = [] + providers = [] + wire = json.dumps({"id": "synthetic", "object": "chat.completion", "model": "synthetic", + "choices": [{"message": {"role": "assistant", "content": "answer"}, "finish_reason": "stop"}]}).encode() + def send(**kwargs): + tokens.append(kwargs["headers"]["Authorization"]) + payloads.append(kwargs["data"]) + providers.append(kwargs["api_provider"]) + if fallback and kwargs["api_provider"] == "ce-gpt-plus": + return Response(b'{"error":{"message":"rate limited"}}', status=429, mimetype="application/json") + return Response(wire, mimetype="application/json") + monkeypatch.setattr(app_module.ProxyService, "make_request", send) + monkeypatch.setattr(app_module.ProxyService, "probe_nanogpt_key", lambda *args: pytest.fail("unexpected live probe")) + from services.protocol_translation import responses as responses_translation + monkeypatch.setattr(responses_translation, "new_id", lambda prefix: f"{prefix}_synthetic") + CredentialPool.record("ce-gpt-plus", "synthetic-first", 429) + client = app.test_client() + headers = {"Authorization": "Bearer admin-test-key", "Session-Id": "conversation"} + first = client.post(path, json={"model": "auto:synthetic", **body}, headers=headers, buffered=True) + assert first.status_code == 200, first.data + CredentialPool.reset() + second = client.post(path, json={"model": "auto:synthetic", **body}, headers=headers, buffered=True) + assert second.status_code == 200, second.data + assert first.data == second.data + if fallback: + assert providers == ["ce-gpt-plus", "ce-gpt-pro"] * 2 + assert payloads[:2] == payloads[2:] + assert tokens == ["Bearer synthetic-second", "Bearer synthetic-first", "Bearer synthetic-first", "Bearer synthetic-first"] + if flag == "true": + assert affinity.store.entries()[0][1].model == OTHER + else: + assert payloads[0] == payloads[1] + assert tokens == ["Bearer synthetic-second", "Bearer " + ("synthetic-second" if flag == "true" else "synthetic-first")] + assert len(affinity.store) == (1 if flag == "true" else 0) diff --git a/worker/container-env.mjs b/worker/container-env.mjs index 241cf668..c52a445c 100644 --- a/worker/container-env.mjs +++ b/worker/container-env.mjs @@ -63,6 +63,8 @@ const DIRECT_ENV_KEYS = [ "CONTEXT_ANALYSIS_CACHE_TTL_SECONDS", "CONTEXT_ANALYSIS_CACHE_MAX_ENTRIES", "PROMPT_CACHE_ENABLED", + "PROMPT_CACHE_AFFINITY_ENABLED", + "PROMPT_CACHE_AFFINITY_TTL_SECONDS", "PROMPT_CACHE_USAGE_BUCKETS_ENABLED", "PROMPT_CACHE_PRICE_METADATA_JSON", "TOOL_CALL_REPAIR_DEFAULT",