diff --git a/docs/model-cooldown.md b/docs/model-cooldown.md new file mode 100644 index 00000000..b696b549 --- /dev/null +++ b/docs/model-cooldown.md @@ -0,0 +1,127 @@ +# Model-scoped credential cooldown + +`MODEL_COOLDOWN_ENABLED` defaults to `false`. Disabled pools keep their existing +selection, cooldown, validation and response behavior, including the legacy +Codex Everywhere fallback to the first key when every key is resting. + +With the flag enabled, model scoping applies only where the request path supplies +a model or an explicit quota bucket. Paths without either keep the same +provider-wide cooldown bookkeeping, selection and exceptions as the disabled +path, including 429 and 401 handling. + +Enable with: + +```text +MODEL_COOLDOWN_ENABLED=true +MODEL_COOLDOWN_MAX_SECONDS=3600 +``` + +Disable with `MODEL_COOLDOWN_ENABLED=false`, or clear it to an empty value. +An empty maximum uses 3600 seconds. Whitespace is ignored. Malformed settings +log once per setting name, without the supplied value, and disable this feature. +The maximum must be an integer from 1 through 2147483647. + +## Evidence and scope + +The pool methods accept keyword-only `model`, `quota_bucket`, `outcome`, +`credential_wide_auth` and `retry_after_seconds` inputs where applicable. +Selection and observation must receive the same provider, model and quota bucket. +Use the actual upstream model, without changing an explicit model selection. +A quota bucket is an explicit provider configuration shared by its member models; +never infer it from a response body, an arbitrary caller field or a model prefix. +Model names and quota bucket names occupy distinct namespaces. + +Use `UpstreamOutcome` evidence. A `throttled` credential outcome cools only +that credential and model, or its explicitly configured quota bucket. +A classified quota-exhaustion observation can also use `throttled`; a bare 402 +is a classified caller error and cannot establish exhaustion. Caller errors, transport +failures, cancellation and provider failures create no cooldown. + +A classified credential rejection is global only with genuine auth evidence. +Pool adapters treat a classified 401 as credential-wide unless +`credential_wide_auth=False` is supplied. A 403 needs an explicit +`credential_wide_auth=True`; endpoint/model permission denial is not enough. +A supplied classification overrides the status classification, including for +cancellation. Neither health evidence nor cooldown grants replay permission. + +Trusted normalized reset advice is passed as finite nonnegative delay seconds, +relative to the observation's monotonic time. The store does not parse raw +headers or provider bodies. Invalid advice uses the existing pool's fallback: +60 seconds for throttling, 300 seconds for Codex Everywhere auth, and the +configured NanoGPT validation TTL for auth. The configured maximum caps each +observation; positive fractional delays round up only in `Retry-After`. +At least one second is retained for zero delay to prevent immediate redispatch. +Success clears only its model/bucket observation. It does not clear another +model, an auth rejection, a legacy credential-wide rest or a capacity guard. +Unscoped recording uses the legacy provider-wide rules and leaves model state +unchanged. Genuine authentication rejection on a scoped path also rests the +credential in the legacy map, using the same capped deadline, so unscoped +selection observes it. + +## Selection and errors + +Scoped selection excludes keys resting in either the model store or the legacy +provider-wide map. Available keys retain configured order. NanoGPT retains its existing active-key +preference, single-configured-key probe bypass, validation TTL and request-count +rules. Filtering several configured keys down to one does not skip validation. +During scoped selection, catalog probes carry no model quota evidence: only +genuine auth rejection can cool every model from that probe. Unscoped validation +keeps its legacy rejection rules. Raw and unified NanoGPT pools remain isolated. + +`CredentialPool.available` returns the eligible list. `CredentialPool.select`, +`NanoGPTKeyPool.select_key` and `select_available_key` with model/bucket context raise +`ModelCooldownExhausted` when all configured eligible keys are cooling. There is +no secret dispatch, additional wait, retry or synthetic completion. An empty +configuration retains the existing no-key result. The error is a real 429 with +`error: model_cooldown`; the named Flask adapter emits bounded integer +`Retry-After` advice to the earliest eligible key's release. A credential's +release waits for both its auth and model/bucket observations to expire. + +Register `cooldown_error_response` for `ModelCooldownExhausted` and +`ModelCooldownCapacity` in the Flask application. Preserve these +errors through route wrappers instead of translating them to 500/503 or a +candidate fallback. Without the adapter, the existing APIError handler does not +include `Retry-After`. + +## Storage and operational limits + +Model-scoped state is process-local, protected by a lock and limited to 10000 live +observations across both pool implementations. Credential IDs are SHA-256 HMACs +with a random process-local salt and provider/pool namespace. Scope labels are +also HMACs. New cooldown state stores only digests and monotonic expiry times; +it never serializes or logs keys, prompts, completions or model names. +The existing pools still retain configured/active keys for dispatch. + +Expired observations are pruned on reads and writes. A process restart discards +observations and changes HMAC identities. This feature does not provide +multi-process, Container or distributed consistency. Configured credentials and +existing raw/unified pool precedence remain authoritative. + +At capacity, live observations are never evicted. A scalar overflow deadline +covers the omitted observation's capped TTL. During that interval new admission +fails closed with 503 and bounded `Retry-After`; known all-cooling pools still +produce 429. This capacity failure is not a credential-wide cooldown. It may +briefly block unrelated traffic; that is the cost of preserving live cooldowns +within a hard memory bound. Observation itself cannot interrupt an already +produced response. There are no extra provider calls or charges from cooldown +tracking. Existing permitted validation probes retain their existing cost. + +## Request integration + +Pass the resolved upstream model and configured quota bucket to both selection +and recording. Forward classified outcomes, genuine credential-wide auth +evidence and normalized trusted reset advice to recording. For `available` +callers, an empty eligible result needs the same 429 admission decision using +`model_cooldown.select` with the original keys and legacy rest deadlines. An +empty configuration remains distinct. Preserve cooldown exceptions through +route wrappers rather than treating them as generic candidate failures. + +Native Worker roleplay maintains independent health state. Worker-forwarded +Flask traffic uses these pools. Forward `MODEL_COOLDOWN_ENABLED` and +`MODEL_COOLDOWN_MAX_SECONDS` through the Container environment allowlist. +Expose `Retry-After` through Flask and Worker CORS for browser clients. +No migration, table, new production route or Worker route/module is required. + +Tests cover pool selection and recording plus a registered Flask error adapter +with fake dispatch. Live provider quota semantics, deployment, billing and +distributed consistency require separate operational verification. diff --git a/services/credential_pool.py b/services/credential_pool.py index ff59bfad..398e8773 100644 --- a/services/credential_pool.py +++ b/services/credential_pool.py @@ -18,6 +18,8 @@ from providers.codex_everywhere import CODEX_EVERYWHERE_POOLS from services.auth_primitives import usable_credential +from services.model_cooldown import model_cooldown, settings +from services.upstream_outcome import UpstreamOutcome, classify_upstream_outcome REFUSED_REST_SECONDS = 300 RATE_LIMITED_REST_SECONDS = 60 @@ -64,26 +66,109 @@ def keys(provider: str, environ: Mapping[str, str] | None = None) -> list[str]: @classmethod def available( - cls, provider: str, keys: list[str] | None = None, *, now: float | None = None + cls, + provider: str, + keys: list[str] | None = None, + *, + now: float | None = None, + model: str | None = None, + quota_bucket: str | None = None, ) -> list[str]: """Configured keys that are not resting, in preference order.""" keys = cls.keys(provider) if keys is None else keys + config = settings() + if config.enabled and (model or quota_bucket): + with cls._lock: + legacy = {key: cls._resting.get((provider, key), 0) for key in keys} + return model_cooldown.available( + provider, + keys, + model=model, + quota_bucket=quota_bucket, + max_seconds=config.max_seconds, + now=now, + legacy_rest_until=legacy, + ) current = time.monotonic() if now is None else now with cls._lock: - return [key for key in keys if cls._resting.get((provider, key), 0) <= current] + return [ + key for key in keys if cls._resting.get((provider, key), 0) <= current + ] @classmethod - def select(cls, provider: str, *, now: float | None = None) -> str | None: + def select( + cls, + provider: str, + *, + now: float | None = None, + model: str | None = None, + quota_bucket: str | None = None, + ) -> str | None: """The key to use now; the first configured one when every key is resting.""" keys = cls.keys(provider) + config = settings() + if config.enabled and (model or quota_bucket): + with cls._lock: + legacy = {key: cls._resting.get((provider, key), 0) for key in keys} + return model_cooldown.select( + provider, + keys, + model=model, + quota_bucket=quota_bucket, + max_seconds=config.max_seconds, + now=now, + legacy_rest_until=legacy, + ) return next(iter(cls.available(provider, keys, now=now) or keys), None) @classmethod def record( - cls, provider: str, key: str | None, status: int, *, now: float | None = None + cls, + provider: str, + key: str | None, + status: int, + *, + now: float | None = None, + model: str | None = None, + quota_bucket: str | None = None, + outcome: UpstreamOutcome | None = None, + credential_wide_auth: bool | None = None, + retry_after_seconds: float | None = None, ) -> None: if not key or provider not in POOLED_KEY_ENVS: return + config = settings() + if config.enabled and (model or quota_bucket): + classified = outcome or classify_upstream_outcome(status) + wide_auth = ( + status == 401 if credential_wide_auth is None else credential_wide_auth + ) + until = model_cooldown.record( + provider, + key, + classified, + model=model, + quota_bucket=quota_bucket, + now=now, + credential_wide_auth=wide_auth, + retry_after_seconds=retry_after_seconds, + max_seconds=config.max_seconds, + fallback_seconds=( + REFUSED_REST_SECONDS + if classified.credential_health == "rejected" + else RATE_LIMITED_REST_SECONDS + ), + ) + if ( + classified.credential_health == "rejected" + and wide_auth + and until is not None + ): + with cls._lock: + cls._resting[(provider, key)] = max( + cls._resting.get((provider, key), 0), until + ) + return current = time.monotonic() if now is None else now with cls._lock: if status in REFUSED_STATUSES: @@ -94,15 +179,42 @@ def record( cls._resting.pop((provider, key), None) @classmethod - def record_headers(cls, provider: str, headers: Mapping[str, str], status: int) -> None: + def record_headers( + cls, + provider: str, + headers: Mapping[str, str], + status: int, + *, + model: str | None = None, + quota_bucket: str | None = None, + outcome: UpstreamOutcome | None = None, + credential_wide_auth: bool | None = None, + retry_after_seconds: float | None = None, + now: float | None = None, + ) -> None: """Record the result for the bearer key a request carried.""" if provider not in POOLED_KEY_ENVS: return authorization = next( - (value for name, value in headers.items() if name.lower() == "authorization"), "" + ( + value + for name, value in headers.items() + if name.lower() == "authorization" + ), + "", ) if isinstance(authorization, str) and authorization.startswith("Bearer "): - cls.record(provider, authorization[len("Bearer "):].strip(), status) + cls.record( + provider, + authorization[len("Bearer ") :].strip(), + status, + model=model, + quota_bucket=quota_bucket, + outcome=outcome, + credential_wide_auth=credential_wide_auth, + retry_after_seconds=retry_after_seconds, + now=now, + ) @classmethod def reset(cls) -> None: diff --git a/services/model_cooldown.py b/services/model_cooldown.py new file mode 100644 index 00000000..462cd7df --- /dev/null +++ b/services/model_cooldown.py @@ -0,0 +1,255 @@ +"""Bounded, process-local credential/model admission from classified evidence.""" + +from __future__ import annotations + +import hashlib +import hmac +import logging +import math +import os +import secrets +import threading +import time +from collections.abc import Mapping, Sequence +from dataclasses import dataclass + +from flask import jsonify + +from error_handlers import APIError +from services.upstream_outcome import UpstreamOutcome + +logger = logging.getLogger(__name__) +MAX_ENTRIES = 10_000 +_warning_lock = threading.Lock() +_warned_settings: set[str] = set() + + +@dataclass(frozen=True) +class CooldownSettings: + enabled: bool = False + max_seconds: int = 3600 + + +def _invalid_setting(name: str) -> None: + with _warning_lock: + if name not in _warned_settings: + _warned_settings.add(name) + logger.warning("Invalid %s; model cooldown disabled", name) + + +def settings(environ: Mapping[str, str] | None = None) -> CooldownSettings: + source = os.environ if environ is None else environ + enabled = (source.get("MODEL_COOLDOWN_ENABLED") or "false").strip().lower() + if not enabled: + enabled = "false" + if enabled not in {"true", "false", "1", "0", "yes", "no", "on", "off"}: + _invalid_setting("MODEL_COOLDOWN_ENABLED") + return CooldownSettings() + cap = (source.get("MODEL_COOLDOWN_MAX_SECONDS") or "3600").strip() or "3600" + try: + seconds = int(cap) + if not 1 <= seconds <= 2_147_483_647: + raise ValueError + except (TypeError, ValueError): + _invalid_setting("MODEL_COOLDOWN_MAX_SECONDS") + return CooldownSettings() + return CooldownSettings(enabled in {"true", "1", "yes", "on"}, seconds) + + +class ModelCooldownExhausted(APIError): + """No eligible credential can be dispatched before its cooldown expires.""" + + def __init__(self, retry_after: int): + super().__init__( + "All eligible credentials are cooling down.", + 429, + {"error": "model_cooldown"}, + ) + self.retry_after = retry_after + + +class ModelCooldownCapacity(APIError): + """Admission fails closed rather than discarding a live cooldown.""" + + def __init__(self, retry_after: int): + super().__init__("Model cooldown capacity is temporarily unavailable.", 503) + self.retry_after = retry_after + + +def cooldown_error_response(error: ModelCooldownExhausted | ModelCooldownCapacity): + """Emit the cooldown status and bounded Retry-After advice.""" + response = jsonify(error.to_dict()) + response.status_code = error.status_code + response.headers["Retry-After"] = str(error.retry_after) + return response + + +def _delay(value: float | None, fallback: int, cap: int) -> float: + if isinstance(value, (int, float)) and not isinstance(value, bool): + try: + if math.isfinite(value) and value >= 0: + return min(cap, max(1, value)) + except OverflowError: + pass + return min(cap, max(1, fallback)) + + +class ModelCooldown: + """Keep HMAC identities and monotonic deadlines; never retain credentials.""" + + def __init__(self, *, max_entries: int = MAX_ENTRIES): + self._max_entries = min(MAX_ENTRIES, max(1, max_entries)) + self._salt = secrets.token_bytes(32) + self._lock = threading.RLock() + self._entries: dict[tuple[str, str | None], float] = {} + self._overflow_until = 0.0 + + def _digest(self, *parts: str) -> str: + digest = hmac.new(self._salt, digestmod=hashlib.sha256) + for part in parts: + encoded = part.encode("utf-8") + digest.update(len(encoded).to_bytes(8, "big")) + digest.update(encoded) + return digest.hexdigest() + + def credential_id(self, provider: str, credential: str) -> str: + return self._digest("credential", provider, credential) + + def _scope(self, model: str | None, quota_bucket: str | None) -> str | None: + if quota_bucket: + return self._digest("quota_bucket", quota_bucket) + return self._digest("model", model) if model else None + + def _prune(self, now: float) -> None: + self._entries = { + key: until for key, until in self._entries.items() if until > now + } + if self._overflow_until <= now: + self._overflow_until = 0.0 + + @property + def entry_count(self) -> int: + with self._lock: + return len(self._entries) + + def reset(self) -> None: + with self._lock: + self._entries.clear() + self._overflow_until = 0.0 + + def record( + self, + provider: str, + credential: str, + outcome: UpstreamOutcome, + *, + model: str | None = None, + quota_bucket: str | None = None, + credential_wide_auth: bool = False, + retry_after_seconds: float | None = None, + fallback_seconds: int = 60, + max_seconds: int = 3600, + now: float | None = None, + ) -> float | None: + """Return the recorded deadline so adapters can mirror global auth rests.""" + current = time.monotonic() if now is None else now + scope = self._scope(model, quota_bucket) + identity = self.credential_id(provider, credential) + wide_auth = outcome.credential_health == "rejected" and credential_wide_auth + with self._lock: + self._prune(current) + if outcome.credential_health == "accepted": + if scope is not None: + self._entries.pop((identity, scope), None) + return None + if not wide_auth and ( + outcome.credential_health != "throttled" or scope is None + ): + return None + entry = (identity, None if wide_auth else scope) + until = current + _delay(retry_after_seconds, fallback_seconds, max_seconds) + if entry not in self._entries and len(self._entries) >= self._max_entries: + # Do not evict live state and silently dispatch a cooling key. + self._overflow_until = max(self._overflow_until, until) + return until + self._entries[entry] = max(self._entries.get(entry, 0), until) + return self._entries[entry] + + def _remaining( + self, + provider: str, + key: str, + scope: str | None, + now: float, + legacy_rest_until: Mapping[str, float] | None = None, + ) -> float: + identity = self.credential_id(provider, key) + until = max( + self._entries.get((identity, None), 0.0), + (legacy_rest_until or {}).get(key, 0.0), + ) + if scope is not None: + until = max(until, self._entries.get((identity, scope), 0.0)) + return max(0.0, until - now) + + def available( + self, + provider: str, + keys: Sequence[str], + *, + model: str | None = None, + quota_bucket: str | None = None, + max_seconds: int = 3600, + now: float | None = None, + legacy_rest_until: Mapping[str, float] | None = None, + ) -> list[str]: + current = time.monotonic() if now is None else now + scope = self._scope(model, quota_bucket) + with self._lock: + self._prune(current) + available = [ + key + for key in keys + if not self._remaining(provider, key, scope, current, legacy_rest_until) + ] + if available and self._overflow_until > current: + raise ModelCooldownCapacity( + min(max_seconds, max(1, math.ceil(self._overflow_until - current))) + ) + return available + + def select( + self, + provider: str, + keys: Sequence[str], + *, + model: str | None = None, + quota_bucket: str | None = None, + max_seconds: int = 3600, + now: float | None = None, + legacy_rest_until: Mapping[str, float] | None = None, + ) -> str | None: + current = time.monotonic() if now is None else now + with self._lock: + available = self.available( + provider, + keys, + model=model, + quota_bucket=quota_bucket, + max_seconds=max_seconds, + now=current, + legacy_rest_until=legacy_rest_until, + ) + if available: + return available[0] + if not keys: + return None + scope = self._scope(model, quota_bucket) + remaining = min( + self._remaining(provider, key, scope, current, legacy_rest_until) + for key in keys + ) + raise ModelCooldownExhausted(min(max_seconds, max(1, math.ceil(remaining)))) + + +model_cooldown = ModelCooldown() diff --git a/services/nanogpt_key_pool.py b/services/nanogpt_key_pool.py index 3566062d..44ad3229 100644 --- a/services/nanogpt_key_pool.py +++ b/services/nanogpt_key_pool.py @@ -10,6 +10,8 @@ from typing import ClassVar from services.auth_primitives import is_placeholder_credential +from services.model_cooldown import model_cooldown, settings +from services.upstream_outcome import UpstreamOutcome, classify_upstream_outcome _DIRECT_KEY_NAMES = ("NANOGPT_API_KEY", "NANO_GPT_KEY") _LIST_KEY_NAMES = ("NANOGPT_API_KEYS", "NANO_GPT_KEYS") @@ -97,6 +99,7 @@ def configured_nanogpt_keys( class NanoGPTKeyPool: """Select and retain one validated configured key per application process.""" + _model_cooldown_provider: ClassVar[str] = "nanogpt" _lock: ClassVar[threading.RLock] = threading.RLock() _active_key: ClassVar[str | None] = None _active_until: ClassVar[float] = 0.0 @@ -104,15 +107,36 @@ class NanoGPTKeyPool: _rejected_until: ClassVar[dict[str, float]] = {} @classmethod - def select_available_key(cls, keys: Sequence[str]) -> str | None: + def select_available_key( + cls, + keys: Sequence[str], + *, + model: str | None = None, + quota_bucket: str | None = None, + now: float | None = None, + ) -> str | None: """Select an uncooled credential without claiming a successful probe.""" configured = list(dict.fromkeys(key.strip() for key in keys if key.strip())) - now = time.monotonic() + now = time.monotonic() if now is None else now + config = settings() + scoped = config.enabled and bool(model or quota_bucket) with cls._lock: cls._prune(configured, now) - ordered = ([cls._active_key] if cls._active_key in configured else []) + ordered = [cls._active_key] if cls._active_key in configured else [] ordered.extend(key for key in configured if key not in ordered) - return next((key for key in ordered if cls._rejected_until.get(key, 0) <= now), None) + if scoped: + return model_cooldown.select( + cls._model_cooldown_provider, + ordered, + model=model, + quota_bucket=quota_bucket, + max_seconds=config.max_seconds, + now=now, + legacy_rest_until=cls._rejected_until, + ) + return next( + (key for key in ordered if cls._rejected_until.get(key, 0) <= now), None + ) @classmethod def select_key( @@ -124,18 +148,44 @@ def select_key( check_every_requests: int = 50, rejected_cooldown_seconds: int = 60, now: float | None = None, + model: str | None = None, + quota_bucket: str | None = None, ) -> str | None: configured = list(dict.fromkeys(key.strip() for key in keys if key.strip())) if not configured: return None checked_at = time.monotonic() if now is None else now + config = settings() + scoped = config.enabled and bool(model or quota_bucket) + single_configured_key = len(configured) == 1 with cls._lock: cls._prune(configured, checked_at) + if scoped: + eligible = model_cooldown.available( + cls._model_cooldown_provider, + configured, + model=model, + quota_bucket=quota_bucket, + max_seconds=config.max_seconds, + now=checked_at, + legacy_rest_until=cls._rejected_until, + ) + if not eligible: + return model_cooldown.select( + cls._model_cooldown_provider, + configured, + model=model, + quota_bucket=quota_bucket, + max_seconds=config.max_seconds, + now=checked_at, + legacy_rest_until=cls._rejected_until, + ) + configured = eligible if ( - len(configured) == 1 + single_configured_key and cls._active_key is None - and cls._rejected_until.get(configured[0], 0) <= checked_at + and (scoped or cls._rejected_until.get(configured[0], 0) <= checked_at) ): cls._active_key = configured[0] cls._active_until = checked_at + max(1, check_ttl_seconds) @@ -145,7 +195,9 @@ def select_key( cls._active_key in configured and cls._active_until > checked_at and cls._active_requests < max(1, check_every_requests) - and cls._rejected_until.get(cls._active_key, 0) <= checked_at + and ( + scoped or cls._rejected_until.get(cls._active_key, 0) <= checked_at + ) ): return cls._active_key @@ -175,23 +227,42 @@ def select_key( cls._active_key = key cls._active_until = checked_at + max(1, check_ttl_seconds) cls._active_requests = 0 - cls._rejected_until.pop(key, None) + if not scoped: + cls._rejected_until.pop(key, None) return key - if not is_nanogpt_credential_rejection(status): - ambiguous_active = ambiguous_active or key == previous_active - cls._mark_rejected( - key, - status, - checked_at, - check_ttl_seconds, - rejected_cooldown_seconds, + probe_rejected = ( + classify_upstream_outcome(status).credential_health == "rejected" + and status == 401 + if scoped + else is_nanogpt_credential_rejection(status) ) + if not probe_rejected: + ambiguous_active = ambiguous_active or key == previous_active + if scoped: + # Catalog validation has no model-specific quota evidence. + cls._record_model_result( + key, + status, + now=checked_at, + check_ttl_seconds=check_ttl_seconds, + rejected_cooldown_seconds=rejected_cooldown_seconds, + ) + else: + cls._mark_rejected( + key, + status, + checked_at, + check_ttl_seconds, + rejected_cooldown_seconds, + ) if ( ambiguous_active and previous_active in configured - and cls._rejected_until.get(previous_active, 0) <= checked_at + and ( + scoped or cls._rejected_until.get(previous_active, 0) <= checked_at + ) ): cls._active_key = previous_active cls._active_until = checked_at + min( @@ -204,6 +275,17 @@ def select_key( cls._active_key = None cls._active_until = 0.0 cls._active_requests = 0 + if scoped: + # Genuine probe auth failures can make the entire pool cooling. + model_cooldown.select( + cls._model_cooldown_provider, + configured, + model=model, + quota_bucket=quota_bucket, + max_seconds=config.max_seconds, + now=checked_at, + legacy_rest_until=cls._rejected_until, + ) if attempted == 0: raise NanoGPTKeyPoolExhausted( "All configured NanoGPT keys are cooling down", @@ -221,10 +303,32 @@ def record_result( check_ttl_seconds: int = 300, rejected_cooldown_seconds: int = 60, now: float | None = None, + model: str | None = None, + quota_bucket: str | None = None, + outcome: UpstreamOutcome | None = None, + credential_wide_auth: bool | None = None, + retry_after_seconds: float | None = None, ) -> None: """Record one upstream request and invalidate definite key failures.""" if not key: return + if settings().enabled and (model or quota_bucket): + cls._record_model_result( + key, + status, + model=model, + quota_bucket=quota_bucket, + outcome=outcome, + credential_wide_auth=credential_wide_auth, + retry_after_seconds=retry_after_seconds, + now=now, + check_ttl_seconds=check_ttl_seconds, + rejected_cooldown_seconds=rejected_cooldown_seconds, + ) + with cls._lock: + if cls._active_key == key: + cls._active_requests += 1 + return if is_nanogpt_credential_rejection(status): cls.invalidate( key, @@ -248,9 +352,28 @@ def invalidate( check_ttl_seconds: int = 300, rejected_cooldown_seconds: int = 60, now: float | None = None, + model: str | None = None, + quota_bucket: str | None = None, + outcome: UpstreamOutcome | None = None, + credential_wide_auth: bool | None = None, + retry_after_seconds: float | None = None, ) -> None: if not key: return + if settings().enabled and (model or quota_bucket): + cls._record_model_result( + key, + status, + model=model, + quota_bucket=quota_bucket, + outcome=outcome, + credential_wide_auth=credential_wide_auth, + retry_after_seconds=retry_after_seconds, + now=now, + check_ttl_seconds=check_ttl_seconds, + rejected_cooldown_seconds=rejected_cooldown_seconds, + ) + return rejected_at = time.monotonic() if now is None else now with cls._lock: if cls._active_key == key: @@ -265,6 +388,54 @@ def invalidate( rejected_cooldown_seconds, ) + @classmethod + def _record_model_result( + cls, + key: str, + status: int, + *, + model: str | None = None, + quota_bucket: str | None = None, + outcome: UpstreamOutcome | None = None, + credential_wide_auth: bool | None = None, + retry_after_seconds: float | None = None, + now: float | None = None, + check_ttl_seconds: int = 300, + rejected_cooldown_seconds: int = 60, + ) -> None: + classified = outcome or classify_upstream_outcome(status) + wide_auth = ( + status == 401 if credential_wide_auth is None else credential_wide_auth + ) + until = model_cooldown.record( + cls._model_cooldown_provider, + key, + classified, + model=model, + quota_bucket=quota_bucket, + credential_wide_auth=wide_auth, + retry_after_seconds=retry_after_seconds, + now=now, + max_seconds=settings().max_seconds, + fallback_seconds=( + check_ttl_seconds + if classified.credential_health == "rejected" + else rejected_cooldown_seconds + ), + ) + + if ( + classified.credential_health == "rejected" + and wide_auth + and until is not None + ): + with cls._lock: + cls._rejected_until[key] = max(cls._rejected_until.get(key, 0), until) + if cls._active_key == key: + cls._active_key = None + cls._active_until = 0.0 + cls._active_requests = 0 + @classmethod def reset(cls) -> None: """Clear process-local selection state for tests and app reinitialization.""" @@ -308,6 +479,8 @@ def _mark_rejected( class NanoGPTUnifiedKeyPool(NanoGPTKeyPool): """Keep unified/subscription credential health separate from raw traffic.""" + _model_cooldown_provider: ClassVar[str] = "nanogpt-unified" + _lock: ClassVar[threading.RLock] = threading.RLock() _active_key: ClassVar[str | None] = None _active_until: ClassVar[float] = 0.0 diff --git a/tests/test_model_cooldown.py b/tests/test_model_cooldown.py new file mode 100644 index 00000000..2f5798df --- /dev/null +++ b/tests/test_model_cooldown.py @@ -0,0 +1,454 @@ +"""Model cooldown evidence, selection, and compatibility contracts.""" + +import math +from concurrent.futures import ThreadPoolExecutor +from unittest.mock import Mock, patch + +import pytest +from flask import Flask + +from error_handlers import init_error_handlers +from services.credential_pool import CredentialPool +from services.nanogpt_key_pool import ( + NanoGPTKeyPool, + NanoGPTUnifiedKeyPool, + NanoGPTKeyPoolExhausted, +) +from services.model_cooldown import ( + ModelCooldown, + ModelCooldownExhausted, + ModelCooldownCapacity, + cooldown_error_response, + model_cooldown, + settings, +) +from services.upstream_outcome import UpstreamOutcome, classify_upstream_outcome + +PROVIDER = "ce-gpt-pro" +KEYS = ["synthetic-first", "synthetic-second"] + + +@pytest.fixture(autouse=True) +def isolated(monkeypatch): + monkeypatch.setenv("MODEL_COOLDOWN_ENABLED", "true") + monkeypatch.delenv("MODEL_COOLDOWN_MAX_SECONDS", raising=False) + CredentialPool.reset() + NanoGPTKeyPool.reset() + NanoGPTUnifiedKeyPool.reset() + model_cooldown.reset() + yield + CredentialPool.reset() + NanoGPTKeyPool.reset() + NanoGPTUnifiedKeyPool.reset() + model_cooldown.reset() + + +def test_throttle_leaves_other_model_and_provider_available(): + CredentialPool.record(PROVIDER, KEYS[0], 429, model="A", now=100) + assert CredentialPool.available(PROVIDER, KEYS, model="A", now=101) == KEYS[1:] + assert CredentialPool.available(PROVIDER, KEYS, model="B", now=101) == KEYS + assert CredentialPool.available("ce-gpt-plus", KEYS, model="A", now=101) == KEYS + assert CredentialPool._resting == {} + + +def test_explicit_shared_bucket_and_success_clear_only_that_bucket(): + CredentialPool.record( + PROVIDER, KEYS[0], 429, model="A", quota_bucket="shared", now=100 + ) + CredentialPool.record(PROVIDER, KEYS[0], 429, model="C", now=100) + assert ( + CredentialPool.available( + PROVIDER, KEYS, model="B", quota_bucket="shared", now=101 + ) + == KEYS[1:] + ) + assert CredentialPool.available(PROVIDER, KEYS, model="B", now=101) == KEYS + CredentialPool.record( + PROVIDER, KEYS[0], 200, model="B", quota_bucket="shared", now=102 + ) + assert ( + CredentialPool.available( + PROVIDER, KEYS, model="A", quota_bucket="shared", now=103 + ) + == KEYS + ) + assert CredentialPool.available(PROVIDER, KEYS, model="C", now=103) == KEYS[1:] + + +def test_auth_is_wide_and_success_cannot_clear_it(): + CredentialPool.record(PROVIDER, KEYS[0], 401, model="A", now=100) + CredentialPool.record(PROVIDER, KEYS[0], 200, model="B", now=101) + assert CredentialPool.available(PROVIDER, KEYS, model="B", now=102) == KEYS[1:] + assert CredentialPool.available(PROVIDER, KEYS, model="B", now=400) == KEYS + + +@pytest.mark.parametrize("status", [400, 402, 403, 404, 422, 500]) +def test_status_alone_does_not_invent_quota_or_wide_auth(status): + CredentialPool.record(PROVIDER, KEYS[0], status, model="A", now=100) + assert CredentialPool.available(PROVIDER, KEYS, model="A", now=101) == KEYS + + +def test_classified_cancelled_and_scoped_auth_evidence(): + CredentialPool.record( + PROVIDER, + KEYS[0], + 429, + model="A", + now=100, + outcome=classify_upstream_outcome(429, cancelled=True), + ) + CredentialPool.record( + PROVIDER, KEYS[0], 401, model="A", now=100, credential_wide_auth=False + ) + assert CredentialPool.available(PROVIDER, KEYS, model="A", now=101) == KEYS + CredentialPool.record( + PROVIDER, KEYS[0], 403, model="A", now=100, credential_wide_auth=True + ) + assert CredentialPool.available(PROVIDER, KEYS, model="B", now=101) == KEYS[1:] + + +def test_exhaustion_requires_classified_evidence(): + exhausted = UpstreamOutcome(False, "neutral", "throttled", "quota_exhausted") + CredentialPool.record(PROVIDER, KEYS[0], 402, model="A", outcome=exhausted, now=100) + assert CredentialPool.available(PROVIDER, KEYS, model="A", now=101) == KEYS[1:] + assert CredentialPool.available(PROVIDER, KEYS, model="B", now=101) == KEYS + assert exhausted.replay_permission is False + + +def test_all_cooling_select_raises_bounded_real_429_without_dispatch(monkeypatch): + monkeypatch.setenv("MODEL_COOLDOWN_MAX_SECONDS", "10") + for key in KEYS: + CredentialPool.record( + PROVIDER, key, 429, model="A", now=100, retry_after_seconds=999 + ) + with patch.object(CredentialPool, "keys", return_value=KEYS): + with pytest.raises(ModelCooldownExhausted) as caught: + CredentialPool.select(PROVIDER, model="A", now=101) + assert CredentialPool.select(PROVIDER, model="B", now=101) == KEYS[0] + assert caught.value.status_code == 429 + assert caught.value.retry_after == 9 + assert all(key not in str(caught.value) for key in KEYS) + + +def test_registered_error_adapter_preserves_json_and_retry_after(): + for key in KEYS: + CredentialPool.record(PROVIDER, key, 429, model="A", now=100) + app = Flask(__name__) + init_error_handlers(app) + app.register_error_handler(ModelCooldownExhausted, cooldown_error_response) + send = Mock() + + @app.post("/v1/test-cooldown") + def dispatch(): + CredentialPool.select(PROVIDER, model="A", now=101) + send() + return {"ok": True} + + with patch.object(CredentialPool, "keys", return_value=KEYS): + response = app.test_client().post("/v1/test-cooldown", json={}) + assert response.status_code == 429 + assert response.json["error"] == "model_cooldown" + assert response.headers["Retry-After"] == "59" + send.assert_not_called() + + +@pytest.mark.parametrize("enabled", [None, "", "false", "0", "garbage"]) +def test_disabled_preserves_legacy_pool_behaviour(monkeypatch, enabled): + if enabled is None: + monkeypatch.delenv("MODEL_COOLDOWN_ENABLED", raising=False) + else: + monkeypatch.setenv("MODEL_COOLDOWN_ENABLED", enabled) + for key in KEYS: + CredentialPool.record(PROVIDER, key, 402, model="A", now=100) + with patch.object(CredentialPool, "keys", return_value=KEYS): + assert CredentialPool.select(PROVIDER, model="A", now=101) == KEYS[0] + assert CredentialPool.available(PROVIDER, KEYS, model="B", now=101) == [] + assert model_cooldown.entry_count == 0 + NanoGPTKeyPool.record_result(KEYS[0], 402, model="A", now=100) + assert KEYS[0] in NanoGPTKeyPool._rejected_until + + +@pytest.mark.parametrize("value", ["bad", "0", "-1", "NaN", "1.5"]) +def test_bad_config_fails_off_and_logs_once_without_value(monkeypatch, caplog, value): + monkeypatch.setenv("MODEL_COOLDOWN_MAX_SECONDS", value) + assert not settings().enabled + assert not settings().enabled + messages = [ + r.message for r in caplog.records if "MODEL_COOLDOWN_MAX_SECONDS" in r.message + ] + assert len(messages) <= 1 + assert all(value not in message for message in messages) + + +def test_empty_cap_means_default(monkeypatch): + monkeypatch.setenv("MODEL_COOLDOWN_MAX_SECONDS", "") + assert settings().enabled + assert settings().max_seconds == 3600 + + +@pytest.mark.parametrize("delay", [math.nan, math.inf, -1, "untrusted", True]) +def test_malformed_reset_advice_uses_default(delay): + CredentialPool.record( + PROVIDER, KEYS[0], 429, model="A", now=100, retry_after_seconds=delay + ) + assert CredentialPool.available(PROVIDER, KEYS[:1], model="A", now=159) == [] + assert CredentialPool.available(PROVIDER, KEYS[:1], model="A", now=160) == KEYS[:1] + + +@pytest.mark.parametrize("status,duration", [(429, 60), (401, 300)]) +def test_unwired_credential_calls_match_flag_off(monkeypatch, status, duration): + snapshots = [] + for enabled in ("false", "true"): + monkeypatch.setenv("MODEL_COOLDOWN_ENABLED", enabled) + CredentialPool.reset() + model_cooldown.reset() + with patch.object(CredentialPool, "keys", return_value=KEYS): + CredentialPool.record_headers( + PROVIDER, {"Authorization": "Bearer " + KEYS[0]}, status, now=100 + ) + assert CredentialPool.select(PROVIDER, now=101) == KEYS[1] + assert CredentialPool.available(PROVIDER, KEYS, now=101) == KEYS[1:] + assert CredentialPool.available(PROVIDER, KEYS, now=100 + duration) == KEYS + CredentialPool.record(PROVIDER, KEYS[1], status, now=100) + assert CredentialPool.select(PROVIDER, now=101) == KEYS[0] + assert CredentialPool.available(PROVIDER, KEYS, now=101) == [] + snapshots.append(dict(CredentialPool._resting)) + CredentialPool.record_headers( + PROVIDER, {"Authorization": "Bearer " + KEYS[0]}, 200, now=102 + ) + assert CredentialPool.available(PROVIDER, KEYS, now=103) == KEYS[:1] + assert model_cooldown.entry_count == 0 + assert snapshots[0] == snapshots[1] + + +@pytest.mark.parametrize("pool", [NanoGPTKeyPool, NanoGPTUnifiedKeyPool]) +@pytest.mark.parametrize("status,duration", [(429, 17), (401, 91)]) +def test_unwired_nanogpt_calls_match_flag_off(monkeypatch, pool, status, duration): + snapshots = [] + for enabled in ("false", "true"): + monkeypatch.setenv("MODEL_COOLDOWN_ENABLED", enabled) + pool.reset() + model_cooldown.reset() + probe = Mock(return_value=200) + assert pool.select_key(KEYS, probe, now=99) == KEYS[0] + pool.record_result( + KEYS[0], + status, + now=100, + check_ttl_seconds=91, + rejected_cooldown_seconds=17, + ) + assert pool._active_key is None + assert pool.select_available_key(KEYS, now=101) == KEYS[1] + assert pool.select_key(KEYS, probe, now=101) == KEYS[1] + pool.record_result( + KEYS[1], + status, + now=100, + check_ttl_seconds=91, + rejected_cooldown_seconds=17, + ) + assert pool.select_available_key(KEYS, now=101) is None + probe.reset_mock() + with pytest.raises(NanoGPTKeyPoolExhausted, match="All configured"): + pool.select_key(KEYS, probe, now=101) + probe.assert_not_called() + snapshots.append(dict(pool._rejected_until)) + assert pool.select_available_key(KEYS, now=100 + duration) == KEYS[0] + assert model_cooldown.entry_count == 0 + assert snapshots[0] == snapshots[1] + + +@pytest.mark.parametrize("scope", [{"model": "A"}, {"quota_bucket": "shared"}]) +@pytest.mark.parametrize("status", [429, 401]) +def test_scoped_credential_selection_respects_legacy_rest(scope, status): + for key in KEYS: + CredentialPool.record(PROVIDER, key, status, now=100) + assert CredentialPool.available(PROVIDER, KEYS, now=101, **scope) == [] + with patch.object(CredentialPool, "keys", return_value=KEYS): + with pytest.raises(ModelCooldownExhausted) as caught: + CredentialPool.select(PROVIDER, now=101, **scope) + assert caught.value.retry_after == (59 if status == 429 else 299) + assert CredentialPool.available(PROVIDER, KEYS, now=400, **scope) == KEYS + + +@pytest.mark.parametrize("pool", [NanoGPTKeyPool, NanoGPTUnifiedKeyPool]) +@pytest.mark.parametrize("scope", [{"model": "A"}, {"quota_bucket": "shared"}]) +def test_scoped_nanogpt_selection_respects_legacy_rest(pool, scope): + assert pool.select_key(KEYS, Mock(return_value=200), now=99) == KEYS[0] + pool.record_result(KEYS[0], 429, now=100) + assert pool.select_available_key(KEYS, now=101, **scope) == KEYS[1] + probe = Mock(return_value=200) + assert pool.select_key(KEYS, probe, now=101, **scope) == KEYS[1] + probe.assert_called_once_with(KEYS[1]) + pool.record_result(KEYS[1], 401, now=100) + probe.reset_mock() + with pytest.raises(ModelCooldownExhausted) as caught: + pool.select_key(KEYS, probe, now=101, **scope) + assert caught.value.retry_after == 59 + with pytest.raises(ModelCooldownExhausted): + pool.select_available_key(KEYS, now=101, **scope) + probe.assert_not_called() + + +def test_scoped_auth_rests_legacy_credential_until_same_capped_deadline(monkeypatch): + monkeypatch.setenv("MODEL_COOLDOWN_MAX_SECONDS", "10") + CredentialPool.record( + PROVIDER, + KEYS[0], + 403, + model="A", + credential_wide_auth=True, + retry_after_seconds=999, + now=100, + ) + assert CredentialPool._resting == {(PROVIDER, KEYS[0]): 110} + with patch.object(CredentialPool, "keys", return_value=KEYS): + assert CredentialPool.select(PROVIDER, now=101) == KEYS[1] + assert CredentialPool.available(PROVIDER, KEYS, now=110) == KEYS + + +@pytest.mark.parametrize("pool", [NanoGPTKeyPool, NanoGPTUnifiedKeyPool]) +def test_scoped_nanogpt_auth_rests_legacy_key_and_invalidates_active(pool): + assert pool.select_key(KEYS, Mock(return_value=200), model="A", now=99) == KEYS[0] + pool.record_result(KEYS[0], 401, model="A", now=100, retry_after_seconds=11) + assert pool._active_key is None + assert pool._rejected_until == {KEYS[0]: 111} + assert pool.select_available_key(KEYS, now=101) == KEYS[1] + assert pool.select_available_key(KEYS, now=111) == KEYS[0] + + +@pytest.mark.parametrize("status", [429, 401]) +def test_scoped_success_preserves_legacy_credential_rest(status): + CredentialPool.record(PROVIDER, KEYS[0], 429, model="A", now=99) + CredentialPool.record(PROVIDER, KEYS[0], status, now=100) + before = dict(CredentialPool._resting) + CredentialPool.record(PROVIDER, KEYS[0], 200, model="A", now=101) + assert CredentialPool._resting == before + assert model_cooldown.entry_count == 0 + assert CredentialPool.available(PROVIDER, KEYS, now=102) == KEYS[1:] + assert CredentialPool.available(PROVIDER, KEYS, model="A", now=102) == KEYS[1:] + + +@pytest.mark.parametrize("pool", [NanoGPTKeyPool, NanoGPTUnifiedKeyPool]) +def test_scoped_nanogpt_success_preserves_legacy_rest(pool): + pool.record_result(KEYS[0], 429, model="A", now=99) + pool.record_result(KEYS[0], 401, now=100) + before = dict(pool._rejected_until) + pool.record_result(KEYS[0], 200, model="A", now=101) + assert pool._rejected_until == before + assert model_cooldown.entry_count == 0 + assert pool.select_available_key(KEYS, now=102) == KEYS[1] + assert pool.select_available_key(KEYS, model="A", now=102) == KEYS[1] + + +def test_record_headers_accepts_explicit_context(): + CredentialPool.record_headers( + PROVIDER, {"aUtHoRiZaTiOn": "Bearer " + KEYS[0]}, 429, model="A", now=100 + ) + assert CredentialPool.available(PROVIDER, KEYS, model="A", now=101) == KEYS[1:] + + +@pytest.mark.parametrize("pool", [NanoGPTKeyPool, NanoGPTUnifiedKeyPool]) +def test_nanogpt_model_selection_keeps_precedence_and_skips_probes_when_cooling(pool): + probe = Mock(return_value=200) + assert pool.select_key(KEYS, probe, model="A", now=100) == KEYS[0] + pool.record_result(KEYS[0], 429, model="A", now=101) + assert pool.select_key(KEYS, probe, model="B", now=102) == KEYS[0] + assert pool.select_key(KEYS, probe, model="A", now=102) == KEYS[1] + pool.record_result(KEYS[1], 429, model="A", now=103) + probe.reset_mock() + with pytest.raises(ModelCooldownExhausted): + pool.select_key(KEYS, probe, model="A", now=104) + probe.assert_not_called() + assert pool._rejected_until == {} + + +def test_nanogpt_available_and_invalidate_use_same_model_state(): + NanoGPTUnifiedKeyPool.invalidate(KEYS[0], 429, model="A", now=100) + assert ( + NanoGPTUnifiedKeyPool.select_available_key(KEYS, model="A", now=101) == KEYS[1] + ) + assert ( + NanoGPTUnifiedKeyPool.select_available_key(KEYS, model="B", now=101) == KEYS[0] + ) + assert NanoGPTKeyPool.select_available_key(KEYS, model="A", now=101) == KEYS[0] + + +def test_nanogpt_probe_rejection_is_auth_evidence_not_model_quota(): + probe = Mock(side_effect=[401, 200]) + assert NanoGPTKeyPool.select_key(KEYS, probe, model="A", now=100) == KEYS[1] + assert NanoGPTKeyPool.select_available_key(KEYS, model="B", now=101) == KEYS[1] + + +def test_nanogpt_filtering_does_not_turn_a_multi_key_pool_into_a_probe_bypass(): + NanoGPTKeyPool.record_result(KEYS[0], 429, model="A", now=100) + probe = Mock(return_value=200) + assert NanoGPTKeyPool.select_key(KEYS, probe, model="A", now=101) == KEYS[1] + probe.assert_called_once_with(KEYS[1]) + + +def test_nanogpt_probe_permission_error_keeps_previously_validated_key(): + assert NanoGPTKeyPool.select_key(KEYS[:1], Mock(), model="A", now=100) == KEYS[0] + NanoGPTKeyPool.record_result(KEYS[0], 200, model="A", now=101) + assert ( + NanoGPTKeyPool.select_key( + KEYS[:1], + Mock(return_value=403), + model="A", + now=102, + check_every_requests=1, + ) + == KEYS[0] + ) + + +def test_nanogpt_scoped_success_does_not_clear_other_bucket(): + NanoGPTKeyPool.record_result(KEYS[0], 429, model="A", now=100) + NanoGPTKeyPool.record_result(KEYS[0], 200, model="B", now=101) + assert NanoGPTKeyPool.select_available_key(KEYS, model="A", now=102) == KEYS[1] + + +def test_storage_is_hmac_bounded_expires_and_fails_closed_at_capacity(): + store = ModelCooldown(max_entries=2) + outcome = classify_upstream_outcome(429) + for key in KEYS: + store.record(PROVIDER, key, outcome, model="A", now=100, max_seconds=10) + assert store.entry_count == 2 + assert all(key not in repr(store.__dict__) for key in KEYS) + digest = store.credential_id(PROVIDER, KEYS[0]) + assert len(digest) == 64 + assert digest == store.credential_id(PROVIDER, KEYS[0]) + store.record( + PROVIDER, "synthetic-overflow", outcome, model="A", now=100, max_seconds=10 + ) + assert store.entry_count == 2 + with pytest.raises(ModelCooldownCapacity): + store.available( + PROVIDER, ["synthetic-overflow"], model="A", now=101, max_seconds=10 + ) + assert store.available(PROVIDER, KEYS, model="A", now=110, max_seconds=10) == KEYS + assert store.entry_count == 0 + + +def test_concurrent_updates_and_fractional_retry_bound(): + store = ModelCooldown() + with ThreadPoolExecutor(max_workers=4) as executor: + list( + executor.map( + lambda _: store.record( + PROVIDER, + KEYS[0], + classify_upstream_outcome(429), + model="A", + now=100, + retry_after_seconds=1.1, + max_seconds=2, + ), + range(100), + ) + ) + with pytest.raises(ModelCooldownExhausted) as caught: + store.select(PROVIDER, KEYS[:1], model="A", now=100.2, max_seconds=2) + assert caught.value.retry_after == 1 + assert store.entry_count == 1 diff --git a/worker/container-env.mjs b/worker/container-env.mjs index f678771b..3d938ddf 100644 --- a/worker/container-env.mjs +++ b/worker/container-env.mjs @@ -128,6 +128,8 @@ const DIRECT_ENV_KEYS = [ "NANOGPT_KEY_CHECK_TTL_SECONDS", "NANOGPT_KEY_CHECK_EVERY_REQUESTS", "NANOGPT_KEY_REJECTED_COOLDOWN_SECONDS", + "MODEL_COOLDOWN_ENABLED", + "MODEL_COOLDOWN_MAX_SECONDS", "NANOGPT_PREFERRED_KEY_INDEX", "NAVYAI_API_KEY", "NAVYAI_BASE_URL",