From 2e27ecc0b147a660f3167c24952752c489e002c8 Mon Sep 17 00:00:00 2001 From: Abhyudaya Baiju Tharakan Date: Fri, 9 Oct 2026 07:31:22 +0530 Subject: [PATCH] feat(pools): add opt-in reset-aware account pool scheduling Behind POOL_RESET_SCHEDULING_ENABLED, account pools use fresh reset observations from usage snapshots and response metadata the gateway already receives. Within the existing pool and priority tier, freshly exhausted accounts are skipped and the earliest fresh reset goes first; stale, missing or contradictory observations keep the existing order. When every account is cooling or exhausted, the existing 429 carries a Retry-After from the soonest fresh reset instead of sending another request. NanoGPT key validation keeps its existing cadence, and the scheduler adds no probes. Observations live in process for POOL_OBSERVATION_TTL_SECONDS and store only HMAC credential identities. --- docs/pool-reset-scheduling.md | 93 +++++ services/credential_pool.py | 54 ++- services/nanogpt_key_pool.py | 58 ++- services/pool_reset_schedule.py | 240 +++++++++++++ services/provider_usage_normalizers.py | 67 ++++ services/provider_usage_service.py | 12 + tests/test_pool_reset_scheduling.py | 471 +++++++++++++++++++++++++ worker/container-env.mjs | 2 + 8 files changed, 995 insertions(+), 2 deletions(-) create mode 100644 docs/pool-reset-scheduling.md create mode 100644 services/pool_reset_schedule.py create mode 100644 tests/test_pool_reset_scheduling.py diff --git a/docs/pool-reset-scheduling.md b/docs/pool-reset-scheduling.md new file mode 100644 index 00000000..d8f44af4 --- /dev/null +++ b/docs/pool-reset-scheduling.md @@ -0,0 +1,93 @@ +# Reset-aware account pool scheduling + +Reset scheduling is disabled by default. It refines account choice inside the +provider pool and priority tier already selected by the gateway. It does not +change provider order, model choice, authorization, request bodies or replay +permission. + +```sh +POOL_RESET_SCHEDULING_ENABLED=false +POOL_OBSERVATION_TTL_SECONDS=60 +``` + +To enable local reset scheduling: + +```sh +POOL_RESET_SCHEDULING_ENABLED=true +POOL_OBSERVATION_TTL_SECONDS=60 +``` + +An empty flag means disabled. An empty TTL means 60 seconds. The TTL accepts +integers from 5 through 3600 seconds. A malformed setting disables scheduling +and emits one warning per setting per process, without its value. + +## Account choice + +The scheduler first applies the existing credential and model cooldowns. Fresh +quota windows with zero remaining quota also exclude an account. Among accounts +with fresh usable observations, the earliest relevant reset wins, with the +configured account order breaking ties. Accounts with no fresh usable evidence +retain their existing positions; only positions with fresh evidence reorder. +For example, two observed accounts with resets in 100 and 20 seconds exchange +order, while an unobserved account between them stays in its position. + +Account observations apply across models. Model observations apply only to the +same model; an explicitly supplied quota bucket shares observations only within +that bucket. Raw and unified NanoGPT pools have distinct identities. Existing +prompt cache affinity may prefer an account only after this eligibility filter. + +Contradictory quota totals, negative or nonfinite values, conflicting absolute +and relative reset timestamps, missing reset/quota pairs, expired windows and +stale observations do not change legacy eligibility or ordering. Remaining quota +can be derived from consistent limit and used values. Multiple relevant windows +are checked independently: any fresh exhausted window excludes the account. +An expired reset stops supplying advice; it does not assert that quota renewed. + +When every configured account is cooling or freshly exhausted, selection raises +the existing model cooldown HTTP 429 error before dispatch. Its error body stays +the same. `Retry-After` uses the soonest fresh reset, rounded up and bounded to +1–3600 seconds, also respecting a smaller configured model cooldown maximum. +Without fresh reset evidence it uses the existing cooldown advice. Advice never +authorizes another request or clears a cooldown. The gateway's cooldown error +adapter must be registered to expose `Retry-After` on HTTP responses. + +## Observation sources and cost + +The scheduler consumes normalized windows only from authorized usage snapshots +already fetched by the gateway and from metadata already supplied with classified +request results. Cached snapshots do not renew observation freshness. NanoGPT's +subscription usage endpoint is excluded as a scheduling source. No observations +come from browser sessions, subscriptions, rewards or check-ins. + +`record_pool_usage(provider, credential, windows, observed_at=..., model=..., +quota_bucket=...)` consumes already received normalized windows. Pool result +methods also accept `usage_windows`; a classified quota throttle with existing +`retry_after_seconds` supplies exhausted quota and a relative reset. Callers must +pass trusted response metadata with the actual credential and dispatch scope. +Unscoped metadata is account-wide; no model scope is inferred from an unrelated +request. The scheduler itself performs no network, storage or refresh calls. + +With reset scheduling enabled, NanoGPT's existing catalog validation probes +keep their cadence. The scheduler adds no probes of its own; it filters freshly +exhausted keys and orders eligible keys before the existing validation logic +runs. A validated active key stays in use until its existing re-check unless a +fresh observation shows it exhausted. The selection method that does not probe +continues to use local cooldown and quota evidence without claiming validation. + +## Retention and bounds + +State is process-local and disappears on restart. There is no distributed +account-state guarantee and no persistent table. At most 10,000 account/scope +observations are retained, with at most 32 quota windows per observation. +Expired observations are removed before the oldest received observation is +evicted for capacity. Newer observations supersede older observations for the +same identity and scope; delayed older updates cannot replace fresh evidence. + +Only salted HMAC credential identities are stored, never keys or key prefixes. +Neither observations nor requests are logged. Freshness is checked with wall +and monotonic clocks. Future timestamp skew is tolerated up to five seconds, +without extending the TTL; larger skew and backward clock movement invalidate +observations. Reset timestamps are accepted up to one year after observation. +TTL expiration, missing evidence and capacity eviction restore existing account +selection; these observations are scheduling advice, not a quota accounting +authority. They cannot certify provider metering or live reset behavior. diff --git a/services/credential_pool.py b/services/credential_pool.py index 574bb38b..6c1fbfc9 100644 --- a/services/credential_pool.py +++ b/services/credential_pool.py @@ -18,7 +18,8 @@ from providers.codex_everywhere import CODEX_EVERYWHERE_POOLS from services.auth_primitives import usable_credential -from services.model_cooldown import model_cooldown, settings +from services.model_cooldown import ModelCooldownExhausted, model_cooldown, settings +from services import pool_reset_schedule as reset_schedule from services.upstream_outcome import UpstreamOutcome, classify_upstream_outcome REFUSED_REST_SECONDS = 300 @@ -77,6 +78,11 @@ def available( ) -> list[str]: """Configured keys that are not resting, in preference order.""" keys = cls.keys(provider) if keys is None else keys + if reset_schedule.settings().enabled: + return cls._scheduled_available( + provider, keys, now=now, model=model, quota_bucket=quota_bucket, + require_eligible=require_eligible, + ) config = settings() if config.enabled and (model or quota_bucket): with cls._lock: @@ -102,6 +108,37 @@ def available( key for key in keys if cls._resting.get((provider, key), 0) <= current ] + @classmethod + def _scheduled_available( + cls, provider: str, keys: list[str], *, now: float | None = None, + model: str | None = None, quota_bucket: str | None = None, + require_eligible: bool = False, + ) -> list[str]: + current = time.monotonic() if now is None else now + config = settings() + with cls._lock: + legacy = {key: cls._resting.get((provider, key), 0) for key in keys} + eligible = [key for key in keys if legacy[key] <= current] + fallback = min((max(1, until - current) for until in legacy.values()), default=60) + if config.enabled and (model or quota_bucket): + eligible = model_cooldown.available( + provider, keys, model=model, quota_bucket=quota_bucket, + max_seconds=config.max_seconds, now=current, legacy_rest_until=legacy, + ) + if not eligible and require_eligible: + try: + model_cooldown.select( + provider, keys, model=model, quota_bucket=quota_bucket, + max_seconds=config.max_seconds, now=current, legacy_rest_until=legacy, + ) + except ModelCooldownExhausted as error: + fallback = error.retry_after + return reset_schedule.pool_reset_schedule.admit( + provider, keys, eligible, model=model, quota_bucket=quota_bucket, + require_eligible=require_eligible, fallback=fallback, + max_seconds=config.max_seconds if config.enabled else 3600, + ) + @classmethod def select( cls, @@ -113,6 +150,13 @@ def select( ) -> str | None: """The key to use now; the first configured one when every key is resting.""" keys = cls.keys(provider) + if reset_schedule.settings().enabled: + available = cls._scheduled_available( + provider, keys, now=now, model=model, quota_bucket=quota_bucket, + require_eligible=True, + ) + from services.prompt_cache_affinity import prefer_credential + return prefer_credential(provider, model, available) or next(iter(available), None) config = settings() if config.enabled and (model or quota_bucket): with cls._lock: @@ -148,9 +192,15 @@ def record( outcome: UpstreamOutcome | None = None, credential_wide_auth: bool | None = None, retry_after_seconds: float | None = None, + usage_windows: list[Mapping] | None = None, ) -> None: if not key or provider not in POOLED_KEY_ENVS: return + reset_schedule.record_result_observation( + provider, key, status, model=model, quota_bucket=quota_bucket, + outcome=outcome, retry_after_seconds=retry_after_seconds, + usage_windows=usage_windows, + ) config = settings() if config.enabled and (model or quota_bucket): classified = outcome or classify_upstream_outcome(status) @@ -204,6 +254,7 @@ def record_headers( outcome: UpstreamOutcome | None = None, credential_wide_auth: bool | None = None, retry_after_seconds: float | None = None, + usage_windows: list[Mapping] | None = None, now: float | None = None, ) -> None: """Record the result for the bearer key a request carried.""" @@ -227,6 +278,7 @@ def record_headers( outcome=outcome, credential_wide_auth=credential_wide_auth, retry_after_seconds=retry_after_seconds, + usage_windows=usage_windows, now=now, ) diff --git a/services/nanogpt_key_pool.py b/services/nanogpt_key_pool.py index 44ad3229..d3f1baf4 100644 --- a/services/nanogpt_key_pool.py +++ b/services/nanogpt_key_pool.py @@ -10,7 +10,8 @@ from typing import ClassVar from services.auth_primitives import is_placeholder_credential -from services.model_cooldown import model_cooldown, settings +from services.model_cooldown import ModelCooldownExhausted, model_cooldown, settings +from services import pool_reset_schedule as reset_schedule from services.upstream_outcome import UpstreamOutcome, classify_upstream_outcome _DIRECT_KEY_NAMES = ("NANOGPT_API_KEY", "NANO_GPT_KEY") @@ -118,6 +119,8 @@ def select_available_key( """Select an uncooled credential without claiming a successful probe.""" configured = list(dict.fromkeys(key.strip() for key in keys if key.strip())) now = time.monotonic() if now is None else now + if reset_schedule.settings().enabled: + return cls._scheduled_key(configured, model=model, quota_bucket=quota_bucket, now=now) config = settings() scoped = config.enabled and bool(model or quota_bucket) with cls._lock: @@ -138,6 +141,48 @@ def select_available_key( (key for key in ordered if cls._rejected_until.get(key, 0) <= now), None ) + @classmethod + def _scheduled_key( + cls, configured: list[str], *, model: str | None = None, + quota_bucket: str | None = None, now: float, + ) -> str | None: + with cls._lock: + cls._prune(configured, now) + ordered = [cls._active_key] if cls._active_key in configured else [] + ordered.extend(key for key in configured if key not in ordered) + return next(iter(cls._scheduled_candidates( + configured, ordered, model=model, quota_bucket=quota_bucket, now=now, + )), None) + + @classmethod + def _scheduled_candidates( + cls, configured: list[str], ordered: list[str], *, + model: str | None = None, quota_bucket: str | None = None, now: float, + ) -> list[str]: + config = settings() + eligible = [key for key in ordered if cls._rejected_until.get(key, 0) <= now] + fallback = min((max(1, cls._rejected_until.get(key, now + 60) - now) for key in configured), default=60) + if config.enabled and (model or quota_bucket): + eligible = model_cooldown.available( + cls._model_cooldown_provider, ordered, model=model, + quota_bucket=quota_bucket, max_seconds=config.max_seconds, + now=now, legacy_rest_until=cls._rejected_until, + ) + if not eligible: + try: + model_cooldown.select( + cls._model_cooldown_provider, ordered, model=model, + quota_bucket=quota_bucket, max_seconds=config.max_seconds, + now=now, legacy_rest_until=cls._rejected_until, + ) + except ModelCooldownExhausted as error: + fallback = error.retry_after + return reset_schedule.pool_reset_schedule.admit( + cls._model_cooldown_provider, configured, eligible, model=model, + quota_bucket=quota_bucket, require_eligible=True, fallback=fallback, + max_seconds=config.max_seconds if config.enabled else 3600, + ) + @classmethod def select_key( cls, @@ -161,6 +206,11 @@ def select_key( single_configured_key = len(configured) == 1 with cls._lock: cls._prune(configured, checked_at) + if reset_schedule.settings().enabled: + configured = cls._scheduled_candidates( + configured, configured, model=model, + quota_bucket=quota_bucket, now=checked_at, + ) if scoped: eligible = model_cooldown.available( cls._model_cooldown_provider, @@ -308,10 +358,16 @@ def record_result( outcome: UpstreamOutcome | None = None, credential_wide_auth: bool | None = None, retry_after_seconds: float | None = None, + usage_windows: list[Mapping] | None = None, ) -> None: """Record one upstream request and invalidate definite key failures.""" if not key: return + reset_schedule.record_result_observation( + cls._model_cooldown_provider, key, status, model=model, + quota_bucket=quota_bucket, outcome=outcome, + retry_after_seconds=retry_after_seconds, usage_windows=usage_windows, + ) if settings().enabled and (model or quota_bucket): cls._record_model_result( key, diff --git a/services/pool_reset_schedule.py b/services/pool_reset_schedule.py new file mode 100644 index 00000000..ed8f026e --- /dev/null +++ b/services/pool_reset_schedule.py @@ -0,0 +1,240 @@ +"""Local reset observations refine scheduling only within an eligible account pool.""" + +from __future__ import annotations + +import hashlib +import hmac +import logging +import math +import os +import re +import secrets +import threading +import time +from collections import OrderedDict +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from typing import Any + +from services.model_cooldown import ModelCooldownExhausted +from services.provider_usage_normalizers import reset_usage_windows +from services.upstream_outcome import UpstreamOutcome, classify_upstream_outcome + +logger = logging.getLogger(__name__) +MAX_OBSERVATIONS = 10_000 +MAX_RETRY_SECONDS = 3600 +_warning_lock = threading.Lock() +_warned_settings: set[str] = set() + + +@dataclass(frozen=True) +class ResetSettings: + enabled: bool = False + ttl_seconds: int = 60 + + +def _invalid_setting(name: str) -> None: + with _warning_lock: + if name not in _warned_settings: + _warned_settings.add(name) + logger.warning("Invalid %s; pool reset scheduling disabled", name) + + +def settings(environ: Mapping[str, str] | None = None) -> ResetSettings: + source = os.environ if environ is None else environ + flag = (source.get("POOL_RESET_SCHEDULING_ENABLED") or "false").strip().lower() or "false" + if flag not in {"true", "false", "1", "0", "yes", "no", "on", "off"}: + _invalid_setting("POOL_RESET_SCHEDULING_ENABLED") + return ResetSettings() + ttl = (source.get("POOL_OBSERVATION_TTL_SECONDS") or "60").strip() or "60" + try: + if not re.fullmatch(r"[0-9]+", ttl): + raise ValueError + seconds = int(ttl) + if not 5 <= seconds <= 3600: + raise ValueError + except ValueError: + _invalid_setting("POOL_OBSERVATION_TTL_SECONDS") + return ResetSettings() + return ResetSettings(flag in {"true", "1", "yes", "on"}, seconds) + + +@dataclass(frozen=True) +class ResetObservation: + observed_at: float + received_at: float + windows: tuple[tuple[float, float], ...] + + +class PoolResetSchedule: + """Retain opaque identities and validated windows, never credential material.""" + + def __init__( + self, *, clock: Callable[[], float] | None = None, + monotonic: Callable[[], float] | None = None, + max_entries: int = MAX_OBSERVATIONS, + ) -> None: + self._clock = clock or time.time + self._monotonic = monotonic or time.monotonic + self._max_entries = min(MAX_OBSERVATIONS, max(1, max_entries)) + self._salt = secrets.token_bytes(32) + self._lock = threading.RLock() + self._entries: OrderedDict[tuple[str, str], ResetObservation] = OrderedDict() + + def credential_id(self, provider: str, credential: str) -> str: + digest = hmac.new(self._salt, digestmod=hashlib.sha256) + for part in (provider, credential): + encoded = part.encode("utf-8") + digest.update(len(encoded).to_bytes(8, "big")) + digest.update(encoded) + return digest.hexdigest() + + def _scope(self, model: str | None, quota_bucket: str | None) -> str: + if quota_bucket: + return "bucket:" + quota_bucket + return "model:" + model if model else "account" + + def _prune(self, config: ResetSettings) -> None: + wall, mono = self._clock(), self._monotonic() + self._entries = OrderedDict( + (identity, observation) for identity, observation in self._entries.items() + if 0 <= wall - observation.observed_at < config.ttl_seconds + and 0 <= mono - observation.received_at < config.ttl_seconds + ) + + @property + def entry_count(self) -> int: + with self._lock: + self._prune(settings()) + return len(self._entries) + + def reset(self) -> None: + with self._lock: + self._entries.clear() + + def observe( + self, provider: str, credential_id: str, windows: Any, *, + observed_at: float | None = None, model: str | None = None, + quota_bucket: str | None = None, + ) -> None: + """Consume already authorized data; this method performs no I/O.""" + config = settings() + if not config.enabled or not isinstance(credential_id, str) or not re.fullmatch(r"[a-f0-9]{64}", credential_id): + return + if any(value is not None and (not isinstance(value, str) or len(value) > 256) for value in (model, quota_bucket)): + return + wall, mono = self._clock(), self._monotonic() + observed = wall if observed_at is None else observed_at + identity = (credential_id, self._scope(model, quota_bucket)) + with self._lock: + self._prune(config) + previous = self._entries.get(identity) + if previous is not None and not isinstance(observed, bool) and isinstance(observed, (int, float)) and observed < previous.observed_at: + return + self._entries.pop(identity, None) + try: + if isinstance(observed, bool) or not isinstance(observed, (int, float)) or not math.isfinite(observed): + return + except OverflowError: + return + # Small positive timestamp skew is tolerated without extending freshness. + if not -5 <= wall - observed < config.ttl_seconds: + return + observed = min(wall, observed) + normalized = reset_usage_windows(windows, observed) + if normalized is None: + return + self._entries[identity] = ResetObservation(observed, mono, normalized) + while len(self._entries) > self._max_entries: + self._entries.popitem(last=False) + + def _windows( + self, provider: str, key: str, model: str | None, quota_bucket: str | None, + ) -> list[tuple[float, float]]: + identity = self.credential_id(provider, key) + scope = self._scope(model, quota_bucket) + windows = [] + for name in dict.fromkeys(("account", scope)): + observation = self._entries.get((identity, name)) + if observation is not None: + windows.extend(window for window in observation.windows if window[1] > self._clock()) + return windows + + def available( + self, provider: str, eligible: Sequence[str], *, model: str | None = None, + quota_bucket: str | None = None, configured_order: Sequence[str] | None = None, + ) -> list[str]: + config = settings() + if not config.enabled: + return list(eligible) + with self._lock: + self._prune(config) + windows = {key: self._windows(provider, key, model, quota_bucket) for key in eligible} + available = [key for key in eligible if not any(remaining == 0 for remaining, _ in windows[key])] + order = {key: index for index, key in enumerate(configured_order or eligible)} + ranked = sorted( + (key for key in available if windows[key]), + key=lambda key: (min(reset for _, reset in windows[key]), order[key]), + ) + # Unknown accounts retain their existing position; only fresh slots reorder. + iterator = iter(ranked) + return [next(iterator) if windows[key] else key for key in available] + + def retry_after( + self, provider: str, keys: Sequence[str], *, model: str | None = None, + quota_bucket: str | None = None, fallback: float = 60, + max_seconds: int = MAX_RETRY_SECONDS, + ) -> int: + cap = min(MAX_RETRY_SECONDS, max(1, max_seconds)) + resets = [] + if settings().enabled: + with self._lock: + self._prune(settings()) + resets = [reset for key in keys for _, reset in self._windows(provider, key, model, quota_bucket)] + delay = min(resets) - self._clock() if resets else fallback + if not isinstance(delay, (int, float)) or isinstance(delay, bool) or not math.isfinite(delay): + delay = 60 + return min(cap, max(1, math.ceil(delay))) + + def admit( + self, provider: str, keys: Sequence[str], eligible: Sequence[str], *, + model: str | None = None, quota_bucket: str | None = None, + require_eligible: bool = False, fallback: float = 60, max_seconds: int = MAX_RETRY_SECONDS, + ) -> list[str]: + available = self.available(provider, eligible, model=model, quota_bucket=quota_bucket, configured_order=keys) + if keys and not available and require_eligible: + raise ModelCooldownExhausted(self.retry_after( + provider, keys, model=model, quota_bucket=quota_bucket, + fallback=fallback, max_seconds=max_seconds, + )) + return available + + +pool_reset_schedule = PoolResetSchedule() + + +def record_pool_usage( + provider: str, credential: str, windows: Any, *, observed_at: float | None = None, + model: str | None = None, quota_bucket: str | None = None, +) -> None: + if settings().enabled: + pool_reset_schedule.observe( + provider, pool_reset_schedule.credential_id(provider, credential), windows, + observed_at=observed_at, model=model, quota_bucket=quota_bucket, + ) + + +def record_result_observation( + provider: str, key: str, status: int, *, outcome: UpstreamOutcome | None = None, + retry_after_seconds: float | None = None, usage_windows: Any = None, + model: str | None = None, quota_bucket: str | None = None, +) -> None: + if not settings().enabled: + return + classified = outcome or classify_upstream_outcome(status) + if usage_windows is not None and classified.credential_health == "accepted": + record_pool_usage(provider, key, usage_windows, model=model, quota_bucket=quota_bucket) + elif classified.credential_health == "throttled": + if isinstance(retry_after_seconds, (int, float)) and not isinstance(retry_after_seconds, bool): + if 0 < retry_after_seconds <= 31_622_400: + record_pool_usage(provider, key, [{"remaining": 0, "resets_in_ms": retry_after_seconds * 1000}], model=model, quota_bucket=quota_bucket) diff --git a/services/provider_usage_normalizers.py b/services/provider_usage_normalizers.py index fad178a5..60215645 100644 --- a/services/provider_usage_normalizers.py +++ b/services/provider_usage_normalizers.py @@ -71,6 +71,73 @@ def _window( } +def _reset_epoch(value: Any) -> float | None: + if not isinstance(value, str): + return None + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + return parsed.timestamp() if parsed.tzinfo is not None else None + except (OverflowError, OSError, ValueError): + return None + + +def _available_quota(window: Mapping[str, Any]) -> float | None: + values = {name: _number(window.get(name)) for name in ("limit", "used", "remaining", "percent_used")} + if any(window.get(name) is not None and value is None for name, value in values.items()): + return None + try: + if any(value is not None and (value < 0 or not math.isfinite(value)) for value in values.values()): + return None + except OverflowError: + return None + limit, used, remaining, percent = (values[name] for name in values) + if percent is not None and percent > 100: + return None + if limit is not None and used is not None: + calculated = limit - used + if calculated < 0 or (remaining is not None and not math.isclose(calculated, remaining, abs_tol=1e-6)): + return None + remaining = calculated + if remaining is None or (limit is not None and remaining > limit): + return None + if percent is not None and limit is not None and limit > 0: + expected = 100 * (limit - remaining) / limit + if not math.isclose(expected, percent, abs_tol=0.01): + return None + return float(remaining) + + +def reset_usage_windows( + windows: Any, observed_at: float, +) -> tuple[tuple[float, float], ...] | None: + """Validate quota/reset pairs without changing the public usage snapshot.""" + if not isinstance(windows, (list, tuple)) or not windows or len(windows) > 32: + return None + normalized = [] + for window in windows: + if not isinstance(window, Mapping): + return None + remaining = _available_quota(window) + absolute = _reset_epoch(window.get("resets_at")) + relative = _number(window.get("resets_in_ms")) + if window.get("resets_at") is not None and absolute is None: + return None + if window.get("resets_in_ms") is not None and (relative is None or relative <= 0): + return None + try: + relative_at = observed_at + float(relative) / 1000 if relative is not None else None + except OverflowError: + return None + if absolute is not None and relative_at is not None and abs(absolute - relative_at) > 5: + return None + reset = absolute if absolute is not None else relative_at + # A bounded horizon includes annual quotas but rejects implausible clocks. + if remaining is None or reset is None or not 0 < reset - observed_at <= 31_622_400: + return None + normalized.append((remaining, reset)) + return tuple(normalized) + + def _normalize_navyai(payload: Mapping[str, Any]) -> dict[str, Any]: limits = _mapping(payload.get("limits")) usage = _mapping(payload.get("usage")) diff --git a/services/provider_usage_service.py b/services/provider_usage_service.py index 6b697a32..41c2e790 100644 --- a/services/provider_usage_service.py +++ b/services/provider_usage_service.py @@ -241,6 +241,7 @@ def _fetch( if 200 <= last_status < 300: payload = self._response_json(response) normalized = NORMALIZERS[probe.provider](payload) + self._observe_reset_usage(probe, credential, normalized) return { "status": "available", **normalized, @@ -280,6 +281,17 @@ def _fetch( ) return self._error_result(probe, error_code, last_status) + def _observe_reset_usage( + self, probe: ProviderUsageProbe, credential: str, normalized: Mapping[str, Any], + ) -> None: + # Subscription-account data is not an account-pool scheduling source. + if probe.provider == "nanogpt": + return + from services.pool_reset_schedule import record_pool_usage, settings + if settings().enabled: + observed = datetime.fromisoformat(self._utcnow().replace("Z", "+00:00")).timestamp() + record_pool_usage(probe.provider, credential, normalized.get("windows"), observed_at=observed) + def _response_json(self, response: Any) -> Mapping[str, Any]: content = bytes(response.content or b"") if not content or len(content) > MAX_USAGE_RESPONSE_BYTES: diff --git a/tests/test_pool_reset_scheduling.py b/tests/test_pool_reset_scheduling.py new file mode 100644 index 00000000..ca4cd7d6 --- /dev/null +++ b/tests/test_pool_reset_scheduling.py @@ -0,0 +1,471 @@ +"""Reset scheduling with synthetic identities, clocks, and upstream responses.""" + +import importlib +import json +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +from unittest.mock import Mock +from unittest.mock import patch + +import pytest +from flask import Flask, g + +PROVIDER = "ce-gpt-pro" +KEYS = ["synthetic-first", "synthetic-second", "synthetic-third"] + + +@pytest.fixture(autouse=True) +def isolated(monkeypatch): + global schedule, credentials, nano, cooldown + schedule = importlib.import_module("services.pool_reset_schedule") + credentials = importlib.import_module("services.credential_pool") + nano = importlib.import_module("services.nanogpt_key_pool") + cooldown = importlib.import_module("services.model_cooldown") + monkeypatch.setenv("POOL_RESET_SCHEDULING_ENABLED", "true") + monkeypatch.setenv("POOL_OBSERVATION_TTL_SECONDS", "60") + monkeypatch.setenv("MODEL_COOLDOWN_ENABLED", "false") + monkeypatch.setenv("MODEL_COOLDOWN_MAX_SECONDS", "3600") + monkeypatch.setenv("PROMPT_CACHE_AFFINITY_ENABLED", "false") + credentials.CredentialPool.reset() + nano.NanoGPTKeyPool.reset() + nano.NanoGPTUnifiedKeyPool.reset() + cooldown.model_cooldown.reset() + clock = [1000.0] + store = schedule.PoolResetSchedule(clock=lambda: clock[0], monotonic=lambda: clock[0]) + monkeypatch.setattr(schedule, "pool_reset_schedule", store) + monkeypatch.setattr(credentials.CredentialPool, "keys", staticmethod(lambda provider: KEYS)) + yield clock, store + credentials.CredentialPool.reset() + nano.NanoGPTKeyPool.reset() + nano.NanoGPTUnifiedKeyPool.reset() + cooldown.model_cooldown.reset() + + +def observe(store, key, delay, *, provider=PROVIDER, remaining=10, **kwargs): + store.observe(provider, store.credential_id(provider, key), [ + {"remaining": remaining, "resets_in_ms": delay * 1000} + ], observed_at=1000, **kwargs) + + +def test_fresh_reset_order_stable_ties_and_unknown_fallback(isolated): + clock, store = isolated + observe(store, KEYS[0], 100) + observe(store, KEYS[1], 20) + observe(store, KEYS[2], 20) + assert credentials.CredentialPool.available(PROVIDER, KEYS, now=1000) == [KEYS[1], KEYS[2], KEYS[0]] + assert credentials.CredentialPool.select(PROVIDER, now=1000) == KEYS[1] + assert store.available(PROVIDER, [KEYS[2], KEYS[1]], configured_order=KEYS) == KEYS[1:] + clock[0] = 1060 + assert credentials.CredentialPool.select(PROVIDER, now=1060) == KEYS[0] + assert store.entry_count == 0 + observe(store, KEYS[1], 200) + assert store.available(PROVIDER, KEYS) == KEYS + + +def test_exhaustion_and_cooling_excluded_despite_near_reset(isolated): + _, store = isolated + observe(store, KEYS[0], 1, remaining=0) + observe(store, KEYS[1], 2) + observe(store, KEYS[2], 100) + credentials.CredentialPool.record(PROVIDER, KEYS[1], 429, now=1000) + assert credentials.CredentialPool.available(PROVIDER, KEYS, now=1000) == KEYS[2:] + assert credentials.CredentialPool.select(PROVIDER, now=1000) == KEYS[2] + + +def test_account_model_bucket_and_pool_isolation(isolated): + _, store = isolated + observe(store, KEYS[0], 20, remaining=0, model="A") + assert store.available(PROVIDER, KEYS, model="A") == KEYS[1:] + assert store.available(PROVIDER, KEYS, model="B") == KEYS + assert store.available("ce-gpt-plus", KEYS, model="A") == KEYS + observe(store, KEYS[1], 30, remaining=0, quota_bucket="shared") + assert store.available(PROVIDER, KEYS, model="B", quota_bucket="shared") == [KEYS[0], KEYS[2]] + observe(store, KEYS[2], 40, remaining=0) + assert store.available(PROVIDER, KEYS, model="B") == KEYS[:2] + + +@pytest.mark.parametrize("window", [ + {"limit": 10, "used": 9, "remaining": 8, "resets_in_ms": 1000}, + {"remaining": -1, "resets_in_ms": 1000}, + {"remaining": True, "resets_in_ms": 1000}, + {"remaining": float("nan"), "resets_in_ms": 1000}, + {"limit": 10, "used": 10, "remaining": 0, "percent_used": 20, "resets_in_ms": 1000}, + {"remaining": 10, "resets_at": "bad-date"}, + {"remaining": 10, "resets_at": "1970-01-01T00:00:01Z"}, + {"remaining": 10, "resets_in_ms": 1000, "resets_at": "1970-01-01T00:30:00Z"}, +]) +def test_contradictory_or_missing_data_clears_old_evidence(isolated, window): + _, store = isolated + observe(store, KEYS[0], 10, remaining=0) + store.observe(PROVIDER, store.credential_id(PROVIDER, KEYS[0]), [window], observed_at=1000) + assert store.available(PROVIDER, KEYS) == KEYS + + +def test_derived_quota_multiple_windows_and_reset_rollover(isolated): + clock, store = isolated + store.observe(PROVIDER, store.credential_id(PROVIDER, KEYS[0]), [ + {"limit": 10, "used": 5, "resets_in_ms": 10000}, + {"remaining": 0, "resets_in_ms": 20000}, + ], observed_at=1000) + assert store.available(PROVIDER, KEYS) == KEYS[1:] + clock[0] = 1021 + assert store.available(PROVIDER, KEYS) == KEYS + + +def test_clock_skew_old_observations_and_retry_bounds(isolated): + clock, store = isolated + for observed_at in (900, 1010, float("inf")): + store.observe(PROVIDER, store.credential_id(PROVIDER, KEYS[0]), [ + {"remaining": 0, "resets_in_ms": 10000} + ], observed_at=observed_at) + assert store.available(PROVIDER, KEYS) == KEYS + observe(store, KEYS[0], 999999, remaining=0) + assert store.retry_after(PROVIDER, KEYS, fallback=5) == 3600 + clock[0] = 999 + assert store.available(PROVIDER, KEYS) == KEYS + assert store.retry_after(PROVIDER, KEYS, fallback=-1) == 1 + + +@pytest.mark.parametrize("flag", [None, "", "false", "0", "malformed-secret-value"]) +@pytest.mark.parametrize("model_cooldown", ["false", "true"]) +def test_disabled_sequences_cooldowns_and_errors_unchanged(isolated, monkeypatch, flag, model_cooldown): + monkeypatch.setenv("MODEL_COOLDOWN_ENABLED", model_cooldown) + + def sequence(value): + if value is None: + monkeypatch.delenv("POOL_RESET_SCHEDULING_ENABLED", raising=False) + else: + monkeypatch.setenv("POOL_RESET_SCHEDULING_ENABLED", value) + credentials.CredentialPool.reset() + nano.NanoGPTKeyPool.reset() + cooldown.model_cooldown.reset() + selected = [] + for key in KEYS: + selected.append(credentials.CredentialPool.select(PROVIDER, now=1000)) + credentials.CredentialPool.record(PROVIDER, key, 429, now=1000) + selected.append(credentials.CredentialPool.select(PROVIDER, now=1001)) + selected.append(credentials.CredentialPool.select(PROVIDER, now=1060)) + probe = Mock(return_value=200) + for key in KEYS: + selected.append(nano.NanoGPTKeyPool.select_key(KEYS, probe, now=1000)) + nano.NanoGPTKeyPool.record_result(key, 429, now=1000) + with pytest.raises(nano.NanoGPTKeyPoolExhausted) as error: + nano.NanoGPTKeyPool.select_key(KEYS, probe, now=1001) + return selected, dict(credentials.CredentialPool._resting), dict(nano.NanoGPTKeyPool._rejected_until), str(error.value), probe.call_args_list + + expected = sequence("false") + assert sequence(flag) == expected + assert isolated[1].entry_count == 0 + + +@pytest.mark.parametrize("value", ["bad-sensitive-value", "4", "3601", "1.5", "-1", "NaN"]) +def test_invalid_ttl_disables_once_without_value(monkeypatch, caplog, value): + schedule._warned_settings.clear() + monkeypatch.setenv("POOL_OBSERVATION_TTL_SECONDS", value) + assert not schedule.settings().enabled + assert not schedule.settings().enabled + records = [r.message for r in caplog.records if "POOL_OBSERVATION_TTL_SECONDS" in r.message] + assert len(records) == 1 + assert value not in records[0] + + +def test_empty_defaults_and_malformed_flag(monkeypatch, caplog): + monkeypatch.setenv("POOL_OBSERVATION_TTL_SECONDS", "") + assert schedule.settings().ttl_seconds == 60 + for value in ("5", "3600"): + monkeypatch.setenv("POOL_OBSERVATION_TTL_SECONDS", value) + assert schedule.settings().enabled + schedule._warned_settings.clear() + monkeypatch.setenv("POOL_RESET_SCHEDULING_ENABLED", "flag-sensitive-value") + assert not schedule.settings().enabled + assert not schedule.settings().enabled + assert len(caplog.records) == 1 + assert "flag-sensitive-value" not in caplog.text + + +@pytest.mark.parametrize("scoped", [False, True]) +def test_real_429_keeps_body_refines_advice_and_never_dispatches(isolated, monkeypatch, scoped): + monkeypatch.setenv("MODEL_COOLDOWN_ENABLED", "true" if scoped else "false") + _, store = isolated + for key, delay in zip(KEYS, [30, 10, 20]): + observe(store, key, delay, remaining=0, model="A") + credentials.CredentialPool.record(PROVIDER, key, 429, model="A", now=1000) + app = Flask(__name__) + app.register_error_handler(cooldown.ModelCooldownExhausted, cooldown.cooldown_error_response) + send = Mock() + + @app.before_request + def request_id(): + g.request_id = "synthetic-request-id" + + @app.post("/v1/test-reset") + def dispatch(): + credentials.CredentialPool.select(PROVIDER, model="A", now=1000) + send() + return {"ok": True} + + response = app.test_client().post("/v1/test-reset", json={"model": "A"}) + with app.test_request_context(): + g.request_id = "synthetic-request-id" + expected = cooldown.ModelCooldownExhausted(60).to_dict() + assert response.status_code == 429 + assert response.json == expected + assert response.headers["Retry-After"] == "10" + send.assert_not_called() + + +@pytest.mark.parametrize("pool_name", ["NanoGPTKeyPool", "NanoGPTUnifiedKeyPool"]) +@pytest.mark.parametrize("scoped", [False, True]) +@pytest.mark.parametrize("single", [False, True]) +def test_nanogpt_without_observations_preserves_validation_sequence(isolated, monkeypatch, pool_name, scoped, single): + monkeypatch.setenv("MODEL_COOLDOWN_ENABLED", "true" if scoped else "false") + pool = getattr(nano, pool_name) + keys = KEYS[:1] if single else KEYS + + def sequence(flag): + monkeypatch.setenv("POOL_RESET_SCHEDULING_ENABLED", flag) + pool.reset() + cooldown.model_cooldown.reset() + probe = Mock(side_effect=[503, 200, 200] if single else [401, 200, 503, 200, 401, 200]) + selected = [] + for now in (1000, 1001, 1002, 1003, 1014, 1015): + key = pool.select_key(keys, probe, now=now, model="A" if scoped else None, + check_ttl_seconds=10, check_every_requests=2) + selected.append(key) + pool.record_result(key, 200, now=now, model="A" if scoped else None) + return selected, probe.call_args_list, pool._active_until, pool._active_requests, dict(pool._rejected_until) + + expected = sequence("false") + assert expected[0] == ([KEYS[0]] * 6 if single else [KEYS[1], KEYS[1], KEYS[2], KEYS[2], KEYS[0], KEYS[0]]) + assert len(expected[1]) == (2 if single else 6) + assert sequence("true") == expected + + +@pytest.mark.parametrize("pool_name", ["NanoGPTKeyPool", "NanoGPTUnifiedKeyPool"]) +@pytest.mark.parametrize("scoped", [False, True]) +def test_nanogpt_exhausted_active_key_validates_replacement(isolated, monkeypatch, pool_name, scoped): + monkeypatch.setenv("MODEL_COOLDOWN_ENABLED", "true" if scoped else "false") + pool = getattr(nano, pool_name) + probe = Mock(return_value=200) + assert pool.select_key(KEYS[:2], probe, now=1000, model="A") == KEYS[0] + observe(isolated[1], KEYS[0], 10, provider=pool._model_cooldown_provider, remaining=0, model="A") + assert pool.select_key(KEYS[:2], probe, now=1001, model="A") == KEYS[1] + assert [call.args[0] for call in probe.call_args_list] == KEYS[:2] + + +@pytest.mark.parametrize("pool_name", ["NanoGPTKeyPool", "NanoGPTUnifiedKeyPool"]) +def test_nanogpt_reset_order_changes_first_probe_but_keeps_validated_active(isolated, pool_name): + pool = getattr(nano, pool_name) + provider = pool._model_cooldown_provider + observe(isolated[1], KEYS[0], 100, provider=provider) + observe(isolated[1], KEYS[1], 10, provider=provider) + probe = Mock(return_value=200) + assert pool.select_key(KEYS, probe, now=1000, check_every_requests=1) == KEYS[1] + observe(isolated[1], KEYS[0], 5, provider=provider) + assert pool.select_key(KEYS, probe, now=1001, check_every_requests=1) == KEYS[1] + probe.assert_called_once_with(KEYS[1]) + pool.record_result(KEYS[1], 200, now=1001) + assert pool.select_key(KEYS, probe, now=1002, check_every_requests=1) == KEYS[1] + assert [call.args[0] for call in probe.call_args_list] == [KEYS[1], KEYS[1]] + + +@pytest.mark.parametrize("pool_name", ["NanoGPTKeyPool", "NanoGPTUnifiedKeyPool"]) +@pytest.mark.parametrize("scoped", [False, True]) +@pytest.mark.parametrize("cooling", [False, True]) +def test_nanogpt_exhausted_or_cooling_pool_raises_without_probe(isolated, monkeypatch, pool_name, scoped, cooling): + monkeypatch.setenv("MODEL_COOLDOWN_ENABLED", "true" if scoped else "false") + pool = getattr(nano, pool_name) + for key, delay in zip(KEYS, [30, 10, 20]): + observe(isolated[1], key, delay, provider=pool._model_cooldown_provider, + remaining=10 if cooling else 0, model="A") + if cooling: + pool.record_result(key, 429, now=1000, model="A") + probe = Mock(side_effect=AssertionError("Exhausted selection must not probe")) + with pytest.raises(cooldown.ModelCooldownExhausted) as error: + pool.select_key(KEYS, probe, now=1000, model="A") + assert error.value.retry_after == 10 + probe.assert_not_called() + + +@pytest.mark.parametrize("pool_name", ["NanoGPTKeyPool", "NanoGPTUnifiedKeyPool"]) +def test_nanogpt_available_selection_keeps_no_probe_behavior(isolated, pool_name): + pool = getattr(nano, pool_name) + provider = pool._model_cooldown_provider + observe(isolated[1], KEYS[0], 100, provider=provider) + observe(isolated[1], KEYS[1], 10, provider=provider) + assert pool.select_available_key(KEYS, now=1000) == KEYS[1] + for key in KEYS: + pool.record_result(key, 429, retry_after_seconds=11, now=1000) + with pytest.raises(cooldown.ModelCooldownExhausted) as error: + pool.select_available_key(KEYS, now=1000) + assert error.value.retry_after == 11 + + +def test_record_result_metadata_and_cancelled_outcome(isolated): + _, store = isolated + credentials.CredentialPool.record(PROVIDER, KEYS[0], 429, retry_after_seconds=15, model="A", now=1000) + assert store.retry_after(PROVIDER, KEYS, model="A") == 15 + cancelled = importlib.import_module("services.upstream_outcome").classify_upstream_outcome(429, cancelled=True) + credentials.CredentialPool.record(PROVIDER, KEYS[1], 429, retry_after_seconds=1, outcome=cancelled, now=1000) + assert store.entry_count == 1 + nano.NanoGPTKeyPool.record_result(KEYS[1], 200, usage_windows=[ + {"remaining": 10, "resets_in_ms": 5000} + ], now=1000) + assert store.entry_count == 2 + + +def test_affinity_is_limited_to_healthy_candidates(isolated, monkeypatch): + affinity = importlib.import_module("services.prompt_cache_affinity") + seen = [] + monkeypatch.setattr(affinity, "prefer_credential", lambda provider, model, eligible: seen.append(eligible) or (KEYS[0] if KEYS[0] in eligible else None)) + observe(isolated[1], KEYS[0], 1, remaining=0) + assert credentials.CredentialPool.select(PROVIDER, model="A", now=1000) == KEYS[1] + assert seen == [KEYS[1:]] + + +def test_bounded_opaque_storage_ttl_oldest_and_concurrency(isolated): + clock, _ = isolated + store = schedule.PoolResetSchedule(clock=lambda: clock[0], monotonic=lambda: clock[0], max_entries=2) + for key in KEYS: + observe(store, key, 100) + assert store.entry_count == 2 + assert store.available(PROVIDER, KEYS[:1]) == KEYS[:1] + assert all(key not in repr(store.__dict__) for key in KEYS) + with ThreadPoolExecutor(max_workers=4) as executor: + list(executor.map(lambda _: observe(store, KEYS[1], 100), range(100))) + assert store.entry_count == 2 + clock[0] = 1061 + assert store.entry_count == 0 + assert schedule.PoolResetSchedule(max_entries=20000)._max_entries == 10000 + + +@pytest.mark.parametrize("enabled", ["false", "true"]) +def test_usage_snapshot_cache_never_refreshes_observation_or_adds_calls(isolated, monkeypatch, enabled): + monkeypatch.setenv("POOL_RESET_SCHEDULING_ENABLED", enabled) + usage = importlib.import_module("services.provider_usage_service") + clock, store = isolated + auth = Mock() + auth.get_api_key.return_value = KEYS[0] + auth.get_api_keys.return_value = KEYS + auth.provider_credential_env_names.return_value = () + metrics = Mock() + metrics.get_cost_summary.return_value = {} + metrics.get_provider_stats.return_value = {} + response = Mock(status_code=200, content=json.dumps({"usage": { + "tokens_remaining_today": 10, "resets_in_ms": 100000 + }}).encode()) + proxy = Mock() + proxy.make_request.return_value = response + service = usage.ProviderUsageService(config={"API_BASE_URLS": {"navyai": "https://invalid.example"}, "PROVIDER_USAGE_CACHE_TTL_SECONDS": 300}, auth_service=auth, metrics_service=metrics, proxy_service=proxy, monotonic=lambda: clock[0]) + monkeypatch.setattr(service, "_utcnow", lambda: datetime.fromtimestamp(clock[0], timezone.utc).isoformat()) + first = service.snapshot(["navyai"]) + assert store.entry_count == (1 if enabled == "true" else 0) + clock[0] = 1061 + second = service.snapshot(["navyai"]) + assert second["providers"][0]["source"]["cached"] + assert first["providers"][0]["windows"] == second["providers"][0]["windows"] + assert store.entry_count == 0 + assert proxy.make_request.call_count == 1 + response.close.assert_called_once() + + +def test_subscription_usage_is_not_a_scheduler_source(isolated, monkeypatch): + usage = importlib.import_module("services.provider_usage_service") + response = Mock(status_code=200, content=json.dumps({"daily": {"used": 0, "remaining": 10, "resetAt": 1010000}}).encode()) + proxy = Mock() + proxy.make_request.return_value = response + service = usage.ProviderUsageService(config={"NANOGPT_SUBSCRIPTION_BASE_URL": "https://invalid.example"}, auth_service=Mock(), metrics_service=Mock(), proxy_service=proxy) + service._fetch(usage.PROVIDER_USAGE_PROBES["nanogpt"], KEYS) + assert isolated[1].entry_count == 0 + proxy.make_request.assert_called_once() + + +def test_disabled_and_enabled_usage_snapshots_match_bytes(isolated, monkeypatch): + usage = importlib.import_module("services.provider_usage_service") + auth = Mock() + auth.get_api_key.return_value = KEYS[0] + auth.provider_credential_env_names.return_value = () + metrics = Mock() + metrics.get_cost_summary.return_value = {} + metrics.get_provider_stats.return_value = {} + proxy = Mock() + proxy.make_request.return_value = Mock(status_code=200, content=b'{"usage":{"tokens_remaining_today":10,"resets_in_ms":10000}}') + snapshots = [] + for flag in ("false", "true"): + monkeypatch.setenv("POOL_RESET_SCHEDULING_ENABLED", flag) + service = usage.ProviderUsageService(config={"API_BASE_URLS": {"navyai": "https://invalid.example"}}, auth_service=auth, metrics_service=metrics, proxy_service=proxy) + monkeypatch.setattr(service, "_utcnow", lambda: "1970-01-01T00:16:40Z") + snapshots.append(json.dumps(service.snapshot(["navyai"]), sort_keys=True)) + assert snapshots[0] == snapshots[1] + assert proxy.make_request.call_count == 2 + + +def test_observation_validation_and_delayed_updates(isolated): + _, store = isolated + identity = store.credential_id(PROVIDER, KEYS[0]) + observe(store, KEYS[0], 50, remaining=0) + store.observe(PROVIDER, identity, [{"remaining": 10, "resets_in_ms": 1000}], observed_at=999) + assert store.available(PROVIDER, KEYS) == KEYS[1:] + for invalid in (None, 42, "credential-prefix"): + store.observe(PROVIDER, invalid, [{"remaining": 0, "resets_in_ms": 1000}]) + store.observe(PROVIDER, identity, [{"remaining": 0, "resets_in_ms": 1000}], model="x" * 257) + assert store.entry_count == 1 + store.observe(PROVIDER, identity, [{"remaining": 10**1000, "resets_in_ms": 1000}]) + assert store.available(PROVIDER, KEYS) == KEYS + + +def test_monotonic_expiry_and_tolerated_future_skew(): + mono = [0.0] + store = schedule.PoolResetSchedule(clock=lambda: 1000, monotonic=lambda: mono[0]) + store.observe(PROVIDER, store.credential_id(PROVIDER, KEYS[0]), [ + {"remaining": 0, "resets_in_ms": 100000} + ], observed_at=1005) + assert store.available(PROVIDER, KEYS) == KEYS[1:] + mono[0] = 60 + assert store.available(PROVIDER, KEYS) == KEYS + assert store.entry_count == 0 + + +@pytest.mark.parametrize("enabled", ["false", "true"]) +def test_registered_raw_pool_path_preserves_wire_and_stops_before_post(isolated, monkeypatch, tmp_path, enabled): + monkeypatch.setenv("POOL_RESET_SCHEDULING_ENABLED", enabled) + monkeypatch.setenv("ADMIN_API_KEY", "admin-test-key") + monkeypatch.setenv("FLASK_SECRET_KEY", "flask-test-secret") + monkeypatch.setenv("JWT_SECRET", "jwt-test-secret") + for name in ("AUTH_DB_PATH", "RATE_LIMIT_DB_PATH", "MODEL_REGISTRY_DB_PATH"): + monkeypatch.setenv(name, str(tmp_path / (name + ".sqlite3"))) + with patch("env_loader.load_runtime_env"), patch("config.load_runtime_env"): + app_module = importlib.import_module("app") + with patch.object(app_module, "load_runtime_env"), patch("config.load_runtime_env"): + app = app_module.create_app() + app.config.update(WTF_CSRF_ENABLED=False, IMAGE_RELAY_CATALOG_AUTO_REFRESH=False, + PROVIDER_CATALOG_AUTO_REFRESH=False, PROMPT_CACHE_ENABLED=False) + monkeypatch.setattr(app_module.AuthService, "verify_api_key", lambda *args: {"username": "alice", "is_admin": True}) + store = isolated[1] + observe(store, KEYS[0], 100) + observe(store, KEYS[1], 10) + upstream = Mock(status_code=200, raw=None, content=b'{"exact":"provider-response"}', headers={"content-type": "application/json"}) + send = Mock(side_effect=lambda **kwargs: Mock(status_code=200, raw=None, content=upstream.content, headers=upstream.headers)) + monkeypatch.setattr(app_module.ProxyService, "make_request", send) + client = app.test_client() + headers = {"Authorization": "Bearer admin-test-key", "Content-Type": "application/json"} + wire = b'{ "model":"gpt-6.1-sol", "messages":[] }' + response = client.post("/ce-gpt-pro/v1/chat/completions", data=wire, headers=headers) + assert response.status_code == 200 + assert response.data == upstream.content + assert send.call_args.kwargs["data"] == wire + assert send.call_args.kwargs["headers"]["Authorization"] == "Bearer " + (KEYS[1] if enabled == "true" else KEYS[0]) + assert send.call_count == 1 + for key in KEYS: + observe(store, key, 10, remaining=0) + send.reset_mock() + response = client.post("/ce-gpt-pro/v1/chat/completions", data=wire, headers=headers) + if enabled == "true": + assert response.status_code == 429 + assert response.headers["Retry-After"] == "10" + assert response.json["error"] == "model_cooldown" + send.assert_not_called() + else: + assert response.status_code == 200 + assert response.data == upstream.content + assert send.call_count == 1 diff --git a/worker/container-env.mjs b/worker/container-env.mjs index 1a7994db..46fd0227 100644 --- a/worker/container-env.mjs +++ b/worker/container-env.mjs @@ -40,6 +40,8 @@ const DIRECT_ENV_KEYS = [ "CONNECTION_PROFILES_DB_PATH", "CONFIG_REVISION_SYNC_ENABLED", "CONFIG_SYNC_TTL_SECONDS", + "POOL_RESET_SCHEDULING_ENABLED", + "POOL_OBSERVATION_TTL_SECONDS", "GATEWAY_ALERTS_ENABLED", "GATEWAY_ALERT_WEBHOOK_ALLOWLIST", "CONFIG_SECURITY_TTL_SECONDS",