diff --git a/docs/pool-reset-scheduling.md b/docs/pool-reset-scheduling.md new file mode 100644 index 00000000..d8f44af4 --- /dev/null +++ b/docs/pool-reset-scheduling.md @@ -0,0 +1,93 @@ +# Reset-aware account pool scheduling + +Reset scheduling is disabled by default. It refines account choice inside the +provider pool and priority tier already selected by the gateway. It does not +change provider order, model choice, authorization, request bodies or replay +permission. + +```sh +POOL_RESET_SCHEDULING_ENABLED=false +POOL_OBSERVATION_TTL_SECONDS=60 +``` + +To enable local reset scheduling: + +```sh +POOL_RESET_SCHEDULING_ENABLED=true +POOL_OBSERVATION_TTL_SECONDS=60 +``` + +An empty flag means disabled. An empty TTL means 60 seconds. The TTL accepts +integers from 5 through 3600 seconds. A malformed setting disables scheduling +and emits one warning per setting per process, without its value. + +## Account choice + +The scheduler first applies the existing credential and model cooldowns. Fresh +quota windows with zero remaining quota also exclude an account. Among accounts +with fresh usable observations, the earliest relevant reset wins, with the +configured account order breaking ties. Accounts with no fresh usable evidence +retain their existing positions; only positions with fresh evidence reorder. +For example, two observed accounts with resets in 100 and 20 seconds exchange +order, while an unobserved account between them stays in its position. + +Account observations apply across models. Model observations apply only to the +same model; an explicitly supplied quota bucket shares observations only within +that bucket. Raw and unified NanoGPT pools have distinct identities. Existing +prompt cache affinity may prefer an account only after this eligibility filter. + +Contradictory quota totals, negative or nonfinite values, conflicting absolute +and relative reset timestamps, missing reset/quota pairs, expired windows and +stale observations do not change legacy eligibility or ordering. Remaining quota +can be derived from consistent limit and used values. Multiple relevant windows +are checked independently: any fresh exhausted window excludes the account. +An expired reset stops supplying advice; it does not assert that quota renewed. + +When every configured account is cooling or freshly exhausted, selection raises +the existing model cooldown HTTP 429 error before dispatch. Its error body stays +the same. `Retry-After` uses the soonest fresh reset, rounded up and bounded to +1–3600 seconds, also respecting a smaller configured model cooldown maximum. +Without fresh reset evidence it uses the existing cooldown advice. Advice never +authorizes another request or clears a cooldown. The gateway's cooldown error +adapter must be registered to expose `Retry-After` on HTTP responses. + +## Observation sources and cost + +The scheduler consumes normalized windows only from authorized usage snapshots +already fetched by the gateway and from metadata already supplied with classified +request results. Cached snapshots do not renew observation freshness. NanoGPT's +subscription usage endpoint is excluded as a scheduling source. No observations +come from browser sessions, subscriptions, rewards or check-ins. + +`record_pool_usage(provider, credential, windows, observed_at=..., model=..., +quota_bucket=...)` consumes already received normalized windows. Pool result +methods also accept `usage_windows`; a classified quota throttle with existing +`retry_after_seconds` supplies exhausted quota and a relative reset. Callers must +pass trusted response metadata with the actual credential and dispatch scope. +Unscoped metadata is account-wide; no model scope is inferred from an unrelated +request. The scheduler itself performs no network, storage or refresh calls. + +With reset scheduling enabled, NanoGPT's existing catalog validation probes +keep their cadence. The scheduler adds no probes of its own; it filters freshly +exhausted keys and orders eligible keys before the existing validation logic +runs. A validated active key stays in use until its existing re-check unless a +fresh observation shows it exhausted. The selection method that does not probe +continues to use local cooldown and quota evidence without claiming validation. + +## Retention and bounds + +State is process-local and disappears on restart. There is no distributed +account-state guarantee and no persistent table. At most 10,000 account/scope +observations are retained, with at most 32 quota windows per observation. +Expired observations are removed before the oldest received observation is +evicted for capacity. Newer observations supersede older observations for the +same identity and scope; delayed older updates cannot replace fresh evidence. + +Only salted HMAC credential identities are stored, never keys or key prefixes. +Neither observations nor requests are logged. Freshness is checked with wall +and monotonic clocks. Future timestamp skew is tolerated up to five seconds, +without extending the TTL; larger skew and backward clock movement invalidate +observations. Reset timestamps are accepted up to one year after observation. +TTL expiration, missing evidence and capacity eviction restore existing account +selection; these observations are scheduling advice, not a quota accounting +authority. They cannot certify provider metering or live reset behavior. diff --git a/services/credential_pool.py b/services/credential_pool.py index 574bb38b..6c1fbfc9 100644 --- a/services/credential_pool.py +++ b/services/credential_pool.py @@ -18,7 +18,8 @@ from providers.codex_everywhere import CODEX_EVERYWHERE_POOLS from services.auth_primitives import usable_credential -from services.model_cooldown import model_cooldown, settings +from services.model_cooldown import ModelCooldownExhausted, model_cooldown, settings +from services import pool_reset_schedule as reset_schedule from services.upstream_outcome import UpstreamOutcome, classify_upstream_outcome REFUSED_REST_SECONDS = 300 @@ -77,6 +78,11 @@ def available( ) -> list[str]: """Configured keys that are not resting, in preference order.""" keys = cls.keys(provider) if keys is None else keys + if reset_schedule.settings().enabled: + return cls._scheduled_available( + provider, keys, now=now, model=model, quota_bucket=quota_bucket, + require_eligible=require_eligible, + ) config = settings() if config.enabled and (model or quota_bucket): with cls._lock: @@ -102,6 +108,37 @@ def available( key for key in keys if cls._resting.get((provider, key), 0) <= current ] + @classmethod + def _scheduled_available( + cls, provider: str, keys: list[str], *, now: float | None = None, + model: str | None = None, quota_bucket: str | None = None, + require_eligible: bool = False, + ) -> list[str]: + current = time.monotonic() if now is None else now + config = settings() + with cls._lock: + legacy = {key: cls._resting.get((provider, key), 0) for key in keys} + eligible = [key for key in keys if legacy[key] <= current] + fallback = min((max(1, until - current) for until in legacy.values()), default=60) + if config.enabled and (model or quota_bucket): + eligible = model_cooldown.available( + provider, keys, model=model, quota_bucket=quota_bucket, + max_seconds=config.max_seconds, now=current, legacy_rest_until=legacy, + ) + if not eligible and require_eligible: + try: + model_cooldown.select( + provider, keys, model=model, quota_bucket=quota_bucket, + max_seconds=config.max_seconds, now=current, legacy_rest_until=legacy, + ) + except ModelCooldownExhausted as error: + fallback = error.retry_after + return reset_schedule.pool_reset_schedule.admit( + provider, keys, eligible, model=model, quota_bucket=quota_bucket, + require_eligible=require_eligible, fallback=fallback, + max_seconds=config.max_seconds if config.enabled else 3600, + ) + @classmethod def select( cls, @@ -113,6 +150,13 @@ def select( ) -> str | None: """The key to use now; the first configured one when every key is resting.""" keys = cls.keys(provider) + if reset_schedule.settings().enabled: + available = cls._scheduled_available( + provider, keys, now=now, model=model, quota_bucket=quota_bucket, + require_eligible=True, + ) + from services.prompt_cache_affinity import prefer_credential + return prefer_credential(provider, model, available) or next(iter(available), None) config = settings() if config.enabled and (model or quota_bucket): with cls._lock: @@ -148,9 +192,15 @@ def record( outcome: UpstreamOutcome | None = None, credential_wide_auth: bool | None = None, retry_after_seconds: float | None = None, + usage_windows: list[Mapping] | None = None, ) -> None: if not key or provider not in POOLED_KEY_ENVS: return + reset_schedule.record_result_observation( + provider, key, status, model=model, quota_bucket=quota_bucket, + outcome=outcome, retry_after_seconds=retry_after_seconds, + usage_windows=usage_windows, + ) config = settings() if config.enabled and (model or quota_bucket): classified = outcome or classify_upstream_outcome(status) @@ -204,6 +254,7 @@ def record_headers( outcome: UpstreamOutcome | None = None, credential_wide_auth: bool | None = None, retry_after_seconds: float | None = None, + usage_windows: list[Mapping] | None = None, now: float | None = None, ) -> None: """Record the result for the bearer key a request carried.""" @@ -227,6 +278,7 @@ def record_headers( outcome=outcome, credential_wide_auth=credential_wide_auth, retry_after_seconds=retry_after_seconds, + usage_windows=usage_windows, now=now, ) diff --git a/services/nanogpt_key_pool.py b/services/nanogpt_key_pool.py index 44ad3229..d3f1baf4 100644 --- a/services/nanogpt_key_pool.py +++ b/services/nanogpt_key_pool.py @@ -10,7 +10,8 @@ from typing import ClassVar from services.auth_primitives import is_placeholder_credential -from services.model_cooldown import model_cooldown, settings +from services.model_cooldown import ModelCooldownExhausted, model_cooldown, settings +from services import pool_reset_schedule as reset_schedule from services.upstream_outcome import UpstreamOutcome, classify_upstream_outcome _DIRECT_KEY_NAMES = ("NANOGPT_API_KEY", "NANO_GPT_KEY") @@ -118,6 +119,8 @@ def select_available_key( """Select an uncooled credential without claiming a successful probe.""" configured = list(dict.fromkeys(key.strip() for key in keys if key.strip())) now = time.monotonic() if now is None else now + if reset_schedule.settings().enabled: + return cls._scheduled_key(configured, model=model, quota_bucket=quota_bucket, now=now) config = settings() scoped = config.enabled and bool(model or quota_bucket) with cls._lock: @@ -138,6 +141,48 @@ def select_available_key( (key for key in ordered if cls._rejected_until.get(key, 0) <= now), None ) + @classmethod + def _scheduled_key( + cls, configured: list[str], *, model: str | None = None, + quota_bucket: str | None = None, now: float, + ) -> str | None: + with cls._lock: + cls._prune(configured, now) + ordered = [cls._active_key] if cls._active_key in configured else [] + ordered.extend(key for key in configured if key not in ordered) + return next(iter(cls._scheduled_candidates( + configured, ordered, model=model, quota_bucket=quota_bucket, now=now, + )), None) + + @classmethod + def _scheduled_candidates( + cls, configured: list[str], ordered: list[str], *, + model: str | None = None, quota_bucket: str | None = None, now: float, + ) -> list[str]: + config = settings() + eligible = [key for key in ordered if cls._rejected_until.get(key, 0) <= now] + fallback = min((max(1, cls._rejected_until.get(key, now + 60) - now) for key in configured), default=60) + if config.enabled and (model or quota_bucket): + eligible = model_cooldown.available( + cls._model_cooldown_provider, ordered, model=model, + quota_bucket=quota_bucket, max_seconds=config.max_seconds, + now=now, legacy_rest_until=cls._rejected_until, + ) + if not eligible: + try: + model_cooldown.select( + cls._model_cooldown_provider, ordered, model=model, + quota_bucket=quota_bucket, max_seconds=config.max_seconds, + now=now, legacy_rest_until=cls._rejected_until, + ) + except ModelCooldownExhausted as error: + fallback = error.retry_after + return reset_schedule.pool_reset_schedule.admit( + cls._model_cooldown_provider, configured, eligible, model=model, + quota_bucket=quota_bucket, require_eligible=True, fallback=fallback, + max_seconds=config.max_seconds if config.enabled else 3600, + ) + @classmethod def select_key( cls, @@ -161,6 +206,11 @@ def select_key( single_configured_key = len(configured) == 1 with cls._lock: cls._prune(configured, checked_at) + if reset_schedule.settings().enabled: + configured = cls._scheduled_candidates( + configured, configured, model=model, + quota_bucket=quota_bucket, now=checked_at, + ) if scoped: eligible = model_cooldown.available( cls._model_cooldown_provider, @@ -308,10 +358,16 @@ def record_result( outcome: UpstreamOutcome | None = None, credential_wide_auth: bool | None = None, retry_after_seconds: float | None = None, + usage_windows: list[Mapping] | None = None, ) -> None: """Record one upstream request and invalidate definite key failures.""" if not key: return + reset_schedule.record_result_observation( + cls._model_cooldown_provider, key, status, model=model, + quota_bucket=quota_bucket, outcome=outcome, + retry_after_seconds=retry_after_seconds, usage_windows=usage_windows, + ) if settings().enabled and (model or quota_bucket): cls._record_model_result( key, diff --git a/services/pool_reset_schedule.py b/services/pool_reset_schedule.py new file mode 100644 index 00000000..ed8f026e --- /dev/null +++ b/services/pool_reset_schedule.py @@ -0,0 +1,240 @@ +"""Local reset observations refine scheduling only within an eligible account pool.""" + +from __future__ import annotations + +import hashlib +import hmac +import logging +import math +import os +import re +import secrets +import threading +import time +from collections import OrderedDict +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from typing import Any + +from services.model_cooldown import ModelCooldownExhausted +from services.provider_usage_normalizers import reset_usage_windows +from services.upstream_outcome import UpstreamOutcome, classify_upstream_outcome + +logger = logging.getLogger(__name__) +MAX_OBSERVATIONS = 10_000 +MAX_RETRY_SECONDS = 3600 +_warning_lock = threading.Lock() +_warned_settings: set[str] = set() + + +@dataclass(frozen=True) +class ResetSettings: + enabled: bool = False + ttl_seconds: int = 60 + + +def _invalid_setting(name: str) -> None: + with _warning_lock: + if name not in _warned_settings: + _warned_settings.add(name) + logger.warning("Invalid %s; pool reset scheduling disabled", name) + + +def settings(environ: Mapping[str, str] | None = None) -> ResetSettings: + source = os.environ if environ is None else environ + flag = (source.get("POOL_RESET_SCHEDULING_ENABLED") or "false").strip().lower() or "false" + if flag not in {"true", "false", "1", "0", "yes", "no", "on", "off"}: + _invalid_setting("POOL_RESET_SCHEDULING_ENABLED") + return ResetSettings() + ttl = (source.get("POOL_OBSERVATION_TTL_SECONDS") or "60").strip() or "60" + try: + if not re.fullmatch(r"[0-9]+", ttl): + raise ValueError + seconds = int(ttl) + if not 5 <= seconds <= 3600: + raise ValueError + except ValueError: + _invalid_setting("POOL_OBSERVATION_TTL_SECONDS") + return ResetSettings() + return ResetSettings(flag in {"true", "1", "yes", "on"}, seconds) + + +@dataclass(frozen=True) +class ResetObservation: + observed_at: float + received_at: float + windows: tuple[tuple[float, float], ...] + + +class PoolResetSchedule: + """Retain opaque identities and validated windows, never credential material.""" + + def __init__( + self, *, clock: Callable[[], float] | None = None, + monotonic: Callable[[], float] | None = None, + max_entries: int = MAX_OBSERVATIONS, + ) -> None: + self._clock = clock or time.time + self._monotonic = monotonic or time.monotonic + self._max_entries = min(MAX_OBSERVATIONS, max(1, max_entries)) + self._salt = secrets.token_bytes(32) + self._lock = threading.RLock() + self._entries: OrderedDict[tuple[str, str], ResetObservation] = OrderedDict() + + def credential_id(self, provider: str, credential: str) -> str: + digest = hmac.new(self._salt, digestmod=hashlib.sha256) + for part in (provider, credential): + encoded = part.encode("utf-8") + digest.update(len(encoded).to_bytes(8, "big")) + digest.update(encoded) + return digest.hexdigest() + + def _scope(self, model: str | None, quota_bucket: str | None) -> str: + if quota_bucket: + return "bucket:" + quota_bucket + return "model:" + model if model else "account" + + def _prune(self, config: ResetSettings) -> None: + wall, mono = self._clock(), self._monotonic() + self._entries = OrderedDict( + (identity, observation) for identity, observation in self._entries.items() + if 0 <= wall - observation.observed_at < config.ttl_seconds + and 0 <= mono - observation.received_at < config.ttl_seconds + ) + + @property + def entry_count(self) -> int: + with self._lock: + self._prune(settings()) + return len(self._entries) + + def reset(self) -> None: + with self._lock: + self._entries.clear() + + def observe( + self, provider: str, credential_id: str, windows: Any, *, + observed_at: float | None = None, model: str | None = None, + quota_bucket: str | None = None, + ) -> None: + """Consume already authorized data; this method performs no I/O.""" + config = settings() + if not config.enabled or not isinstance(credential_id, str) or not re.fullmatch(r"[a-f0-9]{64}", credential_id): + return + if any(value is not None and (not isinstance(value, str) or len(value) > 256) for value in (model, quota_bucket)): + return + wall, mono = self._clock(), self._monotonic() + observed = wall if observed_at is None else observed_at + identity = (credential_id, self._scope(model, quota_bucket)) + with self._lock: + self._prune(config) + previous = self._entries.get(identity) + if previous is not None and not isinstance(observed, bool) and isinstance(observed, (int, float)) and observed < previous.observed_at: + return + self._entries.pop(identity, None) + try: + if isinstance(observed, bool) or not isinstance(observed, (int, float)) or not math.isfinite(observed): + return + except OverflowError: + return + # Small positive timestamp skew is tolerated without extending freshness. + if not -5 <= wall - observed < config.ttl_seconds: + return + observed = min(wall, observed) + normalized = reset_usage_windows(windows, observed) + if normalized is None: + return + self._entries[identity] = ResetObservation(observed, mono, normalized) + while len(self._entries) > self._max_entries: + self._entries.popitem(last=False) + + def _windows( + self, provider: str, key: str, model: str | None, quota_bucket: str | None, + ) -> list[tuple[float, float]]: + identity = self.credential_id(provider, key) + scope = self._scope(model, quota_bucket) + windows = [] + for name in dict.fromkeys(("account", scope)): + observation = self._entries.get((identity, name)) + if observation is not None: + windows.extend(window for window in observation.windows if window[1] > self._clock()) + return windows + + def available( + self, provider: str, eligible: Sequence[str], *, model: str | None = None, + quota_bucket: str | None = None, configured_order: Sequence[str] | None = None, + ) -> list[str]: + config = settings() + if not config.enabled: + return list(eligible) + with self._lock: + self._prune(config) + windows = {key: self._windows(provider, key, model, quota_bucket) for key in eligible} + available = [key for key in eligible if not any(remaining == 0 for remaining, _ in windows[key])] + order = {key: index for index, key in enumerate(configured_order or eligible)} + ranked = sorted( + (key for key in available if windows[key]), + key=lambda key: (min(reset for _, reset in windows[key]), order[key]), + ) + # Unknown accounts retain their existing position; only fresh slots reorder. + iterator = iter(ranked) + return [next(iterator) if windows[key] else key for key in available] + + def retry_after( + self, provider: str, keys: Sequence[str], *, model: str | None = None, + quota_bucket: str | None = None, fallback: float = 60, + max_seconds: int = MAX_RETRY_SECONDS, + ) -> int: + cap = min(MAX_RETRY_SECONDS, max(1, max_seconds)) + resets = [] + if settings().enabled: + with self._lock: + self._prune(settings()) + resets = [reset for key in keys for _, reset in self._windows(provider, key, model, quota_bucket)] + delay = min(resets) - self._clock() if resets else fallback + if not isinstance(delay, (int, float)) or isinstance(delay, bool) or not math.isfinite(delay): + delay = 60 + return min(cap, max(1, math.ceil(delay))) + + def admit( + self, provider: str, keys: Sequence[str], eligible: Sequence[str], *, + model: str | None = None, quota_bucket: str | None = None, + require_eligible: bool = False, fallback: float = 60, max_seconds: int = MAX_RETRY_SECONDS, + ) -> list[str]: + available = self.available(provider, eligible, model=model, quota_bucket=quota_bucket, configured_order=keys) + if keys and not available and require_eligible: + raise ModelCooldownExhausted(self.retry_after( + provider, keys, model=model, quota_bucket=quota_bucket, + fallback=fallback, max_seconds=max_seconds, + )) + return available + + +pool_reset_schedule = PoolResetSchedule() + + +def record_pool_usage( + provider: str, credential: str, windows: Any, *, observed_at: float | None = None, + model: str | None = None, quota_bucket: str | None = None, +) -> None: + if settings().enabled: + pool_reset_schedule.observe( + provider, pool_reset_schedule.credential_id(provider, credential), windows, + observed_at=observed_at, model=model, quota_bucket=quota_bucket, + ) + + +def record_result_observation( + provider: str, key: str, status: int, *, outcome: UpstreamOutcome | None = None, + retry_after_seconds: float | None = None, usage_windows: Any = None, + model: str | None = None, quota_bucket: str | None = None, +) -> None: + if not settings().enabled: + return + classified = outcome or classify_upstream_outcome(status) + if usage_windows is not None and classified.credential_health == "accepted": + record_pool_usage(provider, key, usage_windows, model=model, quota_bucket=quota_bucket) + elif classified.credential_health == "throttled": + if isinstance(retry_after_seconds, (int, float)) and not isinstance(retry_after_seconds, bool): + if 0 < retry_after_seconds <= 31_622_400: + record_pool_usage(provider, key, [{"remaining": 0, "resets_in_ms": retry_after_seconds * 1000}], model=model, quota_bucket=quota_bucket) diff --git a/services/provider_usage_normalizers.py b/services/provider_usage_normalizers.py index fad178a5..60215645 100644 --- a/services/provider_usage_normalizers.py +++ b/services/provider_usage_normalizers.py @@ -71,6 +71,73 @@ def _window( } +def _reset_epoch(value: Any) -> float | None: + if not isinstance(value, str): + return None + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + return parsed.timestamp() if parsed.tzinfo is not None else None + except (OverflowError, OSError, ValueError): + return None + + +def _available_quota(window: Mapping[str, Any]) -> float | None: + values = {name: _number(window.get(name)) for name in ("limit", "used", "remaining", "percent_used")} + if any(window.get(name) is not None and value is None for name, value in values.items()): + return None + try: + if any(value is not None and (value < 0 or not math.isfinite(value)) for value in values.values()): + return None + except OverflowError: + return None + limit, used, remaining, percent = (values[name] for name in values) + if percent is not None and percent > 100: + return None + if limit is not None and used is not None: + calculated = limit - used + if calculated < 0 or (remaining is not None and not math.isclose(calculated, remaining, abs_tol=1e-6)): + return None + remaining = calculated + if remaining is None or (limit is not None and remaining > limit): + return None + if percent is not None and limit is not None and limit > 0: + expected = 100 * (limit - remaining) / limit + if not math.isclose(expected, percent, abs_tol=0.01): + return None + return float(remaining) + + +def reset_usage_windows( + windows: Any, observed_at: float, +) -> tuple[tuple[float, float], ...] | None: + """Validate quota/reset pairs without changing the public usage snapshot.""" + if not isinstance(windows, (list, tuple)) or not windows or len(windows) > 32: + return None + normalized = [] + for window in windows: + if not isinstance(window, Mapping): + return None + remaining = _available_quota(window) + absolute = _reset_epoch(window.get("resets_at")) + relative = _number(window.get("resets_in_ms")) + if window.get("resets_at") is not None and absolute is None: + return None + if window.get("resets_in_ms") is not None and (relative is None or relative <= 0): + return None + try: + relative_at = observed_at + float(relative) / 1000 if relative is not None else None + except OverflowError: + return None + if absolute is not None and relative_at is not None and abs(absolute - relative_at) > 5: + return None + reset = absolute if absolute is not None else relative_at + # A bounded horizon includes annual quotas but rejects implausible clocks. + if remaining is None or reset is None or not 0 < reset - observed_at <= 31_622_400: + return None + normalized.append((remaining, reset)) + return tuple(normalized) + + def _normalize_navyai(payload: Mapping[str, Any]) -> dict[str, Any]: limits = _mapping(payload.get("limits")) usage = _mapping(payload.get("usage")) diff --git a/services/provider_usage_service.py b/services/provider_usage_service.py index 6b697a32..41c2e790 100644 --- a/services/provider_usage_service.py +++ b/services/provider_usage_service.py @@ -241,6 +241,7 @@ def _fetch( if 200 <= last_status < 300: payload = self._response_json(response) normalized = NORMALIZERS[probe.provider](payload) + self._observe_reset_usage(probe, credential, normalized) return { "status": "available", **normalized, @@ -280,6 +281,17 @@ def _fetch( ) return self._error_result(probe, error_code, last_status) + def _observe_reset_usage( + self, probe: ProviderUsageProbe, credential: str, normalized: Mapping[str, Any], + ) -> None: + # Subscription-account data is not an account-pool scheduling source. + if probe.provider == "nanogpt": + return + from services.pool_reset_schedule import record_pool_usage, settings + if settings().enabled: + observed = datetime.fromisoformat(self._utcnow().replace("Z", "+00:00")).timestamp() + record_pool_usage(probe.provider, credential, normalized.get("windows"), observed_at=observed) + def _response_json(self, response: Any) -> Mapping[str, Any]: content = bytes(response.content or b"") if not content or len(content) > MAX_USAGE_RESPONSE_BYTES: diff --git a/tests/test_pool_reset_scheduling.py b/tests/test_pool_reset_scheduling.py new file mode 100644 index 00000000..ca4cd7d6 --- /dev/null +++ b/tests/test_pool_reset_scheduling.py @@ -0,0 +1,471 @@ +"""Reset scheduling with synthetic identities, clocks, and upstream responses.""" + +import importlib +import json +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +from unittest.mock import Mock +from unittest.mock import patch + +import pytest +from flask import Flask, g + +PROVIDER = "ce-gpt-pro" +KEYS = ["synthetic-first", "synthetic-second", "synthetic-third"] + + +@pytest.fixture(autouse=True) +def isolated(monkeypatch): + global schedule, credentials, nano, cooldown + schedule = importlib.import_module("services.pool_reset_schedule") + credentials = importlib.import_module("services.credential_pool") + nano = importlib.import_module("services.nanogpt_key_pool") + cooldown = importlib.import_module("services.model_cooldown") + monkeypatch.setenv("POOL_RESET_SCHEDULING_ENABLED", "true") + monkeypatch.setenv("POOL_OBSERVATION_TTL_SECONDS", "60") + monkeypatch.setenv("MODEL_COOLDOWN_ENABLED", "false") + monkeypatch.setenv("MODEL_COOLDOWN_MAX_SECONDS", "3600") + monkeypatch.setenv("PROMPT_CACHE_AFFINITY_ENABLED", "false") + credentials.CredentialPool.reset() + nano.NanoGPTKeyPool.reset() + nano.NanoGPTUnifiedKeyPool.reset() + cooldown.model_cooldown.reset() + clock = [1000.0] + store = schedule.PoolResetSchedule(clock=lambda: clock[0], monotonic=lambda: clock[0]) + monkeypatch.setattr(schedule, "pool_reset_schedule", store) + monkeypatch.setattr(credentials.CredentialPool, "keys", staticmethod(lambda provider: KEYS)) + yield clock, store + credentials.CredentialPool.reset() + nano.NanoGPTKeyPool.reset() + nano.NanoGPTUnifiedKeyPool.reset() + cooldown.model_cooldown.reset() + + +def observe(store, key, delay, *, provider=PROVIDER, remaining=10, **kwargs): + store.observe(provider, store.credential_id(provider, key), [ + {"remaining": remaining, "resets_in_ms": delay * 1000} + ], observed_at=1000, **kwargs) + + +def test_fresh_reset_order_stable_ties_and_unknown_fallback(isolated): + clock, store = isolated + observe(store, KEYS[0], 100) + observe(store, KEYS[1], 20) + observe(store, KEYS[2], 20) + assert credentials.CredentialPool.available(PROVIDER, KEYS, now=1000) == [KEYS[1], KEYS[2], KEYS[0]] + assert credentials.CredentialPool.select(PROVIDER, now=1000) == KEYS[1] + assert store.available(PROVIDER, [KEYS[2], KEYS[1]], configured_order=KEYS) == KEYS[1:] + clock[0] = 1060 + assert credentials.CredentialPool.select(PROVIDER, now=1060) == KEYS[0] + assert store.entry_count == 0 + observe(store, KEYS[1], 200) + assert store.available(PROVIDER, KEYS) == KEYS + + +def test_exhaustion_and_cooling_excluded_despite_near_reset(isolated): + _, store = isolated + observe(store, KEYS[0], 1, remaining=0) + observe(store, KEYS[1], 2) + observe(store, KEYS[2], 100) + credentials.CredentialPool.record(PROVIDER, KEYS[1], 429, now=1000) + assert credentials.CredentialPool.available(PROVIDER, KEYS, now=1000) == KEYS[2:] + assert credentials.CredentialPool.select(PROVIDER, now=1000) == KEYS[2] + + +def test_account_model_bucket_and_pool_isolation(isolated): + _, store = isolated + observe(store, KEYS[0], 20, remaining=0, model="A") + assert store.available(PROVIDER, KEYS, model="A") == KEYS[1:] + assert store.available(PROVIDER, KEYS, model="B") == KEYS + assert store.available("ce-gpt-plus", KEYS, model="A") == KEYS + observe(store, KEYS[1], 30, remaining=0, quota_bucket="shared") + assert store.available(PROVIDER, KEYS, model="B", quota_bucket="shared") == [KEYS[0], KEYS[2]] + observe(store, KEYS[2], 40, remaining=0) + assert store.available(PROVIDER, KEYS, model="B") == KEYS[:2] + + +@pytest.mark.parametrize("window", [ + {"limit": 10, "used": 9, "remaining": 8, "resets_in_ms": 1000}, + {"remaining": -1, "resets_in_ms": 1000}, + {"remaining": True, "resets_in_ms": 1000}, + {"remaining": float("nan"), "resets_in_ms": 1000}, + {"limit": 10, "used": 10, "remaining": 0, "percent_used": 20, "resets_in_ms": 1000}, + {"remaining": 10, "resets_at": "bad-date"}, + {"remaining": 10, "resets_at": "1970-01-01T00:00:01Z"}, + {"remaining": 10, "resets_in_ms": 1000, "resets_at": "1970-01-01T00:30:00Z"}, +]) +def test_contradictory_or_missing_data_clears_old_evidence(isolated, window): + _, store = isolated + observe(store, KEYS[0], 10, remaining=0) + store.observe(PROVIDER, store.credential_id(PROVIDER, KEYS[0]), [window], observed_at=1000) + assert store.available(PROVIDER, KEYS) == KEYS + + +def test_derived_quota_multiple_windows_and_reset_rollover(isolated): + clock, store = isolated + store.observe(PROVIDER, store.credential_id(PROVIDER, KEYS[0]), [ + {"limit": 10, "used": 5, "resets_in_ms": 10000}, + {"remaining": 0, "resets_in_ms": 20000}, + ], observed_at=1000) + assert store.available(PROVIDER, KEYS) == KEYS[1:] + clock[0] = 1021 + assert store.available(PROVIDER, KEYS) == KEYS + + +def test_clock_skew_old_observations_and_retry_bounds(isolated): + clock, store = isolated + for observed_at in (900, 1010, float("inf")): + store.observe(PROVIDER, store.credential_id(PROVIDER, KEYS[0]), [ + {"remaining": 0, "resets_in_ms": 10000} + ], observed_at=observed_at) + assert store.available(PROVIDER, KEYS) == KEYS + observe(store, KEYS[0], 999999, remaining=0) + assert store.retry_after(PROVIDER, KEYS, fallback=5) == 3600 + clock[0] = 999 + assert store.available(PROVIDER, KEYS) == KEYS + assert store.retry_after(PROVIDER, KEYS, fallback=-1) == 1 + + +@pytest.mark.parametrize("flag", [None, "", "false", "0", "malformed-secret-value"]) +@pytest.mark.parametrize("model_cooldown", ["false", "true"]) +def test_disabled_sequences_cooldowns_and_errors_unchanged(isolated, monkeypatch, flag, model_cooldown): + monkeypatch.setenv("MODEL_COOLDOWN_ENABLED", model_cooldown) + + def sequence(value): + if value is None: + monkeypatch.delenv("POOL_RESET_SCHEDULING_ENABLED", raising=False) + else: + monkeypatch.setenv("POOL_RESET_SCHEDULING_ENABLED", value) + credentials.CredentialPool.reset() + nano.NanoGPTKeyPool.reset() + cooldown.model_cooldown.reset() + selected = [] + for key in KEYS: + selected.append(credentials.CredentialPool.select(PROVIDER, now=1000)) + credentials.CredentialPool.record(PROVIDER, key, 429, now=1000) + selected.append(credentials.CredentialPool.select(PROVIDER, now=1001)) + selected.append(credentials.CredentialPool.select(PROVIDER, now=1060)) + probe = Mock(return_value=200) + for key in KEYS: + selected.append(nano.NanoGPTKeyPool.select_key(KEYS, probe, now=1000)) + nano.NanoGPTKeyPool.record_result(key, 429, now=1000) + with pytest.raises(nano.NanoGPTKeyPoolExhausted) as error: + nano.NanoGPTKeyPool.select_key(KEYS, probe, now=1001) + return selected, dict(credentials.CredentialPool._resting), dict(nano.NanoGPTKeyPool._rejected_until), str(error.value), probe.call_args_list + + expected = sequence("false") + assert sequence(flag) == expected + assert isolated[1].entry_count == 0 + + +@pytest.mark.parametrize("value", ["bad-sensitive-value", "4", "3601", "1.5", "-1", "NaN"]) +def test_invalid_ttl_disables_once_without_value(monkeypatch, caplog, value): + schedule._warned_settings.clear() + monkeypatch.setenv("POOL_OBSERVATION_TTL_SECONDS", value) + assert not schedule.settings().enabled + assert not schedule.settings().enabled + records = [r.message for r in caplog.records if "POOL_OBSERVATION_TTL_SECONDS" in r.message] + assert len(records) == 1 + assert value not in records[0] + + +def test_empty_defaults_and_malformed_flag(monkeypatch, caplog): + monkeypatch.setenv("POOL_OBSERVATION_TTL_SECONDS", "") + assert schedule.settings().ttl_seconds == 60 + for value in ("5", "3600"): + monkeypatch.setenv("POOL_OBSERVATION_TTL_SECONDS", value) + assert schedule.settings().enabled + schedule._warned_settings.clear() + monkeypatch.setenv("POOL_RESET_SCHEDULING_ENABLED", "flag-sensitive-value") + assert not schedule.settings().enabled + assert not schedule.settings().enabled + assert len(caplog.records) == 1 + assert "flag-sensitive-value" not in caplog.text + + +@pytest.mark.parametrize("scoped", [False, True]) +def test_real_429_keeps_body_refines_advice_and_never_dispatches(isolated, monkeypatch, scoped): + monkeypatch.setenv("MODEL_COOLDOWN_ENABLED", "true" if scoped else "false") + _, store = isolated + for key, delay in zip(KEYS, [30, 10, 20]): + observe(store, key, delay, remaining=0, model="A") + credentials.CredentialPool.record(PROVIDER, key, 429, model="A", now=1000) + app = Flask(__name__) + app.register_error_handler(cooldown.ModelCooldownExhausted, cooldown.cooldown_error_response) + send = Mock() + + @app.before_request + def request_id(): + g.request_id = "synthetic-request-id" + + @app.post("/v1/test-reset") + def dispatch(): + credentials.CredentialPool.select(PROVIDER, model="A", now=1000) + send() + return {"ok": True} + + response = app.test_client().post("/v1/test-reset", json={"model": "A"}) + with app.test_request_context(): + g.request_id = "synthetic-request-id" + expected = cooldown.ModelCooldownExhausted(60).to_dict() + assert response.status_code == 429 + assert response.json == expected + assert response.headers["Retry-After"] == "10" + send.assert_not_called() + + +@pytest.mark.parametrize("pool_name", ["NanoGPTKeyPool", "NanoGPTUnifiedKeyPool"]) +@pytest.mark.parametrize("scoped", [False, True]) +@pytest.mark.parametrize("single", [False, True]) +def test_nanogpt_without_observations_preserves_validation_sequence(isolated, monkeypatch, pool_name, scoped, single): + monkeypatch.setenv("MODEL_COOLDOWN_ENABLED", "true" if scoped else "false") + pool = getattr(nano, pool_name) + keys = KEYS[:1] if single else KEYS + + def sequence(flag): + monkeypatch.setenv("POOL_RESET_SCHEDULING_ENABLED", flag) + pool.reset() + cooldown.model_cooldown.reset() + probe = Mock(side_effect=[503, 200, 200] if single else [401, 200, 503, 200, 401, 200]) + selected = [] + for now in (1000, 1001, 1002, 1003, 1014, 1015): + key = pool.select_key(keys, probe, now=now, model="A" if scoped else None, + check_ttl_seconds=10, check_every_requests=2) + selected.append(key) + pool.record_result(key, 200, now=now, model="A" if scoped else None) + return selected, probe.call_args_list, pool._active_until, pool._active_requests, dict(pool._rejected_until) + + expected = sequence("false") + assert expected[0] == ([KEYS[0]] * 6 if single else [KEYS[1], KEYS[1], KEYS[2], KEYS[2], KEYS[0], KEYS[0]]) + assert len(expected[1]) == (2 if single else 6) + assert sequence("true") == expected + + +@pytest.mark.parametrize("pool_name", ["NanoGPTKeyPool", "NanoGPTUnifiedKeyPool"]) +@pytest.mark.parametrize("scoped", [False, True]) +def test_nanogpt_exhausted_active_key_validates_replacement(isolated, monkeypatch, pool_name, scoped): + monkeypatch.setenv("MODEL_COOLDOWN_ENABLED", "true" if scoped else "false") + pool = getattr(nano, pool_name) + probe = Mock(return_value=200) + assert pool.select_key(KEYS[:2], probe, now=1000, model="A") == KEYS[0] + observe(isolated[1], KEYS[0], 10, provider=pool._model_cooldown_provider, remaining=0, model="A") + assert pool.select_key(KEYS[:2], probe, now=1001, model="A") == KEYS[1] + assert [call.args[0] for call in probe.call_args_list] == KEYS[:2] + + +@pytest.mark.parametrize("pool_name", ["NanoGPTKeyPool", "NanoGPTUnifiedKeyPool"]) +def test_nanogpt_reset_order_changes_first_probe_but_keeps_validated_active(isolated, pool_name): + pool = getattr(nano, pool_name) + provider = pool._model_cooldown_provider + observe(isolated[1], KEYS[0], 100, provider=provider) + observe(isolated[1], KEYS[1], 10, provider=provider) + probe = Mock(return_value=200) + assert pool.select_key(KEYS, probe, now=1000, check_every_requests=1) == KEYS[1] + observe(isolated[1], KEYS[0], 5, provider=provider) + assert pool.select_key(KEYS, probe, now=1001, check_every_requests=1) == KEYS[1] + probe.assert_called_once_with(KEYS[1]) + pool.record_result(KEYS[1], 200, now=1001) + assert pool.select_key(KEYS, probe, now=1002, check_every_requests=1) == KEYS[1] + assert [call.args[0] for call in probe.call_args_list] == [KEYS[1], KEYS[1]] + + +@pytest.mark.parametrize("pool_name", ["NanoGPTKeyPool", "NanoGPTUnifiedKeyPool"]) +@pytest.mark.parametrize("scoped", [False, True]) +@pytest.mark.parametrize("cooling", [False, True]) +def test_nanogpt_exhausted_or_cooling_pool_raises_without_probe(isolated, monkeypatch, pool_name, scoped, cooling): + monkeypatch.setenv("MODEL_COOLDOWN_ENABLED", "true" if scoped else "false") + pool = getattr(nano, pool_name) + for key, delay in zip(KEYS, [30, 10, 20]): + observe(isolated[1], key, delay, provider=pool._model_cooldown_provider, + remaining=10 if cooling else 0, model="A") + if cooling: + pool.record_result(key, 429, now=1000, model="A") + probe = Mock(side_effect=AssertionError("Exhausted selection must not probe")) + with pytest.raises(cooldown.ModelCooldownExhausted) as error: + pool.select_key(KEYS, probe, now=1000, model="A") + assert error.value.retry_after == 10 + probe.assert_not_called() + + +@pytest.mark.parametrize("pool_name", ["NanoGPTKeyPool", "NanoGPTUnifiedKeyPool"]) +def test_nanogpt_available_selection_keeps_no_probe_behavior(isolated, pool_name): + pool = getattr(nano, pool_name) + provider = pool._model_cooldown_provider + observe(isolated[1], KEYS[0], 100, provider=provider) + observe(isolated[1], KEYS[1], 10, provider=provider) + assert pool.select_available_key(KEYS, now=1000) == KEYS[1] + for key in KEYS: + pool.record_result(key, 429, retry_after_seconds=11, now=1000) + with pytest.raises(cooldown.ModelCooldownExhausted) as error: + pool.select_available_key(KEYS, now=1000) + assert error.value.retry_after == 11 + + +def test_record_result_metadata_and_cancelled_outcome(isolated): + _, store = isolated + credentials.CredentialPool.record(PROVIDER, KEYS[0], 429, retry_after_seconds=15, model="A", now=1000) + assert store.retry_after(PROVIDER, KEYS, model="A") == 15 + cancelled = importlib.import_module("services.upstream_outcome").classify_upstream_outcome(429, cancelled=True) + credentials.CredentialPool.record(PROVIDER, KEYS[1], 429, retry_after_seconds=1, outcome=cancelled, now=1000) + assert store.entry_count == 1 + nano.NanoGPTKeyPool.record_result(KEYS[1], 200, usage_windows=[ + {"remaining": 10, "resets_in_ms": 5000} + ], now=1000) + assert store.entry_count == 2 + + +def test_affinity_is_limited_to_healthy_candidates(isolated, monkeypatch): + affinity = importlib.import_module("services.prompt_cache_affinity") + seen = [] + monkeypatch.setattr(affinity, "prefer_credential", lambda provider, model, eligible: seen.append(eligible) or (KEYS[0] if KEYS[0] in eligible else None)) + observe(isolated[1], KEYS[0], 1, remaining=0) + assert credentials.CredentialPool.select(PROVIDER, model="A", now=1000) == KEYS[1] + assert seen == [KEYS[1:]] + + +def test_bounded_opaque_storage_ttl_oldest_and_concurrency(isolated): + clock, _ = isolated + store = schedule.PoolResetSchedule(clock=lambda: clock[0], monotonic=lambda: clock[0], max_entries=2) + for key in KEYS: + observe(store, key, 100) + assert store.entry_count == 2 + assert store.available(PROVIDER, KEYS[:1]) == KEYS[:1] + assert all(key not in repr(store.__dict__) for key in KEYS) + with ThreadPoolExecutor(max_workers=4) as executor: + list(executor.map(lambda _: observe(store, KEYS[1], 100), range(100))) + assert store.entry_count == 2 + clock[0] = 1061 + assert store.entry_count == 0 + assert schedule.PoolResetSchedule(max_entries=20000)._max_entries == 10000 + + +@pytest.mark.parametrize("enabled", ["false", "true"]) +def test_usage_snapshot_cache_never_refreshes_observation_or_adds_calls(isolated, monkeypatch, enabled): + monkeypatch.setenv("POOL_RESET_SCHEDULING_ENABLED", enabled) + usage = importlib.import_module("services.provider_usage_service") + clock, store = isolated + auth = Mock() + auth.get_api_key.return_value = KEYS[0] + auth.get_api_keys.return_value = KEYS + auth.provider_credential_env_names.return_value = () + metrics = Mock() + metrics.get_cost_summary.return_value = {} + metrics.get_provider_stats.return_value = {} + response = Mock(status_code=200, content=json.dumps({"usage": { + "tokens_remaining_today": 10, "resets_in_ms": 100000 + }}).encode()) + proxy = Mock() + proxy.make_request.return_value = response + service = usage.ProviderUsageService(config={"API_BASE_URLS": {"navyai": "https://invalid.example"}, "PROVIDER_USAGE_CACHE_TTL_SECONDS": 300}, auth_service=auth, metrics_service=metrics, proxy_service=proxy, monotonic=lambda: clock[0]) + monkeypatch.setattr(service, "_utcnow", lambda: datetime.fromtimestamp(clock[0], timezone.utc).isoformat()) + first = service.snapshot(["navyai"]) + assert store.entry_count == (1 if enabled == "true" else 0) + clock[0] = 1061 + second = service.snapshot(["navyai"]) + assert second["providers"][0]["source"]["cached"] + assert first["providers"][0]["windows"] == second["providers"][0]["windows"] + assert store.entry_count == 0 + assert proxy.make_request.call_count == 1 + response.close.assert_called_once() + + +def test_subscription_usage_is_not_a_scheduler_source(isolated, monkeypatch): + usage = importlib.import_module("services.provider_usage_service") + response = Mock(status_code=200, content=json.dumps({"daily": {"used": 0, "remaining": 10, "resetAt": 1010000}}).encode()) + proxy = Mock() + proxy.make_request.return_value = response + service = usage.ProviderUsageService(config={"NANOGPT_SUBSCRIPTION_BASE_URL": "https://invalid.example"}, auth_service=Mock(), metrics_service=Mock(), proxy_service=proxy) + service._fetch(usage.PROVIDER_USAGE_PROBES["nanogpt"], KEYS) + assert isolated[1].entry_count == 0 + proxy.make_request.assert_called_once() + + +def test_disabled_and_enabled_usage_snapshots_match_bytes(isolated, monkeypatch): + usage = importlib.import_module("services.provider_usage_service") + auth = Mock() + auth.get_api_key.return_value = KEYS[0] + auth.provider_credential_env_names.return_value = () + metrics = Mock() + metrics.get_cost_summary.return_value = {} + metrics.get_provider_stats.return_value = {} + proxy = Mock() + proxy.make_request.return_value = Mock(status_code=200, content=b'{"usage":{"tokens_remaining_today":10,"resets_in_ms":10000}}') + snapshots = [] + for flag in ("false", "true"): + monkeypatch.setenv("POOL_RESET_SCHEDULING_ENABLED", flag) + service = usage.ProviderUsageService(config={"API_BASE_URLS": {"navyai": "https://invalid.example"}}, auth_service=auth, metrics_service=metrics, proxy_service=proxy) + monkeypatch.setattr(service, "_utcnow", lambda: "1970-01-01T00:16:40Z") + snapshots.append(json.dumps(service.snapshot(["navyai"]), sort_keys=True)) + assert snapshots[0] == snapshots[1] + assert proxy.make_request.call_count == 2 + + +def test_observation_validation_and_delayed_updates(isolated): + _, store = isolated + identity = store.credential_id(PROVIDER, KEYS[0]) + observe(store, KEYS[0], 50, remaining=0) + store.observe(PROVIDER, identity, [{"remaining": 10, "resets_in_ms": 1000}], observed_at=999) + assert store.available(PROVIDER, KEYS) == KEYS[1:] + for invalid in (None, 42, "credential-prefix"): + store.observe(PROVIDER, invalid, [{"remaining": 0, "resets_in_ms": 1000}]) + store.observe(PROVIDER, identity, [{"remaining": 0, "resets_in_ms": 1000}], model="x" * 257) + assert store.entry_count == 1 + store.observe(PROVIDER, identity, [{"remaining": 10**1000, "resets_in_ms": 1000}]) + assert store.available(PROVIDER, KEYS) == KEYS + + +def test_monotonic_expiry_and_tolerated_future_skew(): + mono = [0.0] + store = schedule.PoolResetSchedule(clock=lambda: 1000, monotonic=lambda: mono[0]) + store.observe(PROVIDER, store.credential_id(PROVIDER, KEYS[0]), [ + {"remaining": 0, "resets_in_ms": 100000} + ], observed_at=1005) + assert store.available(PROVIDER, KEYS) == KEYS[1:] + mono[0] = 60 + assert store.available(PROVIDER, KEYS) == KEYS + assert store.entry_count == 0 + + +@pytest.mark.parametrize("enabled", ["false", "true"]) +def test_registered_raw_pool_path_preserves_wire_and_stops_before_post(isolated, monkeypatch, tmp_path, enabled): + monkeypatch.setenv("POOL_RESET_SCHEDULING_ENABLED", enabled) + monkeypatch.setenv("ADMIN_API_KEY", "admin-test-key") + monkeypatch.setenv("FLASK_SECRET_KEY", "flask-test-secret") + monkeypatch.setenv("JWT_SECRET", "jwt-test-secret") + for name in ("AUTH_DB_PATH", "RATE_LIMIT_DB_PATH", "MODEL_REGISTRY_DB_PATH"): + monkeypatch.setenv(name, str(tmp_path / (name + ".sqlite3"))) + with patch("env_loader.load_runtime_env"), patch("config.load_runtime_env"): + app_module = importlib.import_module("app") + with patch.object(app_module, "load_runtime_env"), patch("config.load_runtime_env"): + app = app_module.create_app() + app.config.update(WTF_CSRF_ENABLED=False, IMAGE_RELAY_CATALOG_AUTO_REFRESH=False, + PROVIDER_CATALOG_AUTO_REFRESH=False, PROMPT_CACHE_ENABLED=False) + monkeypatch.setattr(app_module.AuthService, "verify_api_key", lambda *args: {"username": "alice", "is_admin": True}) + store = isolated[1] + observe(store, KEYS[0], 100) + observe(store, KEYS[1], 10) + upstream = Mock(status_code=200, raw=None, content=b'{"exact":"provider-response"}', headers={"content-type": "application/json"}) + send = Mock(side_effect=lambda **kwargs: Mock(status_code=200, raw=None, content=upstream.content, headers=upstream.headers)) + monkeypatch.setattr(app_module.ProxyService, "make_request", send) + client = app.test_client() + headers = {"Authorization": "Bearer admin-test-key", "Content-Type": "application/json"} + wire = b'{ "model":"gpt-6.1-sol", "messages":[] }' + response = client.post("/ce-gpt-pro/v1/chat/completions", data=wire, headers=headers) + assert response.status_code == 200 + assert response.data == upstream.content + assert send.call_args.kwargs["data"] == wire + assert send.call_args.kwargs["headers"]["Authorization"] == "Bearer " + (KEYS[1] if enabled == "true" else KEYS[0]) + assert send.call_count == 1 + for key in KEYS: + observe(store, key, 10, remaining=0) + send.reset_mock() + response = client.post("/ce-gpt-pro/v1/chat/completions", data=wire, headers=headers) + if enabled == "true": + assert response.status_code == 429 + assert response.headers["Retry-After"] == "10" + assert response.json["error"] == "model_cooldown" + send.assert_not_called() + else: + assert response.status_code == 200 + assert response.data == upstream.content + assert send.call_count == 1 diff --git a/worker/container-env.mjs b/worker/container-env.mjs index 1a7994db..46fd0227 100644 --- a/worker/container-env.mjs +++ b/worker/container-env.mjs @@ -40,6 +40,8 @@ const DIRECT_ENV_KEYS = [ "CONNECTION_PROFILES_DB_PATH", "CONFIG_REVISION_SYNC_ENABLED", "CONFIG_SYNC_TTL_SECONDS", + "POOL_RESET_SCHEDULING_ENABLED", + "POOL_OBSERVATION_TTL_SECONDS", "GATEWAY_ALERTS_ENABLED", "GATEWAY_ALERT_WEBHOOK_ALLOWLIST", "CONFIG_SECURITY_TTL_SECONDS",