diff --git a/docs/retry-advice.md b/docs/retry-advice.md new file mode 100644 index 00000000..d9ee1d51 --- /dev/null +++ b/docs/retry-advice.md @@ -0,0 +1,67 @@ +# Bounded upstream Retry-After advice + +`UPSTREAM_RETRY_AFTER_ADVICE_ENABLED` defaults to false. Unset or empty settings +use their defaults. Disabled traffic keeps its existing retries, waits, headers, +response bodies and free-pool cooldown timing. Native Worker and raw Flask paths +continue to forward upstream bytes without parsing or scheduling retries. + +Enable local advice with: + +```text +UPSTREAM_RETRY_AFTER_ADVICE_ENABLED=true +UPSTREAM_RETRY_AFTER_MAX_SECONDS=3600 +``` + +The maximum defaults to 3600 seconds and accepts whole seconds from 1 to 604800, +matching the existing free-pool cooldown ceiling. An invalid flag or enabled +maximum disables advice and logs the setting name once per process, without its +value. The maximum is ignored while advice is disabled. + +`services.retry_advice.parse_retry_advice` returns immutable local advice with +`source`, `observed_at`, `retry_at`, `delay_seconds` and `scope`. Times are Unix +seconds. It recognizes these formats, using case-insensitive header names: + +- `Retry-After`: nonnegative decimal seconds or an HTTP-date with a timezone. +- `X-RateLimit-Reset`: an absolute Unix timestamp in seconds. +- `X-RateLimit-Reset-Requests` and `X-RateLimit-Reset-Tokens`: duration strings, + such as `1h2m3.4s`, when the matching remaining counter is finite and exhausted. +- `Anthropic-RateLimit-Requests-Reset` and `Anthropic-RateLimit-Tokens-Reset`: + ISO timestamps with a timezone, when the matching remaining counter is exhausted. + +Malformed, nonfinite, zero-delay and stale hints are ignored. Individual values +are limited to 512 characters. When several active hints are valid, the latest +reset wins so an exhausted dimension is not bypassed. Only local advice is +clamped; the upstream header is never replaced. With observation time 100, +`Retry-After: 20` yields retry time 120. With maximum 5 it yields local retry time +105, while the header remains `20`. + +The [upstream outcome](upstream-outcomes.md) classifier keeps 429 throttling neutral for provider health. +Exhausted remaining counters label advice `quota_exhausted`; other 429 advice +uses `rate_limit`; provider failures use `provider`. Authentication and caller +errors do not supply retry advice. Successful exhausted responses can supply +cooldown advice without changing their health classification. + +Managed retries still require the existing method/idempotency/transport policy +and attempt limit. Parsing never grants replay permission. An ambiguous POST or +stream does not gain a retry from headers. While enabled, legacy timeout-body +retries additionally require a safe method or an idempotency key. Connect-timeout +retries remain allowed by the existing policy because nothing reached upstream. +Existing managed response normalization remains in the proxy service. + +For an allowed retry, the scheduler waits for the greater of existing backoff +and bounded advice. Invalid or absent advice uses existing backoff. An optional +monotonic `deadline` on `_make_base_request`, or the existing Flask +`g.cascade_deadline`, must cover the entire wait. A refused wait returns the +original upstream result. The scheduler rechecks the deadline after sleeping and +does not dispatch a late retry. Without a shared deadline, existing per-attempt +timeouts remain unchanged; the feature does not create a request-wide deadline +or reserve time for another attempt. Free-pool `retry_seconds` uses the same +bounded parser while enabled, including a bounded fallback cooldown. + +There are no new routes, response headers, storage tables or dependencies. Advice +reads headers only and retains no credentials, prompts or response content. +Existing cooldown storage and its monotonic extension rules remain in use; +enabling a lower maximum does not shorten a previously stored cooldown. Advice +is neither proof of provider capacity nor authorization for additional cost. +Retries can consume additional quota only when already authorized by transport +policy. The Worker forwards both settings to the Container. diff --git a/services/free_quota_service.py b/services/free_quota_service.py index 7ce2a99b..2cca1a22 100644 --- a/services/free_quota_service.py +++ b/services/free_quota_service.py @@ -11,6 +11,7 @@ from email.utils import parsedate_to_datetime from services import free_quota_d1 +from services.retry_advice import parse_retry_advice, retry_advice_settings from services.sqlite_store import connect, storage_path MAX_COOLDOWN = 7 * 24 * 3600 @@ -30,6 +31,11 @@ def reset_seconds(value: str) -> float | None: def retry_seconds(headers, now: float, default: int = 60) -> int: + settings = retry_advice_settings() + if settings.enabled: + advice = parse_retry_advice(headers, now=now, max_seconds=settings.max_seconds) + delay = advice.delay_seconds if advice is not None else default + return min(settings.max_seconds, max(1, math.ceil(delay))) normalized = {k.lower(): v for k, v in headers.items()} value = normalized.get("retry-after", "") delay = None diff --git a/services/managed_dispatch.py b/services/managed_dispatch.py new file mode 100644 index 00000000..99911a54 --- /dev/null +++ b/services/managed_dispatch.py @@ -0,0 +1,65 @@ +"""Managed attempt scheduling, separate from raw dispatch and normalization.""" + +import math +import time +from typing import Callable + +import requests +from flask import g, has_request_context + +from services.retry_advice import parse_retry_advice, retry_advice_settings +from services.upstream_outcome import UpstreamOutcome +from services.upstream_transport import close_retry_response + +Attempt = Callable[[int], requests.Response] +Outcome = Callable[[], UpstreamOutcome] + + +def retry_managed_attempt( + attempt: Attempt, + outcome: Outcome, + *, + retry_count: int, + max_retries: int, + retry_delay: float, + response: requests.Response | None = None, + deadline: float | None = None, + sleep: Callable[[float], None] = time.sleep, +) -> requests.Response | None: + """Dispatch again only under the existing transport's replay permission. + + A supplied deadline is monotonic. The existing cascade deadline is used when + available; absent one, per-attempt timeouts remain the transport's policy. + None means that the original response/error must finish normally. + """ + classified = outcome() + if not classified.replay_permission or retry_count >= max_retries: + return None + settings = retry_advice_settings() + delay = retry_delay * (retry_count + 1) + if settings.enabled: + if response is not None: + advice = parse_retry_advice( + response.headers, + now=time.time(), + status_code=response.status_code, + max_seconds=settings.max_seconds, + ) + if advice is not None: + delay = max(delay, advice.delay_seconds) + if deadline is None and has_request_context(): + deadline = getattr(g, "cascade_deadline", None) + if deadline is not None: + if not math.isfinite(deadline) or delay >= deadline - time.monotonic(): + return None + # Keep the original response usable if scheduling consumes the budget. + sleep(delay) + if time.monotonic() >= deadline: + return None + if response is not None: + close_retry_response(response) + return attempt(retry_count + 1) + if response is not None: + close_retry_response(response) + sleep(delay) + return attempt(retry_count + 1) diff --git a/services/proxy_service.py b/services/proxy_service.py index b90fab76..e78c1861 100644 --- a/services/proxy_service.py +++ b/services/proxy_service.py @@ -1,6 +1,7 @@ import json import logging import requests +from functools import partial from typing import Optional, Dict, Any, Tuple, List, Generator from concurrent.futures import ThreadPoolExecutor from error_handlers import APIError @@ -14,6 +15,8 @@ from services.credential_pool import CredentialPool from services.resilience_service import ResilienceService from services.upstream_outcome import UpstreamOutcome, classify_upstream_outcome +from services.managed_dispatch import retry_managed_attempt +from services.retry_advice import retry_advice_settings from services.transport_policy import RAW_PASSTHROUGH_PROVIDERS from services.secret_firewall import protect_body from providers.cline_pass import cline_completion_payload @@ -1250,6 +1253,7 @@ def _make_base_request( timeout_override: Optional[Tuple[int, int]] = None, force_raw_passthrough: bool = False, is_streaming: Optional[bool] = None, + deadline: Optional[float] = None, ) -> requests.Response: """ Make a base request with retries and error handling @@ -1262,6 +1266,16 @@ def _make_base_request( force_raw_passthrough or api_provider in RAW_PASSTHROUGH_PROVIDERS ) + next_request = partial( + cls._make_base_request, method=method, url=url, headers=headers, + params=params, data=data, api_provider=api_provider, use_cache=use_cache, + timeout_override=timeout_override, force_raw_passthrough=force_raw_passthrough, + deadline=deadline, + ) + + def next_attempt(count: int) -> requests.Response: + return next_request(retry_count=count, is_streaming=is_streaming) + try: if not raw_passthrough and retry_count == 0: circuit_response = cls._circuit_open_response(api_provider) @@ -1325,21 +1339,14 @@ def _make_base_request( retry_count + 1, MAX_RETRIES, ) - close_retry_response(response) - time.sleep(RETRY_DELAY * (retry_count + 1)) - return cls._make_base_request( - method=method, - url=url, - headers=headers, - params=params, - data=data, - api_provider=api_provider, - use_cache=use_cache, - retry_count=retry_count + 1, - timeout_override=timeout_override, - force_raw_passthrough=force_raw_passthrough, - is_streaming=is_streaming, + retried = retry_managed_attempt( + next_attempt, + lambda: classify_upstream_outcome(response.status_code, replay_permission=True), + retry_count=retry_count, max_retries=MAX_RETRIES, + retry_delay=RETRY_DELAY, response=response, deadline=deadline, sleep=time.sleep, ) + if retried is not None: + return retried if not is_streaming: try: @@ -1384,21 +1391,23 @@ def _make_base_request( retry_count + 1, MAX_RETRIES, ) - close_retry_response(response) - time.sleep(RETRY_DELAY * (retry_count + 1)) - return cls._make_base_request( - method=method, - url=url, - headers=headers, - params=params, - data=data, - api_provider=api_provider, - use_cache=use_cache, - retry_count=retry_count + 1, - timeout_override=timeout_override, - force_raw_passthrough=force_raw_passthrough, - is_streaming=is_streaming, + replay_allowed = ( + not retry_advice_settings().enabled + or (not is_streaming and ( + method.upper() in cls.SAFE_RETRY_METHODS + or cls._has_idempotency_key(headers) + )) + ) + retried = retry_managed_attempt( + next_attempt, + lambda: classify_upstream_outcome( + response.status_code, replay_permission=replay_allowed, + ), + retry_count=retry_count, max_retries=MAX_RETRIES, + retry_delay=RETRY_DELAY, response=response, deadline=deadline, sleep=time.sleep, ) + if retried is not None: + return retried if api_provider == "cline-pass" and response.status_code < 400: normalized_payload = cline_completion_payload(normalized_payload) @@ -1461,20 +1470,16 @@ def _make_base_request( and retry_count < MAX_RETRIES ): logger.info(f"Retrying request (attempt {retry_count + 1}/{MAX_RETRIES})") - time.sleep(RETRY_DELAY * (retry_count + 1)) - return cls._make_base_request( - method=method, - url=url, - headers=headers, - params=params, - data=data, - api_provider=api_provider, - use_cache=use_cache, - retry_count=retry_count + 1, - timeout_override=timeout_override, - force_raw_passthrough=force_raw_passthrough, - is_streaming=is_streaming, + retried = retry_managed_attempt( + next_attempt, + lambda error=e: classify_upstream_outcome( + transport_failure=cls._transport_failure_kind(error), replay_permission=True, + ), + retry_count=retry_count, max_retries=MAX_RETRIES, + retry_delay=RETRY_DELAY, deadline=deadline, sleep=time.sleep, ) + if retried is not None: + return retried if not raw_passthrough: cls._record_circuit_result( api_provider, 503, diff --git a/services/retry_advice.py b/services/retry_advice.py new file mode 100644 index 00000000..7d2419ca --- /dev/null +++ b/services/retry_advice.py @@ -0,0 +1,183 @@ +"""Content-free, bounded retry hints; these never authorize request replay.""" + +import logging +import math +import os +import re +import threading +from dataclasses import dataclass +from datetime import datetime +from email.utils import parsedate_to_datetime +from typing import Mapping + +from services.upstream_outcome import classify_upstream_outcome + +logger = logging.getLogger(__name__) +_DEFAULT_MAX = 3600 +_warned_settings: set[str] = set() +_warning_lock = threading.Lock() +_DURATION = re.compile( + r"(?:(\d+(?:\.\d+)?)d)?(?:(\d+(?:\.\d+)?)h)?(?:(\d+(?:\.\d+)?)m)?" + r"(?:(\d+(?:\.\d+)?)s)?(?:(\d+(?:\.\d+)?)ms)?" +) +_NUMBER = re.compile(r"\d+(?:\.\d+)?") + + +@dataclass(frozen=True) +class RetryAdviceSettings: + enabled: bool = False + max_seconds: int = _DEFAULT_MAX + + +@dataclass(frozen=True) +class RetryAdvice: + source: str + observed_at: float + retry_at: float + delay_seconds: float + scope: str + + +def _warn_once(name: str) -> None: + with _warning_lock: + if name in _warned_settings: + return + _warned_settings.add(name) + logger.warning("Invalid %s; upstream retry advice disabled", name) + + +def retry_advice_settings() -> RetryAdviceSettings: + flag_name = "UPSTREAM_RETRY_AFTER_ADVICE_ENABLED" + flag = os.environ.get(flag_name, "").strip().lower() + if flag in ("", "false", "0", "no", "off"): + return RetryAdviceSettings() + if flag not in ("true", "1", "yes", "on"): + _warn_once(flag_name) + return RetryAdviceSettings() + limit_name = "UPSTREAM_RETRY_AFTER_MAX_SECONDS" + value = os.environ.get(limit_name, "").strip() + try: + limit = int(value) if value else _DEFAULT_MAX + # Match the existing free-pool maximum and keep sleep values operational. + if not 1 <= limit <= 7 * 24 * 3600: + raise ValueError + except ValueError: + _warn_once(limit_name) + return RetryAdviceSettings() + return RetryAdviceSettings(True, limit) + + +def _value(headers: Mapping[str, str], name: str) -> str: + value = headers.get(name, "") + return value.strip() if isinstance(value, str) and len(value) <= 512 else "" + + +def _number(value: str) -> float | None: + if not _NUMBER.fullmatch(value): + return None + number = float(value) + return number if math.isfinite(number) else None + + +def _duration(value: str) -> float | None: + match = _DURATION.fullmatch(value) + if not match or not any(match.groups()): + return None + delay = sum( + float(amount or 0) * unit + for amount, unit in zip(match.groups(), (86400, 3600, 60, 1, 0.001)) + ) + return delay if math.isfinite(delay) else None + + +def _date_delay(value: str, now: float, *, iso: bool = False) -> float | None: + try: + date = ( + datetime.fromisoformat(value.replace("Z", "+00:00")) + if iso + else parsedate_to_datetime(value) + ) + if date.tzinfo is None: + return None + delay = date.timestamp() - now + return delay if math.isfinite(delay) else None + except (ValueError, TypeError, OverflowError, OSError): + return None + + +def _exhausted(headers: Mapping[str, str], name: str) -> bool: + try: + remaining = float(_value(headers, name)) + return math.isfinite(remaining) and remaining <= 0 + except ValueError: + return False + + +def _reset_hints( + headers: Mapping[str, str], now: float +) -> tuple[list[tuple[str, float]], bool]: + hints: list[tuple[str, float]] = [] + exhausted = False + for dimension in ("requests", "tokens"): + for prefix, iso in (("x-ratelimit", False), ("anthropic-ratelimit", True)): + if not _exhausted( + headers, + f"{prefix}-remaining-{dimension}" + if not iso + else f"{prefix}-{dimension}-remaining", + ): + continue + exhausted = True + name = ( + f"{prefix}-reset-{dimension}" + if not iso + else f"{prefix}-{dimension}-reset" + ) + value = _value(headers, name) + delay = _date_delay(value, now, iso=True) if iso else _duration(value) + if delay is not None and delay > 0: + hints.append((name, delay)) + return hints, exhausted + + +def parse_retry_advice( + headers: Mapping[str, str], + *, + now: float, + status_code: int = 429, + max_seconds: int = _DEFAULT_MAX, +) -> RetryAdvice | None: + """Normalize known header formats without reading bodies or mutating headers. + + Exhausted dimensions and Retry-After combine using the latest valid reset. + This avoids sending before either active limit permits another attempt. + """ + if not math.isfinite(now) or not 1 <= max_seconds <= 7 * 24 * 3600: + return None + outcome = classify_upstream_outcome(status_code) + if outcome.reason in ("http_credential_rejected", "http_caller_error"): + return None + normalized = {name.lower(): value for name, value in headers.items()} + hints, exhausted = _reset_hints(normalized, now) + if outcome.reason == "http_throttled": + scope = "quota_exhausted" if exhausted else "rate_limit" + elif outcome.provider_health == "failure": + scope = "provider" + elif outcome.provider_health == "success" and exhausted: + scope = "quota_exhausted" + else: + return None + value = _value(normalized, "retry-after") + delay = _number(value) + if delay is None: + delay = _date_delay(value, now) + if delay is not None and delay > 0: + hints.append(("retry-after", delay)) + epoch = _number(_value(normalized, "x-ratelimit-reset")) + if epoch is not None and epoch > now: + hints.append(("x-ratelimit-reset", epoch - now)) + if not hints: + return None + source, delay = max(hints, key=lambda hint: hint[1]) + delay = min(delay, max_seconds) + return RetryAdvice(source, now, now + delay, delay, scope) diff --git a/tests/test_retry_advice.py b/tests/test_retry_advice.py new file mode 100644 index 00000000..bb233204 --- /dev/null +++ b/tests/test_retry_advice.py @@ -0,0 +1,436 @@ +"""Retry hints change local scheduling, never transport permissions or bytes.""" + +from dataclasses import FrozenInstanceError +from email.utils import formatdate +from unittest.mock import Mock, patch + +import pytest +import requests + + +@pytest.fixture(autouse=True) +def settings(monkeypatch): + monkeypatch.delenv("UPSTREAM_RETRY_AFTER_ADVICE_ENABLED", raising=False) + monkeypatch.delenv("UPSTREAM_RETRY_AFTER_MAX_SECONDS", raising=False) + monkeypatch.setenv("SECRET_SCAN_DEFAULT", "off") + from services.resilience_service import ResilienceService + + ResilienceService.reset() + yield + ResilienceService.reset() + + +def parse(headers, now=100, status=429, maximum=3600): + from services.retry_advice import parse_retry_advice + + return parse_retry_advice(headers, now=now, status_code=status, max_seconds=maximum) + + +def test_delta_date_equivalence_and_immutable_advice(): + delta = parse({"Retry-After": "20"}) + date = parse({"retry-after": formatdate(120, usegmt=True)}) + assert ( + (delta.observed_at, delta.retry_at, delta.delay_seconds, delta.scope) + == (date.observed_at, date.retry_at, date.delay_seconds, date.scope) + == (100, 120, 20, "rate_limit") + ) + assert delta.source == "retry-after" + with pytest.raises(FrozenInstanceError): + delta.delay_seconds = 1 + + +@pytest.mark.parametrize( + "value", + [ + "", + "bad", + "nan", + "inf", + "-1", + "0", + "1e999", + "x" * 513, + formatdate(99, usegmt=True), + ], +) +def test_malformed_nonfinite_and_stale_advice(value): + assert parse({"Retry-After": value}) is None + + +def test_maximum_clamps_only_local_advice(): + headers = {"Retry-After": "999999", "X-Unrelated": "unchanged"} + before = headers.copy() + advice = parse(headers, maximum=10) + assert (advice.delay_seconds, advice.retry_at) == (10, 110) + assert headers == before + + +@pytest.mark.parametrize( + "value,delay", + [("120", 20), ("99", None), ("nan", None), ("-1", None), ("999999999", 3600)], +) +def test_epoch_reset(value, delay): + advice = parse({"X-RateLimit-Reset": value}) + assert (advice.delay_seconds if advice else None) == delay + + +def test_exhausted_dimensions_and_precise_reset_precedence(): + advice = parse( + { + "Retry-After": "2", + "X-RateLimit-Remaining-Requests": "0", + "X-RateLimit-Reset-Requests": "1h2m3.4s", + "X-RateLimit-Remaining-Tokens": "0", + "X-RateLimit-Reset-Tokens": "7.66s", + } + ) + assert (advice.source, advice.delay_seconds, advice.scope) == ( + "x-ratelimit-reset-requests", + 3600, + "quota_exhausted", + ) + advice = parse( + { + "anthropic-ratelimit-tokens-remaining": "0", + "anthropic-ratelimit-tokens-reset": "1970-01-01T00:02:00Z", + } + ) + assert advice.delay_seconds == 20 and advice.scope == "quota_exhausted" + assert ( + parse( + {"X-RateLimit-Remaining-Requests": "1", "X-RateLimit-Reset-Requests": "5s"} + ) + is None + ) + + +@pytest.mark.parametrize("status", [400, 401, 403, 404, 422, 200]) +def test_authentication_and_caller_errors_are_not_throttling(status): + assert parse({"Retry-After": "20"}, status=status) is None + + +@pytest.mark.parametrize( + "flag,maximum,enabled,limit", + [ + ("", "", False, 3600), + ("false", "bad", False, 3600), + ("true", "", True, 3600), + ("1", "10", True, 10), + ("bad", "10", False, 3600), + ("true", "nan", False, 3600), + ("true", "-1", False, 3600), + ("true", "0", False, 3600), + ], +) +def test_configuration_defaults_and_invalid_values( + monkeypatch, flag, maximum, enabled, limit +): + from services.retry_advice import retry_advice_settings + + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_ADVICE_ENABLED", flag) + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_MAX_SECONDS", maximum) + settings = retry_advice_settings() + assert (settings.enabled, settings.max_seconds) == (enabled, limit) + + +def test_invalid_settings_log_once_without_value(monkeypatch, caplog): + from services import retry_advice + + retry_advice._warned_settings.clear() + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_ADVICE_ENABLED", "private-invalid-setting") + for _ in range(2): + assert not retry_advice.retry_advice_settings().enabled + assert len(caplog.records) == 1 + assert "private-invalid-setting" not in caplog.text + + +def response(status=429, hint="20"): + result = requests.Response() + result.status_code = status + result._content = b" {raw bytes}\n" + result.headers.update({"Content-Type": "text/plain", "Retry-After": hint}) + return result + + +def dispatch( + upstreams, + *, + method="GET", + headers=None, + raw=False, + deadline=None, + streaming=False, + provider="openai", +): + from services.proxy_service import ProxyService + + session = Mock() + session.request.side_effect = upstreams + with ( + patch.object(ProxyService, "_get_provider_session", return_value=session), + patch("services.proxy_service.time.sleep") as sleep, + patch("services.managed_dispatch.time.time", return_value=100), + patch("services.managed_dispatch.time.monotonic", return_value=100), + ): + kwargs = {"deadline": deadline} if deadline is not None else {} + result = ProxyService._make_base_request( + method, + "https://provider.invalid/v1/models", + headers or {}, + {}, + b"{}" if method == "POST" else None, + provider, + force_raw_passthrough=raw, + is_streaming=streaming, + timeout_override=(1, 2), + **kwargs, + ) + return result, session, sleep + + +@pytest.mark.parametrize("flag", [None, "", "false", "bad"]) +def test_disabled_transport_retains_waits_and_headers(monkeypatch, flag): + if flag is not None: + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_ADVICE_ENABLED", flag) + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_MAX_SECONDS", "2") + first, last = response(), response(200) + result, session, sleep = dispatch([first, last]) + assert result is last and session.request.call_count == 2 + sleep.assert_called_once_with(1.0) + assert result.headers["Retry-After"] == "20" + assert [c.kwargs["timeout"] for c in session.request.call_args_list] == [ + (1, 2), + (1, 2), + ] + + +def test_allowed_wait_uses_bounded_advice(monkeypatch): + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_ADVICE_ENABLED", "true") + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_MAX_SECONDS", "5") + result, session, sleep = dispatch([response(), response(200)], deadline=106) + assert result.status_code == 200 and session.request.call_count == 2 + sleep.assert_called_once_with(5) + + +def test_deadline_refuses_wait_and_preserves_upstream(monkeypatch): + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_ADVICE_ENABLED", "true") + upstream = response() + original = dict(upstream.headers) + with patch.object(upstream, "close") as close: + result, session, sleep = dispatch([upstream], deadline=105) + assert result is upstream and result.content == b" {raw bytes}\n" + assert dict(result.headers) == original + assert session.request.call_count == 1 + sleep.assert_not_called() + close.assert_not_called() + + +@pytest.mark.parametrize( + "raw,streaming,key", + [(False, False, False), (True, False, True), (False, True, True)], +) +def test_ambiguous_post_raw_and_stream_do_not_gain_replay( + monkeypatch, raw, streaming, key +): + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_ADVICE_ENABLED", "true") + upstream = response() + result, session, sleep = dispatch( + [upstream], + method="POST", + raw=raw, + streaming=streaming, + headers={"Idempotency-Key": "synthetic"} if key else {}, + ) + assert result is upstream and result.content == b" {raw bytes}\n" + assert result.headers["Retry-After"] == "20" + assert session.request.call_count == 1 + sleep.assert_not_called() + + +def test_429_advice_does_not_heal_provider(monkeypatch): + from services.proxy_service import ProxyService + from services.resilience_service import ResilienceService + + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_ADVICE_ENABLED", "true") + with patch.object(ResilienceService, "record_result") as record: + result, _, _ = dispatch([response()], method="POST") + assert result.status_code == 429 + assert record.call_args.kwargs["outcome"].provider_health == "neutral" + assert not ProxyService._should_retry_status("POST", {}, 429, False) + + +def test_free_quota_enabled_clamps_and_disabled_preserves_legacy(monkeypatch): + from services.free_quota_service import FreeQuotaService, retry_seconds + + headers = { + "x-ratelimit-remaining-requests": "0", + "x-ratelimit-reset-requests": "2h", + } + assert retry_seconds(headers, 100) == 7200 + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_ADVICE_ENABLED", "true") + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_MAX_SECONDS", "5") + assert retry_seconds(headers, 100) == 5 + with patch.object(FreeQuotaService, "block") as block: + FreeQuotaService.observe("openai", headers, now=100) + block.assert_called_once_with("provider:openai", 5, now=100) + assert headers["x-ratelimit-reset-requests"] == "2h" + + +def test_free_quota_invalid_hint_bounds_default(monkeypatch): + from services.free_quota_service import retry_seconds + + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_ADVICE_ENABLED", "true") + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_MAX_SECONDS", "5") + assert retry_seconds({"Retry-After": "nan"}, 100) == 5 + + +@pytest.mark.parametrize( + "enabled,key,calls", [(False, False, 2), (True, False, 1), (True, True, 2)] +) +def test_timeout_body_does_not_authorize_ambiguous_post( + monkeypatch, enabled, key, calls +): + import json + + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_ADVICE_ENABLED", str(enabled).lower()) + upstream = response(400) + upstream._content = json.dumps( + {"error": {"message": "timeout", "code": 400}} + ).encode() + upstream.headers["Content-Type"] = "application/json" + result, session, _ = dispatch( + [upstream, response(200)], + method="POST", + provider="opencode", + headers={"Idempotency-Key": "synthetic"} if key else {}, + ) + assert session.request.call_count == calls + assert result.status_code == (200 if calls == 2 else 400) + + +@pytest.mark.parametrize( + "error,method,count", + [ + (requests.ReadTimeout("synthetic"), "POST", 1), + (requests.ConnectionError("synthetic"), "POST", 1), + (requests.ConnectTimeout("synthetic"), "POST", 2), + (requests.ReadTimeout("synthetic"), "GET", 2), + ], +) +def test_enabled_exception_policy_is_unchanged(monkeypatch, error, method, count): + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_ADVICE_ENABLED", "true") + result, session, sleep = dispatch([error, response(200)], method=method) + assert session.request.call_count == count + assert result.status_code == (200 if count == 2 else 502) + assert sleep.call_count == count - 1 + + +def test_deadline_expired_during_wait_keeps_response_open(monkeypatch): + from services.managed_dispatch import retry_managed_attempt + from services.upstream_outcome import classify_upstream_outcome + + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_ADVICE_ENABLED", "true") + upstream = response(hint="2") + attempt, sleep = Mock(), Mock() + with ( + patch("services.managed_dispatch.time.time", return_value=100), + patch("services.managed_dispatch.time.monotonic", side_effect=[100, 104]), + patch.object(upstream, "close") as close, + ): + result = retry_managed_attempt( + attempt, + lambda: classify_upstream_outcome(429, replay_permission=True), + retry_count=0, + max_retries=3, + retry_delay=1, + response=upstream, + deadline=103, + sleep=sleep, + ) + assert result is None + attempt.assert_not_called() + close.assert_not_called() + sleep.assert_called_once_with(2) + + +@pytest.mark.parametrize("permission,count", [(False, 0), (True, 3)]) +def test_permission_and_attempt_limit_precede_parsing(monkeypatch, permission, count): + from services.managed_dispatch import retry_managed_attempt + from services.upstream_outcome import classify_upstream_outcome + + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_ADVICE_ENABLED", "true") + attempt, sleep = Mock(), Mock() + upstream = response() + with patch("services.managed_dispatch.parse_retry_advice") as parse_hint: + result = retry_managed_attempt( + attempt, + lambda: classify_upstream_outcome(429, replay_permission=permission), + retry_count=count, + max_retries=3, + retry_delay=1, + response=upstream, + sleep=sleep, + ) + assert result is None + attempt.assert_not_called() + sleep.assert_not_called() + parse_hint.assert_not_called() + + +def test_existing_cascade_deadline_is_used(monkeypatch): + from flask import Flask, g + + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_ADVICE_ENABLED", "true") + with Flask(__name__).test_request_context(): + g.cascade_deadline = 105 + upstream = response() + result, session, sleep = dispatch([upstream]) + assert result is upstream and session.request.call_count == 1 + sleep.assert_not_called() + + +@pytest.mark.parametrize("flag", ["false", "true"]) +@pytest.mark.parametrize("provider", ["nanogpt", "groq"]) +def test_registered_proxy_path_preserves_raw_error( + monkeypatch, tmp_path, flag, provider +): + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_ADVICE_ENABLED", flag) + for name, value in { + "ADMIN_API_KEY": "admin-test-key", + "FLASK_SECRET_KEY": "test-secret", + "JWT_SECRET": "jwt-test-secret", + "AUTH_DB_PATH": str(tmp_path / "auth.sqlite3"), + "RATE_LIMIT_DB_PATH": str(tmp_path / "limits.sqlite3"), + "MODEL_REGISTRY_DB_PATH": str(tmp_path / "models.sqlite3"), + }.items(): + monkeypatch.setenv(name, value) + with patch("env_loader.load_runtime_env"), patch("config.load_runtime_env"): + import app + with patch("app.load_runtime_env"), patch("config.load_runtime_env"): + application = app.create_app() + application.config.update( + WTF_CSRF_ENABLED=False, IMAGE_RELAY_CATALOG_AUTO_REFRESH=False + ) + upstream = response() + session = Mock() + session.request.side_effect = ( + [upstream, response(200)] if provider == "groq" else [upstream] + ) + with ( + patch.object(app.AuthService, "get_api_key", return_value="synthetic"), + # Other suites can leave pooled NanoGPT keys configured; keep this path keyless. + patch.object(app.AuthService, "get_api_keys", return_value=[]), + patch.object(app.ProxyService, "_get_provider_session", return_value=session), + patch("services.proxy_service.time.sleep") as sleep, + ): + result = application.test_client().get( + f"/{provider}/v1/models", headers={"Authorization": "Bearer admin-test-key"} + ) + if provider == "groq": + assert result.status_code == 200 and session.request.call_count == 2 + sleep.assert_called_once_with(20 if flag == "true" else 1.0) + else: + assert result.status_code == 429 and result.data == upstream.content + assert result.headers["Retry-After"] == "20" + assert session.request.call_count == 1 + sleep.assert_not_called() diff --git a/worker/container-env.mjs b/worker/container-env.mjs index f678771b..b7d7fd62 100644 --- a/worker/container-env.mjs +++ b/worker/container-env.mjs @@ -3,6 +3,8 @@ const DIRECT_ENV_KEYS = [ "SECRET_SCAN_DEFAULT", "UPSTREAM_DEFAULT_USER_AGENT", "UPSTREAM_DEFAULT_ORIGINATOR", + "UPSTREAM_RETRY_AFTER_ADVICE_ENABLED", + "UPSTREAM_RETRY_AFTER_MAX_SECONDS", "A6API_API_KEY", "AIHUBMIX_API_KEY", "AIHUBMIX_BASE_URL",