From aa62d6816d0b836855986a7fdc06208c2a7dbce1 Mon Sep 17 00:00:00 2001 From: Abhyudaya Baiju Tharakan Date: Fri, 9 Oct 2026 03:15:11 +0530 Subject: [PATCH] feat(transport): add opt-in bounded upstream retry advice UPSTREAM_RETRY_AFTER_ADVICE_ENABLED (default off) parses Retry-After and documented reset headers into bounded local advice. An allowed managed retry then waits for the longer of the existing backoff and the advice, and is skipped when the wait would pass the request deadline. Parsing never grants a retry, and upstream headers are never rewritten. The managed retry scheduling moves into services/managed_dispatch.py and behaves exactly as before while the flag is off. --- docs/retry-advice.md | 67 +++++ services/free_quota_service.py | 6 + services/managed_dispatch.py | 65 +++++ services/proxy_service.py | 87 +++---- services/retry_advice.py | 183 ++++++++++++++ tests/test_retry_advice.py | 436 +++++++++++++++++++++++++++++++++ worker/container-env.mjs | 2 + 7 files changed, 805 insertions(+), 41 deletions(-) create mode 100644 docs/retry-advice.md create mode 100644 services/managed_dispatch.py create mode 100644 services/retry_advice.py create mode 100644 tests/test_retry_advice.py diff --git a/docs/retry-advice.md b/docs/retry-advice.md new file mode 100644 index 00000000..d9ee1d51 --- /dev/null +++ b/docs/retry-advice.md @@ -0,0 +1,67 @@ +# Bounded upstream Retry-After advice + +`UPSTREAM_RETRY_AFTER_ADVICE_ENABLED` defaults to false. Unset or empty settings +use their defaults. Disabled traffic keeps its existing retries, waits, headers, +response bodies and free-pool cooldown timing. Native Worker and raw Flask paths +continue to forward upstream bytes without parsing or scheduling retries. + +Enable local advice with: + +```text +UPSTREAM_RETRY_AFTER_ADVICE_ENABLED=true +UPSTREAM_RETRY_AFTER_MAX_SECONDS=3600 +``` + +The maximum defaults to 3600 seconds and accepts whole seconds from 1 to 604800, +matching the existing free-pool cooldown ceiling. An invalid flag or enabled +maximum disables advice and logs the setting name once per process, without its +value. The maximum is ignored while advice is disabled. + +`services.retry_advice.parse_retry_advice` returns immutable local advice with +`source`, `observed_at`, `retry_at`, `delay_seconds` and `scope`. Times are Unix +seconds. It recognizes these formats, using case-insensitive header names: + +- `Retry-After`: nonnegative decimal seconds or an HTTP-date with a timezone. +- `X-RateLimit-Reset`: an absolute Unix timestamp in seconds. +- `X-RateLimit-Reset-Requests` and `X-RateLimit-Reset-Tokens`: duration strings, + such as `1h2m3.4s`, when the matching remaining counter is finite and exhausted. +- `Anthropic-RateLimit-Requests-Reset` and `Anthropic-RateLimit-Tokens-Reset`: + ISO timestamps with a timezone, when the matching remaining counter is exhausted. + +Malformed, nonfinite, zero-delay and stale hints are ignored. Individual values +are limited to 512 characters. When several active hints are valid, the latest +reset wins so an exhausted dimension is not bypassed. Only local advice is +clamped; the upstream header is never replaced. With observation time 100, +`Retry-After: 20` yields retry time 120. With maximum 5 it yields local retry time +105, while the header remains `20`. + +The [upstream outcome](upstream-outcomes.md) classifier keeps 429 throttling neutral for provider health. +Exhausted remaining counters label advice `quota_exhausted`; other 429 advice +uses `rate_limit`; provider failures use `provider`. Authentication and caller +errors do not supply retry advice. Successful exhausted responses can supply +cooldown advice without changing their health classification. + +Managed retries still require the existing method/idempotency/transport policy +and attempt limit. Parsing never grants replay permission. An ambiguous POST or +stream does not gain a retry from headers. While enabled, legacy timeout-body +retries additionally require a safe method or an idempotency key. Connect-timeout +retries remain allowed by the existing policy because nothing reached upstream. +Existing managed response normalization remains in the proxy service. + +For an allowed retry, the scheduler waits for the greater of existing backoff +and bounded advice. Invalid or absent advice uses existing backoff. An optional +monotonic `deadline` on `_make_base_request`, or the existing Flask +`g.cascade_deadline`, must cover the entire wait. A refused wait returns the +original upstream result. The scheduler rechecks the deadline after sleeping and +does not dispatch a late retry. Without a shared deadline, existing per-attempt +timeouts remain unchanged; the feature does not create a request-wide deadline +or reserve time for another attempt. Free-pool `retry_seconds` uses the same +bounded parser while enabled, including a bounded fallback cooldown. + +There are no new routes, response headers, storage tables or dependencies. Advice +reads headers only and retains no credentials, prompts or response content. +Existing cooldown storage and its monotonic extension rules remain in use; +enabling a lower maximum does not shorten a previously stored cooldown. Advice +is neither proof of provider capacity nor authorization for additional cost. +Retries can consume additional quota only when already authorized by transport +policy. The Worker forwards both settings to the Container. diff --git a/services/free_quota_service.py b/services/free_quota_service.py index 7ce2a99b..2cca1a22 100644 --- a/services/free_quota_service.py +++ b/services/free_quota_service.py @@ -11,6 +11,7 @@ from email.utils import parsedate_to_datetime from services import free_quota_d1 +from services.retry_advice import parse_retry_advice, retry_advice_settings from services.sqlite_store import connect, storage_path MAX_COOLDOWN = 7 * 24 * 3600 @@ -30,6 +31,11 @@ def reset_seconds(value: str) -> float | None: def retry_seconds(headers, now: float, default: int = 60) -> int: + settings = retry_advice_settings() + if settings.enabled: + advice = parse_retry_advice(headers, now=now, max_seconds=settings.max_seconds) + delay = advice.delay_seconds if advice is not None else default + return min(settings.max_seconds, max(1, math.ceil(delay))) normalized = {k.lower(): v for k, v in headers.items()} value = normalized.get("retry-after", "") delay = None diff --git a/services/managed_dispatch.py b/services/managed_dispatch.py new file mode 100644 index 00000000..99911a54 --- /dev/null +++ b/services/managed_dispatch.py @@ -0,0 +1,65 @@ +"""Managed attempt scheduling, separate from raw dispatch and normalization.""" + +import math +import time +from typing import Callable + +import requests +from flask import g, has_request_context + +from services.retry_advice import parse_retry_advice, retry_advice_settings +from services.upstream_outcome import UpstreamOutcome +from services.upstream_transport import close_retry_response + +Attempt = Callable[[int], requests.Response] +Outcome = Callable[[], UpstreamOutcome] + + +def retry_managed_attempt( + attempt: Attempt, + outcome: Outcome, + *, + retry_count: int, + max_retries: int, + retry_delay: float, + response: requests.Response | None = None, + deadline: float | None = None, + sleep: Callable[[float], None] = time.sleep, +) -> requests.Response | None: + """Dispatch again only under the existing transport's replay permission. + + A supplied deadline is monotonic. The existing cascade deadline is used when + available; absent one, per-attempt timeouts remain the transport's policy. + None means that the original response/error must finish normally. + """ + classified = outcome() + if not classified.replay_permission or retry_count >= max_retries: + return None + settings = retry_advice_settings() + delay = retry_delay * (retry_count + 1) + if settings.enabled: + if response is not None: + advice = parse_retry_advice( + response.headers, + now=time.time(), + status_code=response.status_code, + max_seconds=settings.max_seconds, + ) + if advice is not None: + delay = max(delay, advice.delay_seconds) + if deadline is None and has_request_context(): + deadline = getattr(g, "cascade_deadline", None) + if deadline is not None: + if not math.isfinite(deadline) or delay >= deadline - time.monotonic(): + return None + # Keep the original response usable if scheduling consumes the budget. + sleep(delay) + if time.monotonic() >= deadline: + return None + if response is not None: + close_retry_response(response) + return attempt(retry_count + 1) + if response is not None: + close_retry_response(response) + sleep(delay) + return attempt(retry_count + 1) diff --git a/services/proxy_service.py b/services/proxy_service.py index b90fab76..e78c1861 100644 --- a/services/proxy_service.py +++ b/services/proxy_service.py @@ -1,6 +1,7 @@ import json import logging import requests +from functools import partial from typing import Optional, Dict, Any, Tuple, List, Generator from concurrent.futures import ThreadPoolExecutor from error_handlers import APIError @@ -14,6 +15,8 @@ from services.credential_pool import CredentialPool from services.resilience_service import ResilienceService from services.upstream_outcome import UpstreamOutcome, classify_upstream_outcome +from services.managed_dispatch import retry_managed_attempt +from services.retry_advice import retry_advice_settings from services.transport_policy import RAW_PASSTHROUGH_PROVIDERS from services.secret_firewall import protect_body from providers.cline_pass import cline_completion_payload @@ -1250,6 +1253,7 @@ def _make_base_request( timeout_override: Optional[Tuple[int, int]] = None, force_raw_passthrough: bool = False, is_streaming: Optional[bool] = None, + deadline: Optional[float] = None, ) -> requests.Response: """ Make a base request with retries and error handling @@ -1262,6 +1266,16 @@ def _make_base_request( force_raw_passthrough or api_provider in RAW_PASSTHROUGH_PROVIDERS ) + next_request = partial( + cls._make_base_request, method=method, url=url, headers=headers, + params=params, data=data, api_provider=api_provider, use_cache=use_cache, + timeout_override=timeout_override, force_raw_passthrough=force_raw_passthrough, + deadline=deadline, + ) + + def next_attempt(count: int) -> requests.Response: + return next_request(retry_count=count, is_streaming=is_streaming) + try: if not raw_passthrough and retry_count == 0: circuit_response = cls._circuit_open_response(api_provider) @@ -1325,21 +1339,14 @@ def _make_base_request( retry_count + 1, MAX_RETRIES, ) - close_retry_response(response) - time.sleep(RETRY_DELAY * (retry_count + 1)) - return cls._make_base_request( - method=method, - url=url, - headers=headers, - params=params, - data=data, - api_provider=api_provider, - use_cache=use_cache, - retry_count=retry_count + 1, - timeout_override=timeout_override, - force_raw_passthrough=force_raw_passthrough, - is_streaming=is_streaming, + retried = retry_managed_attempt( + next_attempt, + lambda: classify_upstream_outcome(response.status_code, replay_permission=True), + retry_count=retry_count, max_retries=MAX_RETRIES, + retry_delay=RETRY_DELAY, response=response, deadline=deadline, sleep=time.sleep, ) + if retried is not None: + return retried if not is_streaming: try: @@ -1384,21 +1391,23 @@ def _make_base_request( retry_count + 1, MAX_RETRIES, ) - close_retry_response(response) - time.sleep(RETRY_DELAY * (retry_count + 1)) - return cls._make_base_request( - method=method, - url=url, - headers=headers, - params=params, - data=data, - api_provider=api_provider, - use_cache=use_cache, - retry_count=retry_count + 1, - timeout_override=timeout_override, - force_raw_passthrough=force_raw_passthrough, - is_streaming=is_streaming, + replay_allowed = ( + not retry_advice_settings().enabled + or (not is_streaming and ( + method.upper() in cls.SAFE_RETRY_METHODS + or cls._has_idempotency_key(headers) + )) + ) + retried = retry_managed_attempt( + next_attempt, + lambda: classify_upstream_outcome( + response.status_code, replay_permission=replay_allowed, + ), + retry_count=retry_count, max_retries=MAX_RETRIES, + retry_delay=RETRY_DELAY, response=response, deadline=deadline, sleep=time.sleep, ) + if retried is not None: + return retried if api_provider == "cline-pass" and response.status_code < 400: normalized_payload = cline_completion_payload(normalized_payload) @@ -1461,20 +1470,16 @@ def _make_base_request( and retry_count < MAX_RETRIES ): logger.info(f"Retrying request (attempt {retry_count + 1}/{MAX_RETRIES})") - time.sleep(RETRY_DELAY * (retry_count + 1)) - return cls._make_base_request( - method=method, - url=url, - headers=headers, - params=params, - data=data, - api_provider=api_provider, - use_cache=use_cache, - retry_count=retry_count + 1, - timeout_override=timeout_override, - force_raw_passthrough=force_raw_passthrough, - is_streaming=is_streaming, + retried = retry_managed_attempt( + next_attempt, + lambda error=e: classify_upstream_outcome( + transport_failure=cls._transport_failure_kind(error), replay_permission=True, + ), + retry_count=retry_count, max_retries=MAX_RETRIES, + retry_delay=RETRY_DELAY, deadline=deadline, sleep=time.sleep, ) + if retried is not None: + return retried if not raw_passthrough: cls._record_circuit_result( api_provider, 503, diff --git a/services/retry_advice.py b/services/retry_advice.py new file mode 100644 index 00000000..7d2419ca --- /dev/null +++ b/services/retry_advice.py @@ -0,0 +1,183 @@ +"""Content-free, bounded retry hints; these never authorize request replay.""" + +import logging +import math +import os +import re +import threading +from dataclasses import dataclass +from datetime import datetime +from email.utils import parsedate_to_datetime +from typing import Mapping + +from services.upstream_outcome import classify_upstream_outcome + +logger = logging.getLogger(__name__) +_DEFAULT_MAX = 3600 +_warned_settings: set[str] = set() +_warning_lock = threading.Lock() +_DURATION = re.compile( + r"(?:(\d+(?:\.\d+)?)d)?(?:(\d+(?:\.\d+)?)h)?(?:(\d+(?:\.\d+)?)m)?" + r"(?:(\d+(?:\.\d+)?)s)?(?:(\d+(?:\.\d+)?)ms)?" +) +_NUMBER = re.compile(r"\d+(?:\.\d+)?") + + +@dataclass(frozen=True) +class RetryAdviceSettings: + enabled: bool = False + max_seconds: int = _DEFAULT_MAX + + +@dataclass(frozen=True) +class RetryAdvice: + source: str + observed_at: float + retry_at: float + delay_seconds: float + scope: str + + +def _warn_once(name: str) -> None: + with _warning_lock: + if name in _warned_settings: + return + _warned_settings.add(name) + logger.warning("Invalid %s; upstream retry advice disabled", name) + + +def retry_advice_settings() -> RetryAdviceSettings: + flag_name = "UPSTREAM_RETRY_AFTER_ADVICE_ENABLED" + flag = os.environ.get(flag_name, "").strip().lower() + if flag in ("", "false", "0", "no", "off"): + return RetryAdviceSettings() + if flag not in ("true", "1", "yes", "on"): + _warn_once(flag_name) + return RetryAdviceSettings() + limit_name = "UPSTREAM_RETRY_AFTER_MAX_SECONDS" + value = os.environ.get(limit_name, "").strip() + try: + limit = int(value) if value else _DEFAULT_MAX + # Match the existing free-pool maximum and keep sleep values operational. + if not 1 <= limit <= 7 * 24 * 3600: + raise ValueError + except ValueError: + _warn_once(limit_name) + return RetryAdviceSettings() + return RetryAdviceSettings(True, limit) + + +def _value(headers: Mapping[str, str], name: str) -> str: + value = headers.get(name, "") + return value.strip() if isinstance(value, str) and len(value) <= 512 else "" + + +def _number(value: str) -> float | None: + if not _NUMBER.fullmatch(value): + return None + number = float(value) + return number if math.isfinite(number) else None + + +def _duration(value: str) -> float | None: + match = _DURATION.fullmatch(value) + if not match or not any(match.groups()): + return None + delay = sum( + float(amount or 0) * unit + for amount, unit in zip(match.groups(), (86400, 3600, 60, 1, 0.001)) + ) + return delay if math.isfinite(delay) else None + + +def _date_delay(value: str, now: float, *, iso: bool = False) -> float | None: + try: + date = ( + datetime.fromisoformat(value.replace("Z", "+00:00")) + if iso + else parsedate_to_datetime(value) + ) + if date.tzinfo is None: + return None + delay = date.timestamp() - now + return delay if math.isfinite(delay) else None + except (ValueError, TypeError, OverflowError, OSError): + return None + + +def _exhausted(headers: Mapping[str, str], name: str) -> bool: + try: + remaining = float(_value(headers, name)) + return math.isfinite(remaining) and remaining <= 0 + except ValueError: + return False + + +def _reset_hints( + headers: Mapping[str, str], now: float +) -> tuple[list[tuple[str, float]], bool]: + hints: list[tuple[str, float]] = [] + exhausted = False + for dimension in ("requests", "tokens"): + for prefix, iso in (("x-ratelimit", False), ("anthropic-ratelimit", True)): + if not _exhausted( + headers, + f"{prefix}-remaining-{dimension}" + if not iso + else f"{prefix}-{dimension}-remaining", + ): + continue + exhausted = True + name = ( + f"{prefix}-reset-{dimension}" + if not iso + else f"{prefix}-{dimension}-reset" + ) + value = _value(headers, name) + delay = _date_delay(value, now, iso=True) if iso else _duration(value) + if delay is not None and delay > 0: + hints.append((name, delay)) + return hints, exhausted + + +def parse_retry_advice( + headers: Mapping[str, str], + *, + now: float, + status_code: int = 429, + max_seconds: int = _DEFAULT_MAX, +) -> RetryAdvice | None: + """Normalize known header formats without reading bodies or mutating headers. + + Exhausted dimensions and Retry-After combine using the latest valid reset. + This avoids sending before either active limit permits another attempt. + """ + if not math.isfinite(now) or not 1 <= max_seconds <= 7 * 24 * 3600: + return None + outcome = classify_upstream_outcome(status_code) + if outcome.reason in ("http_credential_rejected", "http_caller_error"): + return None + normalized = {name.lower(): value for name, value in headers.items()} + hints, exhausted = _reset_hints(normalized, now) + if outcome.reason == "http_throttled": + scope = "quota_exhausted" if exhausted else "rate_limit" + elif outcome.provider_health == "failure": + scope = "provider" + elif outcome.provider_health == "success" and exhausted: + scope = "quota_exhausted" + else: + return None + value = _value(normalized, "retry-after") + delay = _number(value) + if delay is None: + delay = _date_delay(value, now) + if delay is not None and delay > 0: + hints.append(("retry-after", delay)) + epoch = _number(_value(normalized, "x-ratelimit-reset")) + if epoch is not None and epoch > now: + hints.append(("x-ratelimit-reset", epoch - now)) + if not hints: + return None + source, delay = max(hints, key=lambda hint: hint[1]) + delay = min(delay, max_seconds) + return RetryAdvice(source, now, now + delay, delay, scope) diff --git a/tests/test_retry_advice.py b/tests/test_retry_advice.py new file mode 100644 index 00000000..bb233204 --- /dev/null +++ b/tests/test_retry_advice.py @@ -0,0 +1,436 @@ +"""Retry hints change local scheduling, never transport permissions or bytes.""" + +from dataclasses import FrozenInstanceError +from email.utils import formatdate +from unittest.mock import Mock, patch + +import pytest +import requests + + +@pytest.fixture(autouse=True) +def settings(monkeypatch): + monkeypatch.delenv("UPSTREAM_RETRY_AFTER_ADVICE_ENABLED", raising=False) + monkeypatch.delenv("UPSTREAM_RETRY_AFTER_MAX_SECONDS", raising=False) + monkeypatch.setenv("SECRET_SCAN_DEFAULT", "off") + from services.resilience_service import ResilienceService + + ResilienceService.reset() + yield + ResilienceService.reset() + + +def parse(headers, now=100, status=429, maximum=3600): + from services.retry_advice import parse_retry_advice + + return parse_retry_advice(headers, now=now, status_code=status, max_seconds=maximum) + + +def test_delta_date_equivalence_and_immutable_advice(): + delta = parse({"Retry-After": "20"}) + date = parse({"retry-after": formatdate(120, usegmt=True)}) + assert ( + (delta.observed_at, delta.retry_at, delta.delay_seconds, delta.scope) + == (date.observed_at, date.retry_at, date.delay_seconds, date.scope) + == (100, 120, 20, "rate_limit") + ) + assert delta.source == "retry-after" + with pytest.raises(FrozenInstanceError): + delta.delay_seconds = 1 + + +@pytest.mark.parametrize( + "value", + [ + "", + "bad", + "nan", + "inf", + "-1", + "0", + "1e999", + "x" * 513, + formatdate(99, usegmt=True), + ], +) +def test_malformed_nonfinite_and_stale_advice(value): + assert parse({"Retry-After": value}) is None + + +def test_maximum_clamps_only_local_advice(): + headers = {"Retry-After": "999999", "X-Unrelated": "unchanged"} + before = headers.copy() + advice = parse(headers, maximum=10) + assert (advice.delay_seconds, advice.retry_at) == (10, 110) + assert headers == before + + +@pytest.mark.parametrize( + "value,delay", + [("120", 20), ("99", None), ("nan", None), ("-1", None), ("999999999", 3600)], +) +def test_epoch_reset(value, delay): + advice = parse({"X-RateLimit-Reset": value}) + assert (advice.delay_seconds if advice else None) == delay + + +def test_exhausted_dimensions_and_precise_reset_precedence(): + advice = parse( + { + "Retry-After": "2", + "X-RateLimit-Remaining-Requests": "0", + "X-RateLimit-Reset-Requests": "1h2m3.4s", + "X-RateLimit-Remaining-Tokens": "0", + "X-RateLimit-Reset-Tokens": "7.66s", + } + ) + assert (advice.source, advice.delay_seconds, advice.scope) == ( + "x-ratelimit-reset-requests", + 3600, + "quota_exhausted", + ) + advice = parse( + { + "anthropic-ratelimit-tokens-remaining": "0", + "anthropic-ratelimit-tokens-reset": "1970-01-01T00:02:00Z", + } + ) + assert advice.delay_seconds == 20 and advice.scope == "quota_exhausted" + assert ( + parse( + {"X-RateLimit-Remaining-Requests": "1", "X-RateLimit-Reset-Requests": "5s"} + ) + is None + ) + + +@pytest.mark.parametrize("status", [400, 401, 403, 404, 422, 200]) +def test_authentication_and_caller_errors_are_not_throttling(status): + assert parse({"Retry-After": "20"}, status=status) is None + + +@pytest.mark.parametrize( + "flag,maximum,enabled,limit", + [ + ("", "", False, 3600), + ("false", "bad", False, 3600), + ("true", "", True, 3600), + ("1", "10", True, 10), + ("bad", "10", False, 3600), + ("true", "nan", False, 3600), + ("true", "-1", False, 3600), + ("true", "0", False, 3600), + ], +) +def test_configuration_defaults_and_invalid_values( + monkeypatch, flag, maximum, enabled, limit +): + from services.retry_advice import retry_advice_settings + + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_ADVICE_ENABLED", flag) + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_MAX_SECONDS", maximum) + settings = retry_advice_settings() + assert (settings.enabled, settings.max_seconds) == (enabled, limit) + + +def test_invalid_settings_log_once_without_value(monkeypatch, caplog): + from services import retry_advice + + retry_advice._warned_settings.clear() + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_ADVICE_ENABLED", "private-invalid-setting") + for _ in range(2): + assert not retry_advice.retry_advice_settings().enabled + assert len(caplog.records) == 1 + assert "private-invalid-setting" not in caplog.text + + +def response(status=429, hint="20"): + result = requests.Response() + result.status_code = status + result._content = b" {raw bytes}\n" + result.headers.update({"Content-Type": "text/plain", "Retry-After": hint}) + return result + + +def dispatch( + upstreams, + *, + method="GET", + headers=None, + raw=False, + deadline=None, + streaming=False, + provider="openai", +): + from services.proxy_service import ProxyService + + session = Mock() + session.request.side_effect = upstreams + with ( + patch.object(ProxyService, "_get_provider_session", return_value=session), + patch("services.proxy_service.time.sleep") as sleep, + patch("services.managed_dispatch.time.time", return_value=100), + patch("services.managed_dispatch.time.monotonic", return_value=100), + ): + kwargs = {"deadline": deadline} if deadline is not None else {} + result = ProxyService._make_base_request( + method, + "https://provider.invalid/v1/models", + headers or {}, + {}, + b"{}" if method == "POST" else None, + provider, + force_raw_passthrough=raw, + is_streaming=streaming, + timeout_override=(1, 2), + **kwargs, + ) + return result, session, sleep + + +@pytest.mark.parametrize("flag", [None, "", "false", "bad"]) +def test_disabled_transport_retains_waits_and_headers(monkeypatch, flag): + if flag is not None: + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_ADVICE_ENABLED", flag) + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_MAX_SECONDS", "2") + first, last = response(), response(200) + result, session, sleep = dispatch([first, last]) + assert result is last and session.request.call_count == 2 + sleep.assert_called_once_with(1.0) + assert result.headers["Retry-After"] == "20" + assert [c.kwargs["timeout"] for c in session.request.call_args_list] == [ + (1, 2), + (1, 2), + ] + + +def test_allowed_wait_uses_bounded_advice(monkeypatch): + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_ADVICE_ENABLED", "true") + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_MAX_SECONDS", "5") + result, session, sleep = dispatch([response(), response(200)], deadline=106) + assert result.status_code == 200 and session.request.call_count == 2 + sleep.assert_called_once_with(5) + + +def test_deadline_refuses_wait_and_preserves_upstream(monkeypatch): + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_ADVICE_ENABLED", "true") + upstream = response() + original = dict(upstream.headers) + with patch.object(upstream, "close") as close: + result, session, sleep = dispatch([upstream], deadline=105) + assert result is upstream and result.content == b" {raw bytes}\n" + assert dict(result.headers) == original + assert session.request.call_count == 1 + sleep.assert_not_called() + close.assert_not_called() + + +@pytest.mark.parametrize( + "raw,streaming,key", + [(False, False, False), (True, False, True), (False, True, True)], +) +def test_ambiguous_post_raw_and_stream_do_not_gain_replay( + monkeypatch, raw, streaming, key +): + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_ADVICE_ENABLED", "true") + upstream = response() + result, session, sleep = dispatch( + [upstream], + method="POST", + raw=raw, + streaming=streaming, + headers={"Idempotency-Key": "synthetic"} if key else {}, + ) + assert result is upstream and result.content == b" {raw bytes}\n" + assert result.headers["Retry-After"] == "20" + assert session.request.call_count == 1 + sleep.assert_not_called() + + +def test_429_advice_does_not_heal_provider(monkeypatch): + from services.proxy_service import ProxyService + from services.resilience_service import ResilienceService + + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_ADVICE_ENABLED", "true") + with patch.object(ResilienceService, "record_result") as record: + result, _, _ = dispatch([response()], method="POST") + assert result.status_code == 429 + assert record.call_args.kwargs["outcome"].provider_health == "neutral" + assert not ProxyService._should_retry_status("POST", {}, 429, False) + + +def test_free_quota_enabled_clamps_and_disabled_preserves_legacy(monkeypatch): + from services.free_quota_service import FreeQuotaService, retry_seconds + + headers = { + "x-ratelimit-remaining-requests": "0", + "x-ratelimit-reset-requests": "2h", + } + assert retry_seconds(headers, 100) == 7200 + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_ADVICE_ENABLED", "true") + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_MAX_SECONDS", "5") + assert retry_seconds(headers, 100) == 5 + with patch.object(FreeQuotaService, "block") as block: + FreeQuotaService.observe("openai", headers, now=100) + block.assert_called_once_with("provider:openai", 5, now=100) + assert headers["x-ratelimit-reset-requests"] == "2h" + + +def test_free_quota_invalid_hint_bounds_default(monkeypatch): + from services.free_quota_service import retry_seconds + + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_ADVICE_ENABLED", "true") + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_MAX_SECONDS", "5") + assert retry_seconds({"Retry-After": "nan"}, 100) == 5 + + +@pytest.mark.parametrize( + "enabled,key,calls", [(False, False, 2), (True, False, 1), (True, True, 2)] +) +def test_timeout_body_does_not_authorize_ambiguous_post( + monkeypatch, enabled, key, calls +): + import json + + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_ADVICE_ENABLED", str(enabled).lower()) + upstream = response(400) + upstream._content = json.dumps( + {"error": {"message": "timeout", "code": 400}} + ).encode() + upstream.headers["Content-Type"] = "application/json" + result, session, _ = dispatch( + [upstream, response(200)], + method="POST", + provider="opencode", + headers={"Idempotency-Key": "synthetic"} if key else {}, + ) + assert session.request.call_count == calls + assert result.status_code == (200 if calls == 2 else 400) + + +@pytest.mark.parametrize( + "error,method,count", + [ + (requests.ReadTimeout("synthetic"), "POST", 1), + (requests.ConnectionError("synthetic"), "POST", 1), + (requests.ConnectTimeout("synthetic"), "POST", 2), + (requests.ReadTimeout("synthetic"), "GET", 2), + ], +) +def test_enabled_exception_policy_is_unchanged(monkeypatch, error, method, count): + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_ADVICE_ENABLED", "true") + result, session, sleep = dispatch([error, response(200)], method=method) + assert session.request.call_count == count + assert result.status_code == (200 if count == 2 else 502) + assert sleep.call_count == count - 1 + + +def test_deadline_expired_during_wait_keeps_response_open(monkeypatch): + from services.managed_dispatch import retry_managed_attempt + from services.upstream_outcome import classify_upstream_outcome + + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_ADVICE_ENABLED", "true") + upstream = response(hint="2") + attempt, sleep = Mock(), Mock() + with ( + patch("services.managed_dispatch.time.time", return_value=100), + patch("services.managed_dispatch.time.monotonic", side_effect=[100, 104]), + patch.object(upstream, "close") as close, + ): + result = retry_managed_attempt( + attempt, + lambda: classify_upstream_outcome(429, replay_permission=True), + retry_count=0, + max_retries=3, + retry_delay=1, + response=upstream, + deadline=103, + sleep=sleep, + ) + assert result is None + attempt.assert_not_called() + close.assert_not_called() + sleep.assert_called_once_with(2) + + +@pytest.mark.parametrize("permission,count", [(False, 0), (True, 3)]) +def test_permission_and_attempt_limit_precede_parsing(monkeypatch, permission, count): + from services.managed_dispatch import retry_managed_attempt + from services.upstream_outcome import classify_upstream_outcome + + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_ADVICE_ENABLED", "true") + attempt, sleep = Mock(), Mock() + upstream = response() + with patch("services.managed_dispatch.parse_retry_advice") as parse_hint: + result = retry_managed_attempt( + attempt, + lambda: classify_upstream_outcome(429, replay_permission=permission), + retry_count=count, + max_retries=3, + retry_delay=1, + response=upstream, + sleep=sleep, + ) + assert result is None + attempt.assert_not_called() + sleep.assert_not_called() + parse_hint.assert_not_called() + + +def test_existing_cascade_deadline_is_used(monkeypatch): + from flask import Flask, g + + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_ADVICE_ENABLED", "true") + with Flask(__name__).test_request_context(): + g.cascade_deadline = 105 + upstream = response() + result, session, sleep = dispatch([upstream]) + assert result is upstream and session.request.call_count == 1 + sleep.assert_not_called() + + +@pytest.mark.parametrize("flag", ["false", "true"]) +@pytest.mark.parametrize("provider", ["nanogpt", "groq"]) +def test_registered_proxy_path_preserves_raw_error( + monkeypatch, tmp_path, flag, provider +): + monkeypatch.setenv("UPSTREAM_RETRY_AFTER_ADVICE_ENABLED", flag) + for name, value in { + "ADMIN_API_KEY": "admin-test-key", + "FLASK_SECRET_KEY": "test-secret", + "JWT_SECRET": "jwt-test-secret", + "AUTH_DB_PATH": str(tmp_path / "auth.sqlite3"), + "RATE_LIMIT_DB_PATH": str(tmp_path / "limits.sqlite3"), + "MODEL_REGISTRY_DB_PATH": str(tmp_path / "models.sqlite3"), + }.items(): + monkeypatch.setenv(name, value) + with patch("env_loader.load_runtime_env"), patch("config.load_runtime_env"): + import app + with patch("app.load_runtime_env"), patch("config.load_runtime_env"): + application = app.create_app() + application.config.update( + WTF_CSRF_ENABLED=False, IMAGE_RELAY_CATALOG_AUTO_REFRESH=False + ) + upstream = response() + session = Mock() + session.request.side_effect = ( + [upstream, response(200)] if provider == "groq" else [upstream] + ) + with ( + patch.object(app.AuthService, "get_api_key", return_value="synthetic"), + # Other suites can leave pooled NanoGPT keys configured; keep this path keyless. + patch.object(app.AuthService, "get_api_keys", return_value=[]), + patch.object(app.ProxyService, "_get_provider_session", return_value=session), + patch("services.proxy_service.time.sleep") as sleep, + ): + result = application.test_client().get( + f"/{provider}/v1/models", headers={"Authorization": "Bearer admin-test-key"} + ) + if provider == "groq": + assert result.status_code == 200 and session.request.call_count == 2 + sleep.assert_called_once_with(20 if flag == "true" else 1.0) + else: + assert result.status_code == 429 and result.data == upstream.content + assert result.headers["Retry-After"] == "20" + assert session.request.call_count == 1 + sleep.assert_not_called() diff --git a/worker/container-env.mjs b/worker/container-env.mjs index f678771b..b7d7fd62 100644 --- a/worker/container-env.mjs +++ b/worker/container-env.mjs @@ -3,6 +3,8 @@ const DIRECT_ENV_KEYS = [ "SECRET_SCAN_DEFAULT", "UPSTREAM_DEFAULT_USER_AGENT", "UPSTREAM_DEFAULT_ORIGINATOR", + "UPSTREAM_RETRY_AFTER_ADVICE_ENABLED", + "UPSTREAM_RETRY_AFTER_MAX_SECONDS", "A6API_API_KEY", "AIHUBMIX_API_KEY", "AIHUBMIX_BASE_URL",