diff --git a/docs/rate-limit-headers.md b/docs/rate-limit-headers.md new file mode 100644 index 00000000..bda118de --- /dev/null +++ b/docs/rate-limit-headers.md @@ -0,0 +1,82 @@ +# Rate-limit advisory headers + +`RATE_LIMIT_HEADERS_ENABLED` defaults to false. Missing, empty, `false`, `0`, +`no` and `off` leave response headers and usage JSON unchanged. Enable with +`true`, `1`, `yes` or `on`. Values are case insensitive and trimmed. Malformed +values disable advice and log one warning per process without the value. + +Managed, authenticated responses with a gateway admission snapshot can expose: + +- `X-MultiLLM-RateLimit-Limit`: the selected gateway counter's positive limit. +- `X-MultiLLM-RateLimit-Remaining`: capacity at admission, after reserving the + current request. A known exhausted counter has remaining `0`. +- `X-MultiLLM-RateLimit-Reset`: positive integer seconds until capacity expires, + captured at admission rather than recalculated when the response finishes. + +Unknown, unlimited, invalid and unavailable fields are omitted. No header contains +principal names, key prefixes, client addresses, provider credentials or provider +quota information. Provider `RateLimit-*`, `X-RateLimit-*` and `Retry-After` +headers are preserved. Raw provider paths receive no advisory headers. Streaming +bodies are forwarded unchanged; advice does not read or buffer them. + +With advice disabled, a successful managed response has no new headers. With +advice enabled, a request admitted against a two-request minute limit can return: + +```http +HTTP/1.1 200 OK +X-MultiLLM-RateLimit-Limit: 2 +X-MultiLLM-RateLimit-Remaining: 1 +X-MultiLLM-RateLimit-Reset: 60 +``` + +An exhausted minute counter, observed 17 seconds after the first admission: + +```http +HTTP/1.1 429 Too Many Requests +X-MultiLLM-RateLimit-Limit: 2 +X-MultiLLM-RateLimit-Remaining: 0 +X-MultiLLM-RateLimit-Reset: 43 +Retry-After: 43 +``` + +The ordinary counter is requests per minute. On rejection, the selected counter +follows the existing enforcement order: RPM, TPM, then daily requests. The JSON +error identifies the rejected counter. Daily reset advice uses the rolling +24-hour window, not midnight. A token denial conservatively uses the last token +usage expiry; advice is not a guarantee that the next request will fit. Existing +replay rules remain authoritative: neither these headers nor a 429 authorizes a +retry of a request that may have produced output. Upstream errors and bodies are +not replaced. Only a known gateway rejection fills a missing `Retry-After`. + +SQLite snapshots are read under the admission transaction. Request-local copies +survive preprocessing, finalization and subsequent admissions unchanged. The D1 +ledger supplies its current gateway counts, including cached shared figures; +precise expiry is unavailable and `Reset` is omitted. D1 consistency remains +bounded by its existing synchronization behavior; this feature does not make it +a globally atomic limiter. When exact expiry is unavailable on a gateway denial, +`Retry-After` uses the applicable full rolling window as conservative advice. + +When enabled, `GET /v1/usage` adds `rate_limits`, a mapping of at most 32 observed +providers to bounded counter snapshots for the authenticated caller. Selected +principal responses from `/usage/data` include the same shape, respecting existing +administrator authorization. Usage reads are current snapshots, not admissions: +no request is reserved or charged. SQLite reads use a read-only transaction; +missing storage or tables returns an empty mapping. D1 usage snapshots are empty +because its existing public interface lacks a bounded provider inventory; there +is no additional D1 synchronization or remote request. These are gateway limits, +not monetary budgets or provider free quotas. + +No new table, durable state, remote call or content retention is introduced. +Copies live only for the current request. Enabled SQLite requests add one bounded +expiry query under the existing lock; usage reads have a 32-provider bound. +Disabled traffic does not query expiry or usage storage. This feature adds no +provider token cost and does not change admission, reservation or retry policy. + +## Where the headers apply + +Flask adds the headers to managed `/v1/` responses; raw provider routes are +excluded. Counters are never read from request headers or payloads. Requests the +Worker forwards to the Container keep the Container's headers unchanged. Native +Worker routes (codex-easy, linkapi, opencode direct) enforce no gateway request +counters, so their responses carry no advisory headers. Set +`RATE_LIMIT_HEADERS_ENABLED` on the Container. diff --git a/middleware/rate_limit_headers.py b/middleware/rate_limit_headers.py new file mode 100644 index 00000000..538f9f59 --- /dev/null +++ b/middleware/rate_limit_headers.py @@ -0,0 +1,206 @@ +"""Gateway-only admission advice; never infer provider quotas or replay permission.""" + +import logging +import math +import os +import sqlite3 +from contextlib import closing +from datetime import datetime, timedelta, timezone +from typing import Mapping + +from flask import g, has_request_context, request + +logger = logging.getLogger(__name__) +HEADER_PREFIX = "X-MultiLLM-RateLimit-" +MAX_COUNTER = 2**53 - 1 +MAX_USAGE_PROVIDERS = 32 +_warned_invalid = False + + +def enabled() -> bool: + global _warned_invalid + value = os.environ.get("RATE_LIMIT_HEADERS_ENABLED", "").strip().lower() + if value in {"1", "true", "yes", "on"}: + return True + if value not in {"", "0", "false", "no", "off"} and not _warned_invalid: + _warned_invalid = True + logger.warning("Invalid RATE_LIMIT_HEADERS_ENABLED; advisory headers disabled") + return False + + +def _integer(value, minimum=0): + return type(value) is int and minimum <= value <= MAX_COUNTER + + +def bounded_snapshot(snapshot) -> dict: + if not isinstance(snapshot, Mapping) or not _integer(snapshot.get("limit"), 1): + return {} + result = {"limit": snapshot["limit"]} + remaining = snapshot.get("remaining") + if _integer(remaining) and remaining <= result["limit"]: + result["remaining"] = remaining + reset = snapshot.get("reset") + if _integer(reset, 1) and reset <= 86400: + result["reset"] = reset + return result + + +def snapshot_headers(snapshot) -> dict: + return {HEADER_PREFIX + name.title(): str(value) + for name, value in bounded_snapshot(snapshot).items()} + + +def _reset_seconds(connection, metadata, window, now, *, offset=0, latest=False): + """Read expiry from the same SQLite transaction as the admission counters.""" + query = ( + "SELECT created_at FROM request_usage " + "WHERE identity = ? AND provider = ? AND created_at >= ? AND id != ? " + "ORDER BY created_at DESC LIMIT 1 OFFSET ?" + ) if latest else ( + "SELECT created_at FROM request_usage " + "WHERE identity = ? AND provider = ? AND created_at >= ? AND id != ? " + "ORDER BY created_at ASC LIMIT 1 OFFSET ?" + ) + row = connection.execute(query, ( + metadata["identity"], metadata["provider"], (now - timedelta(seconds=window)).isoformat(), + metadata.get("reservation_id", -1), offset, + )).fetchone() + if row is None: + return None + admitted = datetime.fromisoformat(row["created_at"]) + return min(window, max(1, math.ceil((admitted + timedelta(seconds=window) - now).total_seconds()))) + + +def _counter_snapshot(metadata, minute_count, minute_tokens, daily_count, token_limited, *, consume=False): + rpm, daily = metadata["rpm_limit"], metadata["daily_request_limit"] + denied = minute_count >= rpm or token_limited or daily_count >= daily + limit, count, window, kind = rpm, minute_count, 60, "requests" + if minute_count < rpm and token_limited: + limit, count, kind = metadata["tpm_limit"], minute_tokens, "tokens" + elif minute_count < rpm and not token_limited and daily_count >= daily: + limit, count, window = daily, daily_count, 86400 + remaining = max(0, limit - count - int(consume and not denied)) + return bounded_snapshot({"limit": limit, "remaining": remaining}), count, window, kind, denied + + +def capture_admission(metadata, minute_count, minute_tokens, daily_count, + token_limited, now, *, connection=None): + """Small service hook, called under its existing transaction before check-and-admit. + + The first snapshot stays request-local even if transformation or fallback later + rechecks capacity. D1 counts are known to its ledger; precise expiry is not. + """ + if not enabled(): + return None + snapshot, count, window, kind, denied = _counter_snapshot( + metadata, minute_count, minute_tokens, daily_count, token_limited, consume=True) + reset = None + if snapshot and connection is not None: + offset = max(0, count - snapshot["limit"]) if denied and kind != "tokens" else 0 + reset = _reset_seconds(connection, metadata, window, now, offset=offset, + latest=denied and kind == "tokens") + if reset is None and not denied: + reset = window # The about-to-be-admitted request starts this window. + if reset is not None: + snapshot["reset"] = reset + if snapshot: + metadata["rate_limit_advice"] = dict(snapshot) + if has_request_context() and not hasattr(g, "rate_limit_advice"): + g.rate_limit_advice = dict(snapshot) + retry_after = (reset or window) if denied else None + if denied and has_request_context(): + g.gateway_rate_limit_retry_after = retry_after + return retry_after + + +def apply_rate_limit_headers(response, *, managed=True): + if not enabled() or not managed or not getattr(g, "authenticated_user", None): + return response + snapshot = getattr(g, "rate_limit_advice", None) + for name, value in snapshot_headers(snapshot).items(): + response.headers[name] = value + retry_after = getattr(g, "gateway_rate_limit_retry_after", None) + if response.status_code == 429 and _integer(retry_after, 1) and "Retry-After" not in response.headers: + response.headers["Retry-After"] = str(retry_after) + return response + + +def register_rate_limit_headers(app, *, is_managed=None): + """W22 registrar hook; the coordinator may inject its managed-route predicate.""" + if is_managed is None: + is_managed = lambda: request.path.startswith("/v1/") + + @app.after_request + def decorate(response): + if not enabled(): + return response + return apply_rate_limit_headers(response, managed=is_managed()) + + +def _utcnow(): + return datetime.now(timezone.utc) + + +def usage_snapshot(user, remote_addr=None, *, service=None, now=None) -> dict: + """Read at most 32 observed provider counters for one authorized principal. + + D1 has no bounded public provider/expiry inventory; omit unavailable fields + rather than read internal ledger state or start a new synchronization. + """ + if not enabled() or os.environ.get("RATE_LIMIT_ENABLED", "true").lower() in {"0", "false", "no"}: + return {} + if service is None: + from services.rate_limit_service import RateLimitService + service = RateLimitService + from services import rate_limit_d1 + if rate_limit_d1.active(): + return {} + now = now or _utcnow() + identity, _ = service._identity_for_user(user, remote_addr) + path = service._get_storage_path().resolve() + if not path.exists(): + return {} + try: + with closing(sqlite3.connect(path.as_uri() + "?mode=ro", uri=True)) as connection: + connection.row_factory = sqlite3.Row + connection.execute("BEGIN") + providers = connection.execute( + "SELECT DISTINCT provider FROM request_usage WHERE identity = ? AND created_at >= ? " + "ORDER BY provider LIMIT ?", (identity, (now - timedelta(days=1)).isoformat(), MAX_USAGE_PROVIDERS), + ).fetchall() + result = {} + for row in providers: + snapshot = _usage_provider_snapshot(connection, service, identity, row["provider"], now) + if snapshot: + result[row["provider"]] = snapshot + return result + except (sqlite3.Error, ValueError): + logger.warning("Gateway rate-limit usage snapshot unavailable") + return {} + + +def _usage_provider_snapshot(connection, service, identity, provider, now): + counts = connection.execute( + "SELECT SUM(CASE WHEN created_at >= ? THEN 1 ELSE 0 END) AS requests, " + "SUM(CASE WHEN created_at >= ? THEN estimated_tokens ELSE 0 END) AS tokens, " + "COUNT(*) AS daily FROM request_usage WHERE identity = ? AND provider = ? AND created_at >= ?", + ((now - timedelta(seconds=60)).isoformat(), (now - timedelta(seconds=60)).isoformat(), + identity, provider, (now - timedelta(days=1)).isoformat()), + ).fetchone() + metadata = { + "identity": identity, "provider": provider, + "rpm_limit": service._provider_limit(provider, "RATE_LIMIT_RPM", service.RATE_LIMITS.get( + provider, service.RATE_LIMITS["default"])["requests"]), + "tpm_limit": service._provider_limit(provider, "RATE_LIMIT_TPM", 200000), + "daily_request_limit": service._provider_limit(provider, "DAILY_REQUEST_LIMIT", 10000), + } + tokens = counts["tokens"] or 0 + snapshot, count, window, kind, denied = _counter_snapshot( + metadata, counts["requests"] or 0, tokens, counts["daily"], tokens >= metadata["tpm_limit"]) + if snapshot: + reset = _reset_seconds(connection, metadata, window, now, + offset=max(0, count - snapshot["limit"]) if denied and kind != "tokens" else 0, + latest=denied and kind == "tokens") + if reset is not None: + snapshot["reset"] = reset + return snapshot diff --git a/route_helpers.py b/route_helpers.py index 3d8624c4..598045ac 100644 --- a/route_helpers.py +++ b/route_helpers.py @@ -37,7 +37,7 @@ + ", ".join([*CLIENT_HEADER_NAMES.values(), *OPENCODE_CLIENT_HEADER_NAMES.values()]) ) CORS_EXPOSE_HEADERS = ( - "Retry-After, X-Request-ID, X-MultiLLM-Optimization, X-MultiLLM-Tool-Repair, X-MultiLLM-Image-QA, X-MultiLLM-Cascade, X-MultiLLM-Conversion-Fidelity, X-MultiLLM-Conversion-Fields, " + "Retry-After, X-Request-ID, X-MultiLLM-RateLimit-Limit, X-MultiLLM-RateLimit-Remaining, X-MultiLLM-RateLimit-Reset, X-MultiLLM-Optimization, X-MultiLLM-Tool-Repair, X-MultiLLM-Image-QA, X-MultiLLM-Cascade, X-MultiLLM-Conversion-Fidelity, X-MultiLLM-Conversion-Fields, " "X-MultiLLM-Optimization-Mode, X-MultiLLM-Estimated-Input-Before, " "X-MultiLLM-Estimated-Input-After, X-MultiLLM-Image-Prompts-Compacted, " "X-MultiLLM-Messages-Summarized, X-MultiLLM-Optimization-Target-Met, " diff --git a/routes/usage.py b/routes/usage.py index 0a06f7b6..66b99489 100644 --- a/routes/usage.py +++ b/routes/usage.py @@ -7,6 +7,7 @@ from flask import g, jsonify, render_template, request from error_handlers import APIError +from middleware.rate_limit_headers import enabled as rate_limit_headers_enabled, usage_snapshot from request_validation import json_object_body from route_helpers import api_authenticate_only, login_required from services import key_controls, usage_ledger, usage_store @@ -73,6 +74,8 @@ def api_usage(): principal = str(user.get("username") or user.get("id")) controls = key_controls.public(user) return jsonify({ + **({"rate_limits": usage_snapshot(user, request.remote_addr)} + if rate_limit_headers_enabled() else {}), "object": "usage", "principal": principal, "key_prefix": user.get("api_key_prefix"), @@ -105,6 +108,8 @@ def usage_data(): current_user.get("username")) payload["budget"] = BudgetService.status(record or {"username": principal}) payload["controls"] = key_controls.public(record or {}) + if rate_limit_headers_enabled(): + payload["rate_limits"] = usage_snapshot(record or {"username": principal}, request.remote_addr) if is_admin: payload["ledger"] = usage_ledger.LEDGER.stats() return jsonify(payload) diff --git a/services/rate_limit_service.py b/services/rate_limit_service.py index 49955c96..f3504aa0 100644 --- a/services/rate_limit_service.py +++ b/services/rate_limit_service.py @@ -13,6 +13,7 @@ from flask import request from config import Config from services import rate_limit_d1 +from middleware.rate_limit_headers import capture_admission from services.sqlite_store import connect, storage_path logger = logging.getLogger(__name__) @@ -323,15 +324,20 @@ def _denial( rpm_limit: int, daily_limit: int, metadata: Dict[str, Any], + *, minute_tokens=0, connection=None, ) -> Optional[LimitDecision]: """The first exceeded limit, checked in the same order by every ledger.""" + retry_after = capture_admission( + metadata, minute_count, minute_tokens, daily_count, + token_limited, _utcnow(), connection=connection, + ) if minute_count >= rpm_limit: return LimitDecision( False, status_code=429, error="rate_limit_exceeded", message="Request-per-minute limit exceeded.", - retry_after=60, + retry_after=retry_after or 60, metadata=metadata, ) if token_limited: @@ -340,7 +346,7 @@ def _denial( status_code=429, error="token_rate_limit_exceeded", message="Token-per-minute limit exceeded.", - retry_after=60, + retry_after=retry_after or 60, metadata=metadata, ) if daily_count >= daily_limit: @@ -349,7 +355,7 @@ def _denial( status_code=429, error="daily_budget_exceeded", message="Daily request budget exceeded.", - retry_after=60 * 60, + retry_after=retry_after or 60 * 60, metadata=metadata, ) return None @@ -444,6 +450,7 @@ def reserve_request_slot( rpm_limit, daily_limit, metadata, + minute_tokens=minute_tokens, ) if denial is not None: return denial @@ -484,6 +491,8 @@ def reserve_request_slot( rpm_limit, daily_limit, metadata, + minute_tokens=minute_tokens, + connection=connection, ) if denial is not None: connection.rollback() @@ -628,6 +637,7 @@ def finalize_request_slot( rpm_limit, daily_limit, metadata, + minute_tokens=minute_tokens, ) if denial is not None: return denial @@ -688,6 +698,8 @@ def finalize_request_slot( rpm_limit, daily_limit, metadata, + minute_tokens=minute_tokens, + connection=connection, ) if denial is not None: connection.rollback() @@ -810,6 +822,7 @@ def enforce_request( rpm_limit, daily_limit, metadata, + minute_tokens=minute_tokens, ) if denial is not None: return denial @@ -851,6 +864,8 @@ def enforce_request( rpm_limit, daily_limit, metadata, + minute_tokens=minute_tokens, + connection=connection, ) if denial is not None: connection.rollback() diff --git a/tests/test_rate_limit_headers.py b/tests/test_rate_limit_headers.py new file mode 100644 index 00000000..dca580b7 --- /dev/null +++ b/tests/test_rate_limit_headers.py @@ -0,0 +1,333 @@ +"""Admission snapshots and response advice without provider quota disclosure.""" + +import sys +from datetime import datetime, timedelta, timezone + +import pytest +from flask import Flask, Response, g + +from middleware import rate_limit_headers as advice +from services import rate_limit_d1 +from services.rate_limit_service import RateLimitService + +USER = {"username": "private-principal", "api_key_prefix": "private-prefix"} +NOW = datetime(2026, 10, 9, 12, tzinfo=timezone.utc) +PREFIX = "X-MultiLLM-RateLimit-" + + +@pytest.fixture(autouse=True) +def limits(monkeypatch, tmp_path): + monkeypatch.setenv("RATE_LIMIT_HEADERS_ENABLED", "true") + monkeypatch.setenv("RATE_LIMIT_ENABLED", "true") + monkeypatch.setenv("RATE_LIMIT_DB_PATH", str(tmp_path / "limits.sqlite3")) + monkeypatch.setenv("RATE_LIMIT_RPM", "2") + monkeypatch.setenv("OPENAI_RATE_LIMIT_RPM", "2") + monkeypatch.setenv("RATE_LIMIT_TPM", "200000") + monkeypatch.setenv("DAILY_REQUEST_LIMIT", "10000") + pin_storage_and_clock(monkeypatch, NOW) + + +def loaded_namespaces(*names): + """Every loaded copy of these modules: other suites pop and re-import them.""" + namespaces = {id(vars(advice)): vars(advice), id(vars(rate_limit_d1)): vars(rate_limit_d1)} + service = RateLimitService.reserve_request_slot.__func__.__globals__ + namespaces[id(service)] = service + for name in names: + module = sys.modules.get(name) + if module is not None: + namespaces[id(vars(module))] = vars(module) + return namespaces.values() + + +def pin_storage_and_clock(monkeypatch, when): + for namespace in loaded_namespaces( + "services.rate_limit_service", "middleware.rate_limit_headers", "services.rate_limit_d1"): + if "_utcnow" in namespace: + monkeypatch.setitem(namespace, "_utcnow", lambda: when) + if "active" in namespace and namespace.get("__name__", "").endswith("rate_limit_d1"): + monkeypatch.setitem(namespace, "active", lambda: False) + + +def reserve(): + return RateLimitService.reserve_request_slot("openai", USER, "203.0.113.8") + + +def minimal_app(): + app = Flask(__name__) + advice.register_rate_limit_headers(app) + + @app.route("/v1/chat/completions", methods=["POST"]) + @app.route("/openai/v1/chat/completions", methods=["POST"]) + def chat(): + g.authenticated_user = USER + decision = reserve() + response = Response(b"provider-body", status=decision.status_code) + response.headers["X-RateLimit-Remaining"] = "provider-private" + response.headers["RateLimit-Limit"] = "provider-limit" + if not decision.allowed: + response.headers["Retry-After"] = str(decision.retry_after) + return response + + return app + + +@pytest.mark.parametrize("value", [None, "", "false", "0", "no", "off", "malformed-private-value"]) +def test_disabled_is_byte_for_byte_unchanged(value, monkeypatch, caplog): + if value is None: + monkeypatch.delenv("RATE_LIMIT_HEADERS_ENABLED", raising=False) + else: + monkeypatch.setenv("RATE_LIMIT_HEADERS_ENABLED", value) + decision = reserve() + assert "rate_limit_advice" not in decision.metadata + client = minimal_app().test_client() + response = client.post("/v1/chat/completions") + assert response.data == b"provider-body" + assert not any(name.startswith(PREFIX) for name in response.headers.keys()) + assert "malformed-private-value" not in caplog.text + + +def test_malformed_flag_warns_once_without_value(monkeypatch, caplog): + monkeypatch.setattr(advice, "_warned_invalid", False) + monkeypatch.setenv("RATE_LIMIT_HEADERS_ENABLED", "private-bad-flag") + assert not advice.enabled() and not advice.enabled() + assert sum("RATE_LIMIT_HEADERS_ENABLED" in record.message for record in caplog.records) == 1 + assert "private-bad-flag" not in caplog.text + + +def test_allowed_and_denied_snapshots_and_precise_retry(monkeypatch): + first = reserve() + assert first.metadata["rate_limit_advice"] == {"limit": 2, "remaining": 1, "reset": 60} + pin_storage_and_clock(monkeypatch, NOW + timedelta(seconds=17)) + second = reserve() + denied = reserve() + assert second.metadata["rate_limit_advice"] == {"limit": 2, "remaining": 0, "reset": 43} + assert denied.metadata["rate_limit_advice"] == second.metadata["rate_limit_advice"] + assert denied.retry_after == 43 and denied.status_code == 429 + assert first.metadata["rate_limit_advice"]["remaining"] == 1 + + +def test_snapshot_is_stable_across_finalization_and_later_admission(monkeypatch): + app = minimal_app() + with app.test_request_context("/v1/chat/completions"): + g.authenticated_user = USER + first = reserve() + reserve() + pin_storage_and_clock(monkeypatch, NOW + timedelta(seconds=10)) + finalized = RateLimitService.finalize_request_slot( + first.metadata["reservation_id"], "openai", USER, b"{}", {}, "203.0.113.8") + assert finalized.allowed + response = advice.apply_rate_limit_headers(Response("ok")) + assert response.headers[PREFIX + "Remaining"] == "1" + assert response.headers[PREFIX + "Reset"] == "60" + assert not any(value in str(response.headers) for value in USER.values()) + + +def test_registered_middleware_preserves_provider_headers_and_raw_bytes(): + client = minimal_app().test_client() + allowed = client.post("/v1/chat/completions") + assert allowed.headers[PREFIX + "Remaining"] == "1" + assert allowed.headers["X-RateLimit-Remaining"] == "provider-private" + assert allowed.headers["RateLimit-Limit"] == "provider-limit" + assert allowed.data == b"provider-body" + denied = client.post("/v1/chat/completions") + assert denied.status_code == 200 + denied = client.post("/v1/chat/completions") + assert denied.status_code == 429 and denied.headers["Retry-After"] == "60" + assert denied.headers[PREFIX + "Remaining"] == "0" + raw = client.post("/openai/v1/chat/completions") + assert raw.data == b"provider-body" + assert not any(name.startswith(PREFIX) for name in raw.headers.keys()) + + +def test_upstream_retry_after_and_unknown_fields_are_preserved(): + app = minimal_app() + with app.test_request_context("/v1/chat/completions"): + g.authenticated_user = USER + reserve() + response = Response("upstream throttle", status=429, headers={"Retry-After": "upstream-date"}) + assert advice.apply_rate_limit_headers(response).headers["Retry-After"] == "upstream-date" + assert advice.snapshot_headers({"limit": None, "remaining": None, "reset": None}) == {} + assert advice.snapshot_headers({"limit": 0, "remaining": 0, "reset": 0}) == {} + assert advice.snapshot_headers({"limit": 3, "remaining": 0}) == {PREFIX + "Limit": "3", PREFIX + "Remaining": "0"} + assert advice.snapshot_headers({"limit": True, "remaining": "2", "reset": float("inf")}) == {} + + +def test_unlimited_omits_all_advice(monkeypatch): + monkeypatch.setenv("RATE_LIMIT_ENABLED", "false") + response = minimal_app().test_client().post("/v1/chat/completions") + assert not any(name.startswith(PREFIX) for name in response.headers.keys()) + assert "rate_limit_advice" not in reserve().metadata + + +@pytest.mark.parametrize("kind,limit,window", [("DAILY_REQUEST_LIMIT", 1, 86400), ("RATE_LIMIT_TPM", 1, 60)]) +def test_binding_daily_and_token_denials(kind, limit, window, monkeypatch): + monkeypatch.setenv(kind, str(limit)) + payload = {"max_tokens": 1} + first = RateLimitService.enforce_request("openai", USER, b"{}", payload, "203.0.113.8") + assert first.allowed + denied = RateLimitService.enforce_request("openai", USER, b"{}", payload, "203.0.113.8") + assert denied.status_code == 429 + assert denied.metadata["rate_limit_advice"] == {"limit": limit, "remaining": 0, "reset": window} + assert denied.retry_after == window + + +def test_d1_uses_known_counts_without_inventing_reset(monkeypatch): + monkeypatch.setattr(rate_limit_d1, "active", lambda: True) + monkeypatch.setattr(rate_limit_d1, "identity_key", lambda _: "synthetic-hash") + monkeypatch.setattr(rate_limit_d1.control_state_d1, "ensure_running", lambda: None) + monkeypatch.setattr(rate_limit_d1.control_state_d1, "wake", lambda: None) + rate_limit_d1.reset() + try: + assert reserve().metadata["rate_limit_advice"] == {"limit": 2, "remaining": 1} + assert reserve().allowed + assert reserve().metadata["rate_limit_advice"] == {"limit": 2, "remaining": 0} + finally: + rate_limit_d1.reset() + + +def test_usage_registered_route_default_and_bounded_snapshot(monkeypatch): + from routes import usage + from flask_wtf.csrf import CSRFProtect + app = Flask(__name__) + app.config["WTF_CSRF_ENABLED"] = False + monkeypatch.setattr(usage, "api_authenticate_only", lambda **_: lambda view: view) + monkeypatch.setattr(usage, "usage_history", lambda *args, **kwargs: {}) + monkeypatch.setattr(usage.BudgetService, "status", lambda _: {}) + monkeypatch.setattr(usage.key_controls, "public", lambda _: { + "allowed_models": None, "allowed_ips": None, "expires_at": None}) + @app.before_request + def authenticate(): + g.authenticated_user = USER + usage.register_usage_routes(app, CSRFProtect()) + client = app.test_client() + monkeypatch.setenv("RATE_LIMIT_HEADERS_ENABLED", "false") + before = client.get("/v1/usage").data + assert "rate_limits" not in client.get("/v1/usage").json + monkeypatch.setenv("RATE_LIMIT_HEADERS_ENABLED", "") + assert client.get("/v1/usage").data == before + monkeypatch.setenv("RATE_LIMIT_HEADERS_ENABLED", "true") + reserve() + payload = client.get("/v1/usage").json + assert payload["rate_limits"] == {"openai": {"limit": 2, "remaining": 1, "reset": 60}} + assert "private-prefix" not in str(payload["rate_limits"]) + + +def test_admin_usage_registered_route_includes_selected_principal_snapshot(monkeypatch): + from routes import usage + from flask_wtf.csrf import CSRFProtect + app = Flask(__name__) + monkeypatch.setattr(usage, "api_authenticate_only", lambda **_: lambda view: view) + monkeypatch.setattr(usage, "login_required", lambda view: view) + monkeypatch.setattr(usage, "usage_history", lambda *args, **kwargs: {}) + monkeypatch.setattr(usage.AuthService, "get_current_user", lambda: {"username": "admin", "is_admin": True}) + monkeypatch.setattr(usage.AuthService, "get_user_record", lambda _: USER) + monkeypatch.setattr(usage.BudgetService, "status", lambda _: {}) + monkeypatch.setattr(usage.key_controls, "public", lambda _: {}) + monkeypatch.setattr(usage.usage_ledger.LEDGER, "stats", lambda: {}) + usage.register_usage_routes(app, CSRFProtect()) + reserve() + payload = app.test_client().get("/usage/data?principal=private-principal").json + assert payload["rate_limits"] == {"openai": {"limit": 2, "remaining": 1, "reset": 60}} + + +def test_gateway_denial_fills_only_missing_retry_and_requires_authentication(): + app = minimal_app() + with app.test_request_context("/v1/chat/completions"): + reserve() + reserve() + reserve() + response = Response("denied", status=429) + assert not any(name.startswith(PREFIX) for name in advice.apply_rate_limit_headers(response).headers.keys()) + g.authenticated_user = USER + decorated = advice.apply_rate_limit_headers(response) + assert decorated.headers["Retry-After"] == "60" + + +def test_sqlite_concurrent_admission_snapshots_are_atomic(): + from concurrent.futures import ThreadPoolExecutor + with ThreadPoolExecutor(max_workers=4) as pool: + decisions = list(pool.map(lambda _: reserve(), range(4))) + assert sum(decision.allowed for decision in decisions) == 2 + assert sorted(decision.metadata["rate_limit_advice"]["remaining"] for decision in decisions) == [0, 0, 0, 1] + + +def test_real_managed_chat_route_with_mounted_middleware(monkeypatch, tmp_path): + import config + monkeypatch.setattr(config, "load_runtime_env", lambda: None) + for name, value in { + "ADMIN_API_KEY": "synthetic-admin-key", "FLASK_SECRET_KEY": "synthetic-session-secret", + "JWT_SECRET": "synthetic-jwt-secret", "OPENCODE_GO_API_KEY": "synthetic-provider-key", + "OPENCODE_API_KEY": "synthetic-provider-key", "AUTH_DB_PATH": str(tmp_path / "auth.sqlite3"), + "MODEL_REGISTRY_DB_PATH": str(tmp_path / "models.sqlite3"), + "PROVIDER_CATALOG_AUTO_REFRESH": "false", "OPENCODE_RATE_LIMIT_RPM": "2", + }.items(): + monkeypatch.setenv(name, value) + from app import create_app + from tests.unified_api_test_case import UnifiedApiTestCase + from unittest.mock import patch + app = create_app() + pin_storage_and_clock(monkeypatch, NOW) + app.config["WTF_CSRF_ENABLED"] = False + advice.register_rate_limit_headers(app) + client = app.test_client() + with patch("app.ProxyService.make_request", return_value=UnifiedApiTestCase._chat_response()) as upstream: + for remaining in ("1", "0"): + response = client.post("/v1/chat/completions", headers={"Authorization": "Bearer synthetic-admin-key"}, + json={"model": "opencode:glm-5.2", "messages": [{"role": "user", "content": "hi"}]}) + assert response.status_code == 200 + assert response.headers[PREFIX + "Remaining"] == remaining + refused = client.post("/v1/chat/completions", headers={"Authorization": "Bearer synthetic-admin-key"}, + json={"model": "opencode:glm-5.2", "messages": [{"role": "user", "content": "hi"}]}) + assert refused.status_code == 429 and refused.headers["Retry-After"] == "60" + assert upstream.call_count == 2 + usage = client.get("/v1/usage", headers={"Authorization": "Bearer synthetic-admin-key"}) + assert usage.status_code == 200 and usage.json["rate_limits"]["opencode"]["remaining"] == 0 + + +def test_disabled_never_queries_expiry_or_usage(monkeypatch): + monkeypatch.setenv("RATE_LIMIT_HEADERS_ENABLED", "false") + def unexpected(*args, **kwargs): + raise AssertionError("disabled advice must not inspect counters") + monkeypatch.setattr(advice, "_reset_seconds", unexpected) + assert reserve().allowed + assert advice.usage_snapshot(USER) == {} + + +def test_usage_snapshot_reports_binding_daily_counter(monkeypatch): + monkeypatch.setenv("DAILY_REQUEST_LIMIT", "1") + reserve() + assert advice.usage_snapshot(USER) == {"openai": {"limit": 1, "remaining": 0, "reset": 86400}} + + +def test_usage_snapshot_is_bounded_and_read_only(): + from contextlib import closing + with closing(RateLimitService._connect()) as connection: + RateLimitService._ensure_storage(connection) + connection.executemany( + "INSERT INTO request_usage (created_at, identity, provider) VALUES (?, ?, ?)", + [(NOW.isoformat(), USER["username"], f"provider-{index:02}") for index in range(40)], + ) + connection.commit() + snapshot = advice.usage_snapshot(USER) + assert len(snapshot) == 32 + assert list(snapshot) == [f"provider-{index:02}" for index in range(32)] + with closing(RateLimitService._connect()) as connection: + assert connection.execute("SELECT COUNT(*) FROM request_usage").fetchone()[0] == 40 + + +def test_missing_usage_storage_and_tables_are_omitted(tmp_path): + assert advice.usage_snapshot(USER) == {} + from contextlib import closing + with closing(RateLimitService._connect()) as connection: + connection.execute("CREATE TABLE unrelated (value INTEGER)") + connection.commit() + assert advice.usage_snapshot(USER) == {} + + +@pytest.mark.parametrize("snapshot", [ + {"limit": 3, "remaining": 4, "reset": 86401}, + {"limit": 3, "remaining": -1, "reset": 0}, + {"limit": 3, "remaining": True, "reset": "60"}, +]) +def test_invalid_optional_fields_are_omitted(snapshot): + assert advice.snapshot_headers(snapshot) == {PREFIX + "Limit": "3"} diff --git a/worker/container-env.mjs b/worker/container-env.mjs index f678771b..e142829d 100644 --- a/worker/container-env.mjs +++ b/worker/container-env.mjs @@ -26,6 +26,7 @@ const DIRECT_ENV_KEYS = [ "RATE_LIMIT_ENABLED", "RATE_LIMIT_RPM", "RATE_LIMIT_TPM", + "RATE_LIMIT_HEADERS_ENABLED", "DAILY_REQUEST_LIMIT", "FREE_ROUTE_EXTRA_PROVIDERS", "FREE_ROUTE_FREE_TIER_PROVIDERS", diff --git a/worker/cors-policy.mjs b/worker/cors-policy.mjs index 5d773e55..7205a966 100644 --- a/worker/cors-policy.mjs +++ b/worker/cors-policy.mjs @@ -3,4 +3,4 @@ export const CORS_DEFAULT_HEADERS = "X-MultiLLM-Tool-Repair, X-MultiLLM-Image-QA, X-MultiLLM-Cascade, X-MultiLLM-Conversion-Report, Authorization, X-Api-Key, X-Goog-Api-Key, X-MultiLLM-Api-Key, X-Roleplay-Session-ID, Anthropic-Version, Anthropic-Beta, Anthropic-Dangerous-Direct-Browser-Access, Content-Type, Accept, Origin, X-Requested-With, OpenAI-Beta, OpenAI-Organization, OpenAI-Project, Idempotency-Key, Moderation, Moderation-Model, Redaction, X-Client-Request-ID, X-App-Name, X-Billing-Mode, X-BYOK-Provider, X-Encryption-Key, X-Encryption-Passphrase, X-Fal-Object-Lifecycle-Preference, X-PAYMENT, X-Prompt-Caching-Cut-After, X-Provider, X-Team-ID, X-Use-BYOK, x-x402"; export const CORS_EXPOSE_HEADERS = "X-MultiLLM-Secret-Scan, " + - "X-MultiLLM-Tool-Repair, X-MultiLLM-Image-QA, X-MultiLLM-Cascade, X-MultiLLM-Conversion-Fidelity, X-MultiLLM-Conversion-Fields, Retry-After, X-Request-ID, X-MultiLLM-Optimization, X-MultiLLM-Optimization-Mode, X-MultiLLM-Estimated-Input-Before, X-MultiLLM-Estimated-Input-After, X-MultiLLM-Image-Prompts-Compacted, X-MultiLLM-Messages-Summarized, X-MultiLLM-Optimization-Target-Met, X-MultiLLM-Summary, X-MultiLLM-Optimization-Cache-Hits, X-MultiLLM-Optimization-Cache-Misses, X-MultiLLM-Prompt-Cache, X-MultiLLM-Prompt-Cache-Mode, X-MultiLLM-Prompt-Cache-Estimated-Tokens, X-MultiLLM-Provider, X-MultiLLM-Model, X-MultiLLM-Credential-Attempts, X-MultiLLM-Route-Decision, X-MultiLLM-Circuit-State, X-MultiLLM-Auto-Route, X-MultiLLM-Auto-Selected-Model, X-MultiLLM-Auto-Attempts, X-MultiLLM-Auto-Selected-Priority, X-MultiLLM-Auto-Ordering, X-MultiLLM-Auto-Failover-Reasons, X-MultiLLM-Cache, X-MultiLLM-Transport-Failure, X-MultiLLM-Token-Count, X-MultiLLM-Latency-Ms, X-MultiLLM-Estimated-Cost-USD, X-MultiLLM-Cost-Basis, X-MultiLLM-Reasoning-Normalized, X-Roleplay-Session-ID, X-Roleplay-Trace-ID, X-Roleplay-Session-Source, X-Roleplay-Provider, X-Roleplay-Model, X-Roleplay-Selection, X-Roleplay-Memory, X-Roleplay-Estimated-Input-Tokens, X-Roleplay-Max-Output-Tokens, X-Roleplay-Fallback-Count, X-Roleplay-State-Cache, X-Roleplay-Credential-Check, Server-Timing, WWW-Authenticate, X-PAYMENT-RESPONSE, X-Poll-After, X-NanoGPT-Advisor-ID, X-NanoGPT-Data-Endpoint, X-NanoGPT-Direct-Endpoint, X-NanoGPT-Inline-Moderation-Cost-USD, X-NanoGPT-Inline-Moderation-Flagged, X-NanoGPT-Inline-Moderation-Model"; + "X-MultiLLM-Tool-Repair, X-MultiLLM-Image-QA, X-MultiLLM-Cascade, X-MultiLLM-Conversion-Fidelity, X-MultiLLM-Conversion-Fields, Retry-After, X-Request-ID, X-MultiLLM-RateLimit-Limit, X-MultiLLM-RateLimit-Remaining, X-MultiLLM-RateLimit-Reset, X-MultiLLM-Optimization, X-MultiLLM-Optimization-Mode, X-MultiLLM-Estimated-Input-Before, X-MultiLLM-Estimated-Input-After, X-MultiLLM-Image-Prompts-Compacted, X-MultiLLM-Messages-Summarized, X-MultiLLM-Optimization-Target-Met, X-MultiLLM-Summary, X-MultiLLM-Optimization-Cache-Hits, X-MultiLLM-Optimization-Cache-Misses, X-MultiLLM-Prompt-Cache, X-MultiLLM-Prompt-Cache-Mode, X-MultiLLM-Prompt-Cache-Estimated-Tokens, X-MultiLLM-Provider, X-MultiLLM-Model, X-MultiLLM-Credential-Attempts, X-MultiLLM-Route-Decision, X-MultiLLM-Circuit-State, X-MultiLLM-Auto-Route, X-MultiLLM-Auto-Selected-Model, X-MultiLLM-Auto-Attempts, X-MultiLLM-Auto-Selected-Priority, X-MultiLLM-Auto-Ordering, X-MultiLLM-Auto-Failover-Reasons, X-MultiLLM-Cache, X-MultiLLM-Transport-Failure, X-MultiLLM-Token-Count, X-MultiLLM-Latency-Ms, X-MultiLLM-Estimated-Cost-USD, X-MultiLLM-Cost-Basis, X-MultiLLM-Reasoning-Normalized, X-Roleplay-Session-ID, X-Roleplay-Trace-ID, X-Roleplay-Session-Source, X-Roleplay-Provider, X-Roleplay-Model, X-Roleplay-Selection, X-Roleplay-Memory, X-Roleplay-Estimated-Input-Tokens, X-Roleplay-Max-Output-Tokens, X-Roleplay-Fallback-Count, X-Roleplay-State-Cache, X-Roleplay-Credential-Check, Server-Timing, WWW-Authenticate, X-PAYMENT-RESPONSE, X-Poll-After, X-NanoGPT-Advisor-ID, X-NanoGPT-Data-Endpoint, X-NanoGPT-Direct-Endpoint, X-NanoGPT-Inline-Moderation-Cost-USD, X-NanoGPT-Inline-Moderation-Flagged, X-NanoGPT-Inline-Moderation-Model";