diff --git a/CHANGELOG.md b/CHANGELOG.md index 4236f73..5132175 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## 0.4.1 + +`verify()` and `verify_batch()` now treat a **rate-limited 429** (one with a +`Retry-After` header) like a 5xx: they log a warning and return a `degraded` +result (`approved=False` by default, per `fail_open`) instead of raising +`CacheVerifierError`. A rate limit is transient load on your request path, so +it should fall through to your LLM like an outage rather than throw. A 429 +*without* `Retry-After` means the monthly verify quota is used up and still +raises, as do other 4xx responses (bad key, bad request). +`CacheVerifierError` gains a `retry_after` attribute. Upgrade before the +hosted service switches its rate-limit response from 503 to the documented +429. + ## 0.4.0 `cacheverifier healthcheck --base-model` picks the model the local Health diff --git a/README.md b/README.md index 28bd334..95091e2 100644 --- a/README.md +++ b/README.md @@ -71,11 +71,13 @@ cv.feedback(query, answer, was_correct=True, similarity_score=0.86) fine-tuning / feedback / monitoring calls. Warm verification is tens of milliseconds server-side; 1s covers the network round trip and a cold model load after a deploy without letting a stuck verifier stall your request. -- **Fails closed.** On a timeout, connection error, or 5xx, `verify()` does not - raise — it logs a warning on the `cacheverifier` logger and returns a - `VerifyResult` with `degraded=True` and `approved=False`, so you fall through - to your LLM exactly as you would on a cache miss. A `4xx` (bad key, bad - request, rate limit) still raises `CacheVerifierError`. +- **Fails closed.** On a timeout, connection error, 5xx, or a rate-limited 429 + (one carrying `Retry-After`), `verify()` does not raise — it logs a warning on + the `cacheverifier` logger and returns a `VerifyResult` with `degraded=True` + and `approved=False`, so you fall through to your LLM exactly as you would on + a cache miss. Other `4xx` responses still raise `CacheVerifierError`: a bad + key, a bad request, or a 429 without `Retry-After`, which means your monthly + verify quota is used up. ```python cv = CacheVerifier(api_key="cv_...", verify_timeout=1.0, fail_open=False) # the defaults diff --git a/cacheverifier/__init__.py b/cacheverifier/__init__.py index 510f0de..6a2e884 100644 --- a/cacheverifier/__init__.py +++ b/cacheverifier/__init__.py @@ -18,5 +18,5 @@ from cacheverifier.client import CacheVerifier, CacheVerifierError, VerifyResult -__version__ = "0.4.0" +__version__ = "0.4.1" __all__ = ["CacheVerifier", "CacheVerifierError", "VerifyResult", "__version__"] diff --git a/cacheverifier/client.py b/cacheverifier/client.py index 7617116..a97d466 100644 --- a/cacheverifier/client.py +++ b/cacheverifier/client.py @@ -46,13 +46,15 @@ class CacheVerifierError(RuntimeError): """Raised for any non-2xx response from the API. `status_code` is the HTTP status; `detail` is the server's error message - (the JSON body's `detail` field when present, otherwise the raw text). + (the JSON body's `detail` field when present, otherwise the raw text); + `retry_after` is the response's `Retry-After` header, if it sent one. """ - def __init__(self, status_code: int, detail: str) -> None: + def __init__(self, status_code: int, detail: str, retry_after: str | None = None) -> None: super().__init__(f"CacheVerifier API error {status_code}: {detail}") self.status_code = status_code self.detail = detail + self.retry_after = retry_after @dataclass(frozen=True) @@ -69,8 +71,8 @@ class VerifyResult: fallback result (see `degraded`). - `latency_ms`: server-side model inference time, not round-trip time. - `degraded`: True when the verify call failed (timeout / connection error / - 5xx) and this result was synthesized by the client rather than returned by - the API. `approved` then reflects the client's `fail_open` setting + 5xx / rate-limited 429) and this result was synthesized by the client + rather than returned by the API. `approved` then reflects the client's `fail_open` setting (default `False` -> `approved=False`, i.e. fall through to your LLM). """ @@ -112,12 +114,14 @@ class CacheVerifier: - `verify_timeout` (default 1s, vs 10s for everything else) bounds how long a slow or overloaded verifier can stall your request. - - On a timeout, connection error, or 5xx, `verify()` does not raise -- it - logs a warning and returns a synthetic `VerifyResult` with - `degraded=True`. `fail_open=False` (the default) makes that result - `approved=False` so you fall through to your LLM; `fail_open=True` makes - it `approved=True` so you serve the cached answer anyway. 4xx responses - (bad key, bad request, rate limit) still raise `CacheVerifierError`. + - On a timeout, connection error, 5xx, or a rate-limited 429 (one with a + `Retry-After` header), `verify()` does not raise -- it logs a warning and + returns a synthetic `VerifyResult` with `degraded=True`. + `fail_open=False` (the default) makes that result `approved=False` so you + fall through to your LLM; `fail_open=True` makes it `approved=True` so + you serve the cached answer anyway. Other 4xx responses still raise + `CacheVerifierError`: a bad key, a bad request, or a 429 without + `Retry-After`, which means your monthly verify quota is used up. """ def __init__( @@ -157,7 +161,7 @@ def __exit__(self, *_exc: object) -> None: def verify(self, query: str, candidate_answer: str) -> VerifyResult: """Approve or reject one gray-zone cache hit. `POST /v1/verify`. - On a timeout / connection error / 5xx this returns a synthetic + On a timeout / connection error / 5xx / rate-limited 429 this returns a synthetic `VerifyResult` (`degraded=True`, `approved` per `fail_open`) instead of raising -- the verifier being unreachable should degrade to a normal cache miss, not an exception on your request path. @@ -189,14 +193,19 @@ def verify_batch(self, pairs: Sequence[tuple[str, str]]) -> list[VerifyResult]: def _verify_call(self, path: str, payload: dict[str, Any]) -> Any | None: """POST a verify request with the request-path timeout. Returns the parsed body, or None to signal "unavailable, fall back" on a - timeout / connection error / 5xx. 4xx still raises. + timeout / connection error / 5xx / 429 with `Retry-After`. Other 4xx + still raises: a bad key or request is a bug to fix, and a 429 without + `Retry-After` is the monthly quota running out, which needs a decision + (top up or subscribe), not a silent month of fallbacks. A rate limit is + transient load and should degrade like an outage instead. """ try: return self._post(path, json=payload, timeout=self._verify_timeout) except httpx.TransportError as exc: self._warn_degraded(path, exc) except CacheVerifierError as exc: - if exc.status_code < 500: + rate_limited = exc.status_code == 429 and exc.retry_after is not None + if exc.status_code < 500 and not rate_limited: raise self._warn_degraded(path, exc) return None @@ -365,7 +374,7 @@ def _unwrap(resp: httpx.Response) -> Any: detail = body["detail"] if isinstance(body, dict) and "detail" in body else resp.text except ValueError: detail = resp.text - raise CacheVerifierError(resp.status_code, str(detail)) + raise CacheVerifierError(resp.status_code, str(detail), resp.headers.get("retry-after")) def _drop_none(**kwargs: Any) -> dict[str, Any]: diff --git a/pyproject.toml b/pyproject.toml index 87fd0f5..3f11bb5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "cacheverifier" -version = "0.4.0" +version = "0.4.1" description = "Python client for the hosted CacheVerifier semantic-cache verification API" readme = "README.md" requires-python = ">=3.9" diff --git a/tests/test_client.py b/tests/test_client.py index e547b37..0d969ca 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -135,6 +135,7 @@ def handler(request: httpx.Request) -> httpx.Response: assert excinfo.value.status_code == 429 assert excinfo.value.detail == "free tier limit reached" + assert excinfo.value.retry_after is None def test_error_response_without_json_body_falls_back_to_text(): @@ -204,14 +205,35 @@ def handler(request: httpx.Request) -> httpx.Response: assert result.approved is False -def test_verify_still_raises_on_4xx(): +def test_verify_fails_closed_on_rate_limit_429(): def handler(request: httpx.Request) -> httpx.Response: - return httpx.Response(429, json={"detail": "rate limited"}) + return httpx.Response(429, headers={"Retry-After": "1"}, json={"detail": "rate limited"}) + + with make_client(handler) as cv: + result = cv.verify("q", "a") + + assert result.degraded is True + assert result.approved is False + + +def test_verify_batch_degrades_on_rate_limit_429(): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(429, headers={"Retry-After": "1"}, json={"detail": "rate limited"}) + + with make_client(handler) as cv: + results = cv.verify_batch([("q1", "a1"), ("q2", "a2")]) + + assert all(r.degraded and not r.approved for r in results) + + +def test_verify_still_raises_on_other_4xx(): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(401, json={"detail": "invalid API key"}) with make_client(handler) as cv, pytest.raises(CacheVerifierError) as excinfo: cv.verify("q", "a") - assert excinfo.value.status_code == 429 + assert excinfo.value.status_code == 401 def test_verify_batch_degrades_every_pair_on_failure():