Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,18 @@
# Changelog

## 0.4.1

`verify()` and `verify_batch()` now treat a **rate-limited 429** (one with a
`Retry-After` header) like a 5xx: they log a warning and return a `degraded`
result (`approved=False` by default, per `fail_open`) instead of raising
`CacheVerifierError`. A rate limit is transient load on your request path, so
it should fall through to your LLM like an outage rather than throw. A 429
*without* `Retry-After` means the monthly verify quota is used up and still
raises, as do other 4xx responses (bad key, bad request).
`CacheVerifierError` gains a `retry_after` attribute. Upgrade before the
hosted service switches its rate-limit response from 503 to the documented
429.

## 0.4.0

`cacheverifier healthcheck --base-model` picks the model the local Health
Expand Down
12 changes: 7 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -71,11 +71,13 @@ cv.feedback(query, answer, was_correct=True, similarity_score=0.86)
fine-tuning / feedback / monitoring calls. Warm verification is tens of
milliseconds server-side; 1s covers the network round trip and a cold model
load after a deploy without letting a stuck verifier stall your request.
- **Fails closed.** On a timeout, connection error, or 5xx, `verify()` does not
raise — it logs a warning on the `cacheverifier` logger and returns a
`VerifyResult` with `degraded=True` and `approved=False`, so you fall through
to your LLM exactly as you would on a cache miss. A `4xx` (bad key, bad
request, rate limit) still raises `CacheVerifierError`.
- **Fails closed.** On a timeout, connection error, 5xx, or a rate-limited 429
(one carrying `Retry-After`), `verify()` does not raise — it logs a warning on
the `cacheverifier` logger and returns a `VerifyResult` with `degraded=True`
and `approved=False`, so you fall through to your LLM exactly as you would on
a cache miss. Other `4xx` responses still raise `CacheVerifierError`: a bad
key, a bad request, or a 429 without `Retry-After`, which means your monthly
verify quota is used up.

```python
cv = CacheVerifier(api_key="cv_...", verify_timeout=1.0, fail_open=False) # the defaults
Expand Down
2 changes: 1 addition & 1 deletion cacheverifier/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,5 +18,5 @@

from cacheverifier.client import CacheVerifier, CacheVerifierError, VerifyResult

__version__ = "0.4.0"
__version__ = "0.4.1"
__all__ = ["CacheVerifier", "CacheVerifierError", "VerifyResult", "__version__"]
37 changes: 23 additions & 14 deletions cacheverifier/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,13 +46,15 @@ class CacheVerifierError(RuntimeError):
"""Raised for any non-2xx response from the API.

`status_code` is the HTTP status; `detail` is the server's error message
(the JSON body's `detail` field when present, otherwise the raw text).
(the JSON body's `detail` field when present, otherwise the raw text);
`retry_after` is the response's `Retry-After` header, if it sent one.
"""

def __init__(self, status_code: int, detail: str) -> None:
def __init__(self, status_code: int, detail: str, retry_after: str | None = None) -> None:
super().__init__(f"CacheVerifier API error {status_code}: {detail}")
self.status_code = status_code
self.detail = detail
self.retry_after = retry_after


@dataclass(frozen=True)
Expand All @@ -69,8 +71,8 @@ class VerifyResult:
fallback result (see `degraded`).
- `latency_ms`: server-side model inference time, not round-trip time.
- `degraded`: True when the verify call failed (timeout / connection error /
5xx) and this result was synthesized by the client rather than returned by
the API. `approved` then reflects the client's `fail_open` setting
5xx / rate-limited 429) and this result was synthesized by the client
rather than returned by the API. `approved` then reflects the client's `fail_open` setting
(default `False` -> `approved=False`, i.e. fall through to your LLM).
"""

Expand Down Expand Up @@ -112,12 +114,14 @@ class CacheVerifier:

- `verify_timeout` (default 1s, vs 10s for everything else) bounds how long
a slow or overloaded verifier can stall your request.
- On a timeout, connection error, or 5xx, `verify()` does not raise -- it
logs a warning and returns a synthetic `VerifyResult` with
`degraded=True`. `fail_open=False` (the default) makes that result
`approved=False` so you fall through to your LLM; `fail_open=True` makes
it `approved=True` so you serve the cached answer anyway. 4xx responses
(bad key, bad request, rate limit) still raise `CacheVerifierError`.
- On a timeout, connection error, 5xx, or a rate-limited 429 (one with a
`Retry-After` header), `verify()` does not raise -- it logs a warning and
returns a synthetic `VerifyResult` with `degraded=True`.
`fail_open=False` (the default) makes that result `approved=False` so you
fall through to your LLM; `fail_open=True` makes it `approved=True` so
you serve the cached answer anyway. Other 4xx responses still raise
`CacheVerifierError`: a bad key, a bad request, or a 429 without
`Retry-After`, which means your monthly verify quota is used up.
"""

def __init__(
Expand Down Expand Up @@ -157,7 +161,7 @@ def __exit__(self, *_exc: object) -> None:
def verify(self, query: str, candidate_answer: str) -> VerifyResult:
"""Approve or reject one gray-zone cache hit. `POST /v1/verify`.

On a timeout / connection error / 5xx this returns a synthetic
On a timeout / connection error / 5xx / rate-limited 429 this returns a synthetic
`VerifyResult` (`degraded=True`, `approved` per `fail_open`) instead of
raising -- the verifier being unreachable should degrade to a normal
cache miss, not an exception on your request path.
Expand Down Expand Up @@ -189,14 +193,19 @@ def verify_batch(self, pairs: Sequence[tuple[str, str]]) -> list[VerifyResult]:
def _verify_call(self, path: str, payload: dict[str, Any]) -> Any | None:
"""POST a verify request with the request-path timeout. Returns the
parsed body, or None to signal "unavailable, fall back" on a
timeout / connection error / 5xx. 4xx still raises.
timeout / connection error / 5xx / 429 with `Retry-After`. Other 4xx
still raises: a bad key or request is a bug to fix, and a 429 without
`Retry-After` is the monthly quota running out, which needs a decision
(top up or subscribe), not a silent month of fallbacks. A rate limit is
transient load and should degrade like an outage instead.
"""
try:
return self._post(path, json=payload, timeout=self._verify_timeout)
except httpx.TransportError as exc:
self._warn_degraded(path, exc)
except CacheVerifierError as exc:
if exc.status_code < 500:
rate_limited = exc.status_code == 429 and exc.retry_after is not None
if exc.status_code < 500 and not rate_limited:
raise
self._warn_degraded(path, exc)
return None
Expand Down Expand Up @@ -365,7 +374,7 @@ def _unwrap(resp: httpx.Response) -> Any:
detail = body["detail"] if isinstance(body, dict) and "detail" in body else resp.text
except ValueError:
detail = resp.text
raise CacheVerifierError(resp.status_code, str(detail))
raise CacheVerifierError(resp.status_code, str(detail), resp.headers.get("retry-after"))


def _drop_none(**kwargs: Any) -> dict[str, Any]:
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ build-backend = "hatchling.build"

[project]
name = "cacheverifier"
version = "0.4.0"
version = "0.4.1"
description = "Python client for the hosted CacheVerifier semantic-cache verification API"
readme = "README.md"
requires-python = ">=3.9"
Expand Down
28 changes: 25 additions & 3 deletions tests/test_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,7 @@ def handler(request: httpx.Request) -> httpx.Response:

assert excinfo.value.status_code == 429
assert excinfo.value.detail == "free tier limit reached"
assert excinfo.value.retry_after is None


def test_error_response_without_json_body_falls_back_to_text():
Expand Down Expand Up @@ -204,14 +205,35 @@ def handler(request: httpx.Request) -> httpx.Response:
assert result.approved is False


def test_verify_still_raises_on_4xx():
def test_verify_fails_closed_on_rate_limit_429():
def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(429, json={"detail": "rate limited"})
return httpx.Response(429, headers={"Retry-After": "1"}, json={"detail": "rate limited"})

with make_client(handler) as cv:
result = cv.verify("q", "a")

assert result.degraded is True
assert result.approved is False


def test_verify_batch_degrades_on_rate_limit_429():
def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(429, headers={"Retry-After": "1"}, json={"detail": "rate limited"})

with make_client(handler) as cv:
results = cv.verify_batch([("q1", "a1"), ("q2", "a2")])

assert all(r.degraded and not r.approved for r in results)


def test_verify_still_raises_on_other_4xx():
def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(401, json={"detail": "invalid API key"})

with make_client(handler) as cv, pytest.raises(CacheVerifierError) as excinfo:
cv.verify("q", "a")

assert excinfo.value.status_code == 429
assert excinfo.value.status_code == 401


def test_verify_batch_degrades_every_pair_on_failure():
Expand Down
Loading