Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,13 @@
# Changelog

## 0.7.0

`entries_to_refresh()` lists the cache entries your feedback reported stale,
or wrong more often than right, and `resolve_entry(key)` marks one refreshed
(`GET /v1/monitor/entries-to-refresh`, every plan). `feedback()` (on both
`CacheVerifier` and `LocalVerifier`) takes `cache_entry_id`, your own key for
the entry, which the list hands back so you can refresh or drop it directly.

## 0.6.0

New `LocalVerifier` (`pip install "cacheverifier[local]"`, Managed plans):
Expand Down
14 changes: 14 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,19 @@ set_llm_cache(
)
```

## Entries to refresh

Send your cache's own key with feedback, and the service tells you which
entries to fix: ones reported stale, or wrong more often than right.

```python
cv.feedback(query, cached_answer, was_correct=False, stale=True, cache_entry_id="faq:shipping")

for e in cv.entries_to_refresh():
my_cache.delete(e["cache_entry_id"]) # or regenerate the answer
cv.resolve_entry(e["key"])
```

## Fine-tuning

Once you have ~20+ feedback rows (the service found fine-tuning is often a net negative
Expand Down Expand Up @@ -267,6 +280,7 @@ verdict
| `get_finetune_job(id)` / `list_finetune_jobs()` | `GET /v1/finetune/jobs[/id]` |
| `activate_model_version(id)` | `POST /v1/finetune/model-versions/{id}/activate` |
| `drift_status()` | `GET /v1/monitor/drift-status` |
| `entries_to_refresh()` / `resolve_entry(key)` | `GET /v1/monitor/entries-to-refresh` / `POST .../{key}/resolve` |
| `gray_zone_threshold()` | `GET /v1/monitor/gray-zone-threshold` |
| `usage()` / `savings()` | `GET /v1/usage/status` / `/savings` |
| `LocalVerifier.verify(...)` / `verify_batch(...)` | local; decisions to `POST /v1/verify/decisions` |
Expand Down
2 changes: 1 addition & 1 deletion cacheverifier/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
from cacheverifier.client import CacheVerifier, CacheVerifierError, ShadowVerdict, VerifyResult
from cacheverifier.local import LocalVerifier, ModelCheckFailed

__version__ = "0.6.0"
__version__ = "0.7.0"
__all__ = [
"CacheVerifier",
"CacheVerifierError",
Expand Down
28 changes: 26 additions & 2 deletions cacheverifier/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -312,6 +312,7 @@ def feedback(
similarity_score: float | None = None,
cached_query: str | None = None,
verification_id: int | None = None,
cache_entry_id: str | None = None,
stale: bool = False,
idempotency_key: str | None = None,
) -> int:
Expand All @@ -328,6 +329,8 @@ def feedback(
- `cached_query`: the cached entry's own original query, if you have it.
- `verification_id`: from the `VerifyResult` this feedback is about
(always for an `audit` result) -- it ties the label to that verdict.
- `cache_entry_id`: your own key for the cache entry the answer came
from; `entries_to_refresh()` hands it back.
- `stale`: set instead of a bare `was_correct=False` when the answer
is wrong ONLY because a fact changed (price, date, ...), not a
semantic mismatch. Stale rows are excluded from training and drift.
Expand All @@ -340,15 +343,21 @@ def feedback(
"stale": stale,
}
payload.update(
_drop_none(similarity_score=similarity_score, cached_query=cached_query, verification_id=verification_id)
_drop_none(
similarity_score=similarity_score,
cached_query=cached_query,
verification_id=verification_id,
cache_entry_id=cache_entry_id,
)
)
headers = {"Idempotency-Key": idempotency_key} if idempotency_key else None
return int(self._post("/v1/feedback", json=payload, headers=headers)["id"])

def feedback_batch(self, items: Iterable[dict[str, Any]]) -> list[int]:
"""Bulk-upload feedback rows (max 500). Each item is a dict with
`query`, `candidate_answer`, `was_correct`, and optionally
`similarity_score` / `stale`. Returns the created row ids.
`similarity_score` / `cached_query` / `cache_entry_id` / `stale`.
Returns the created row ids.
`POST /v1/feedback/batch`.
"""
data = self._post("/v1/feedback/batch", json={"items": list(items)})
Expand Down Expand Up @@ -412,6 +421,21 @@ def gray_zone_threshold(self) -> dict[str, Any]:
"""
return self._get("/v1/monitor/gray-zone-threshold")

def entries_to_refresh(self) -> list[dict[str, Any]]:
"""Cache entries your feedback reported stale, or wrong more often
than right -- refresh or drop them, then `resolve_entry(e["key"])`.
Each has `key`, `cache_entry_id` (if you sent one), `reason`
(`"stale"` / `"often_wrong"`), the answer, report counts and sample
queries. `GET /v1/monitor/entries-to-refresh`.
"""
return list(self._get("/v1/monitor/entries-to-refresh")["entries"])

def resolve_entry(self, key: str) -> dict[str, Any]:
"""Mark an entry refreshed: earlier feedback stops counting toward it.
`POST /v1/monitor/entries-to-refresh/{key}/resolve`.
"""
return self._post(f"/v1/monitor/entries-to-refresh/{key}/resolve")

def usage(self) -> dict[str, Any]:
"""`GET /v1/usage/status`."""
return self._get("/v1/usage/status")
Expand Down
2 changes: 2 additions & 0 deletions cacheverifier/local.py
Original file line number Diff line number Diff line change
Expand Up @@ -475,6 +475,7 @@ def feedback(
result: VerifyResult | None = None,
similarity_score: float | None = None,
cached_query: str | None = None,
cache_entry_id: str | None = None,
stale: bool = False,
idempotency_key: str | None = None,
) -> int:
Expand All @@ -488,6 +489,7 @@ def feedback(
similarity_score=similarity_score,
cached_query=cached_query,
verification_id=verification_id,
cache_entry_id=cache_entry_id,
stale=stale,
idempotency_key=idempotency_key,
)
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ build-backend = "hatchling.build"

[project]
name = "cacheverifier"
version = "0.6.0"
version = "0.7.0"
description = "Python client for the CacheVerifier semantic-cache verification API, hosted or scored locally"
readme = "README.md"
requires-python = ">=3.9"
Expand Down
24 changes: 24 additions & 0 deletions tests/test_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,13 +133,37 @@ def handler(request: httpx.Request) -> httpx.Response:
def test_feedback_omits_optional_fields_when_unset():
def handler(request: httpx.Request) -> httpx.Response:
assert "similarity_score" not in json.loads(request.content)
assert "cache_entry_id" not in json.loads(request.content)
assert request.headers.get("Idempotency-Key") is None
return httpx.Response(200, json={"id": 1})

with make_client(handler) as cv:
assert cv.feedback("q", "a", was_correct=True) == 1


def test_entries_to_refresh_round_trip():
seen = []

def handler(request: httpx.Request) -> httpx.Response:
seen.append((request.method, request.url.path, request.content))
if request.url.path == "/v1/feedback":
return httpx.Response(200, json={"id": 7})
if request.method == "GET":
return httpx.Response(200, json={"entries": [{"key": "k" * 32, "cache_entry_id": "faq:shipping", "reason": "stale"}]})
return httpx.Response(200, json={"key": "k" * 32, "resolved_at": "2026-10-31T00:00:00Z"})

with make_client(handler) as cv:
cv.feedback("q", "a", was_correct=False, stale=True, cache_entry_id="faq:shipping")
[entry] = cv.entries_to_refresh()
assert entry["cache_entry_id"] == "faq:shipping"
assert cv.resolve_entry(entry["key"])["key"] == entry["key"]
assert json.loads(seen[0][2])["cache_entry_id"] == "faq:shipping"
assert [s[:2] for s in seen[1:]] == [
("GET", "/v1/monitor/entries-to-refresh"),
("POST", f"/v1/monitor/entries-to-refresh/{'k' * 32}/resolve"),
]


def test_finetune_passes_query_params():
def handler(request: httpx.Request) -> httpx.Response:
assert request.url.path == "/v1/finetune/jobs"
Expand Down
Loading