From 91f4ded8fd1eaf38dbb89b325da3b3b9a108d511 Mon Sep 17 00:00:00 2001 From: xin Date: Sat, 31 Oct 2026 14:55:07 +0800 Subject: [PATCH] 0.7.0: entries_to_refresh() / resolve_entry(); feedback takes cache_entry_id --- CHANGELOG.md | 8 ++++++++ README.md | 14 ++++++++++++++ cacheverifier/__init__.py | 2 +- cacheverifier/client.py | 28 ++++++++++++++++++++++++++-- cacheverifier/local.py | 2 ++ pyproject.toml | 2 +- tests/test_client.py | 24 ++++++++++++++++++++++++ 7 files changed, 76 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7de96e0..081ce54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 0.7.0 + +`entries_to_refresh()` lists the cache entries your feedback reported stale, +or wrong more often than right, and `resolve_entry(key)` marks one refreshed +(`GET /v1/monitor/entries-to-refresh`, every plan). `feedback()` (on both +`CacheVerifier` and `LocalVerifier`) takes `cache_entry_id`, your own key for +the entry, which the list hands back so you can refresh or drop it directly. + ## 0.6.0 New `LocalVerifier` (`pip install "cacheverifier[local]"`, Managed plans): diff --git a/README.md b/README.md index 45f764c..47348b0 100644 --- a/README.md +++ b/README.md @@ -146,6 +146,19 @@ set_llm_cache( ) ``` +## Entries to refresh + +Send your cache's own key with feedback, and the service tells you which +entries to fix: ones reported stale, or wrong more often than right. + +```python +cv.feedback(query, cached_answer, was_correct=False, stale=True, cache_entry_id="faq:shipping") + +for e in cv.entries_to_refresh(): + my_cache.delete(e["cache_entry_id"]) # or regenerate the answer + cv.resolve_entry(e["key"]) +``` + ## Fine-tuning Once you have ~20+ feedback rows (the service found fine-tuning is often a net negative @@ -267,6 +280,7 @@ verdict | `get_finetune_job(id)` / `list_finetune_jobs()` | `GET /v1/finetune/jobs[/id]` | | `activate_model_version(id)` | `POST /v1/finetune/model-versions/{id}/activate` | | `drift_status()` | `GET /v1/monitor/drift-status` | +| `entries_to_refresh()` / `resolve_entry(key)` | `GET /v1/monitor/entries-to-refresh` / `POST .../{key}/resolve` | | `gray_zone_threshold()` | `GET /v1/monitor/gray-zone-threshold` | | `usage()` / `savings()` | `GET /v1/usage/status` / `/savings` | | `LocalVerifier.verify(...)` / `verify_batch(...)` | local; decisions to `POST /v1/verify/decisions` | diff --git a/cacheverifier/__init__.py b/cacheverifier/__init__.py index 1aede6c..c272d98 100644 --- a/cacheverifier/__init__.py +++ b/cacheverifier/__init__.py @@ -19,7 +19,7 @@ from cacheverifier.client import CacheVerifier, CacheVerifierError, ShadowVerdict, VerifyResult from cacheverifier.local import LocalVerifier, ModelCheckFailed -__version__ = "0.6.0" +__version__ = "0.7.0" __all__ = [ "CacheVerifier", "CacheVerifierError", diff --git a/cacheverifier/client.py b/cacheverifier/client.py index 9d8e95b..ad1a122 100644 --- a/cacheverifier/client.py +++ b/cacheverifier/client.py @@ -312,6 +312,7 @@ def feedback( similarity_score: float | None = None, cached_query: str | None = None, verification_id: int | None = None, + cache_entry_id: str | None = None, stale: bool = False, idempotency_key: str | None = None, ) -> int: @@ -328,6 +329,8 @@ def feedback( - `cached_query`: the cached entry's own original query, if you have it. - `verification_id`: from the `VerifyResult` this feedback is about (always for an `audit` result) -- it ties the label to that verdict. + - `cache_entry_id`: your own key for the cache entry the answer came + from; `entries_to_refresh()` hands it back. - `stale`: set instead of a bare `was_correct=False` when the answer is wrong ONLY because a fact changed (price, date, ...), not a semantic mismatch. Stale rows are excluded from training and drift. @@ -340,7 +343,12 @@ def feedback( "stale": stale, } payload.update( - _drop_none(similarity_score=similarity_score, cached_query=cached_query, verification_id=verification_id) + _drop_none( + similarity_score=similarity_score, + cached_query=cached_query, + verification_id=verification_id, + cache_entry_id=cache_entry_id, + ) ) headers = {"Idempotency-Key": idempotency_key} if idempotency_key else None return int(self._post("/v1/feedback", json=payload, headers=headers)["id"]) @@ -348,7 +356,8 @@ def feedback( def feedback_batch(self, items: Iterable[dict[str, Any]]) -> list[int]: """Bulk-upload feedback rows (max 500). Each item is a dict with `query`, `candidate_answer`, `was_correct`, and optionally - `similarity_score` / `stale`. Returns the created row ids. + `similarity_score` / `cached_query` / `cache_entry_id` / `stale`. + Returns the created row ids. `POST /v1/feedback/batch`. """ data = self._post("/v1/feedback/batch", json={"items": list(items)}) @@ -412,6 +421,21 @@ def gray_zone_threshold(self) -> dict[str, Any]: """ return self._get("/v1/monitor/gray-zone-threshold") + def entries_to_refresh(self) -> list[dict[str, Any]]: + """Cache entries your feedback reported stale, or wrong more often + than right -- refresh or drop them, then `resolve_entry(e["key"])`. + Each has `key`, `cache_entry_id` (if you sent one), `reason` + (`"stale"` / `"often_wrong"`), the answer, report counts and sample + queries. `GET /v1/monitor/entries-to-refresh`. + """ + return list(self._get("/v1/monitor/entries-to-refresh")["entries"]) + + def resolve_entry(self, key: str) -> dict[str, Any]: + """Mark an entry refreshed: earlier feedback stops counting toward it. + `POST /v1/monitor/entries-to-refresh/{key}/resolve`. + """ + return self._post(f"/v1/monitor/entries-to-refresh/{key}/resolve") + def usage(self) -> dict[str, Any]: """`GET /v1/usage/status`.""" return self._get("/v1/usage/status") diff --git a/cacheverifier/local.py b/cacheverifier/local.py index 1efd215..e2b5cc4 100644 --- a/cacheverifier/local.py +++ b/cacheverifier/local.py @@ -475,6 +475,7 @@ def feedback( result: VerifyResult | None = None, similarity_score: float | None = None, cached_query: str | None = None, + cache_entry_id: str | None = None, stale: bool = False, idempotency_key: str | None = None, ) -> int: @@ -488,6 +489,7 @@ def feedback( similarity_score=similarity_score, cached_query=cached_query, verification_id=verification_id, + cache_entry_id=cache_entry_id, stale=stale, idempotency_key=idempotency_key, ) diff --git a/pyproject.toml b/pyproject.toml index 7744bcb..989c18a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "cacheverifier" -version = "0.6.0" +version = "0.7.0" description = "Python client for the CacheVerifier semantic-cache verification API, hosted or scored locally" readme = "README.md" requires-python = ">=3.9" diff --git a/tests/test_client.py b/tests/test_client.py index 57ca678..4194dc8 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -133,6 +133,7 @@ def handler(request: httpx.Request) -> httpx.Response: def test_feedback_omits_optional_fields_when_unset(): def handler(request: httpx.Request) -> httpx.Response: assert "similarity_score" not in json.loads(request.content) + assert "cache_entry_id" not in json.loads(request.content) assert request.headers.get("Idempotency-Key") is None return httpx.Response(200, json={"id": 1}) @@ -140,6 +141,29 @@ def handler(request: httpx.Request) -> httpx.Response: assert cv.feedback("q", "a", was_correct=True) == 1 +def test_entries_to_refresh_round_trip(): + seen = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append((request.method, request.url.path, request.content)) + if request.url.path == "/v1/feedback": + return httpx.Response(200, json={"id": 7}) + if request.method == "GET": + return httpx.Response(200, json={"entries": [{"key": "k" * 32, "cache_entry_id": "faq:shipping", "reason": "stale"}]}) + return httpx.Response(200, json={"key": "k" * 32, "resolved_at": "2026-10-31T00:00:00Z"}) + + with make_client(handler) as cv: + cv.feedback("q", "a", was_correct=False, stale=True, cache_entry_id="faq:shipping") + [entry] = cv.entries_to_refresh() + assert entry["cache_entry_id"] == "faq:shipping" + assert cv.resolve_entry(entry["key"])["key"] == entry["key"] + assert json.loads(seen[0][2])["cache_entry_id"] == "faq:shipping" + assert [s[:2] for s in seen[1:]] == [ + ("GET", "/v1/monitor/entries-to-refresh"), + ("POST", f"/v1/monitor/entries-to-refresh/{'k' * 32}/resolve"), + ] + + def test_finetune_passes_query_params(): def handler(request: httpx.Request) -> httpx.Response: assert request.url.path == "/v1/finetune/jobs"