From 7eb4f7c6f14d21300c72ad7da5a90978f80473f4 Mon Sep 17 00:00:00 2001 From: xin Date: Fri, 30 Oct 2026 17:02:17 +0800 Subject: [PATCH] 0.6.0: LocalVerifier scores with your fine-tuned model locally; verify() sends similarity and cached_query; healthcheck truncates like the service --- CHANGELOG.md | 20 + README.md | 40 +- cacheverifier/__init__.py | 13 +- cacheverifier/_healthcheck/_finetune.py | 41 +- cacheverifier/_inference.py | 162 ++++++++ cacheverifier/client.py | 51 ++- cacheverifier/local.py | 502 ++++++++++++++++++++++++ pyproject.toml | 10 +- tests/test_local.py | 251 ++++++++++++ 9 files changed, 1046 insertions(+), 44 deletions(-) create mode 100644 cacheverifier/_inference.py create mode 100644 cacheverifier/local.py create mode 100644 tests/test_local.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c8dc353..7de96e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Changelog +## 0.6.0 + +New `LocalVerifier` (`pip install "cacheverifier[local]"`, Managed plans): +scores gray-zone hits on your own machines with the model CacheVerifier +fine-tuned for you, deciding exactly as the hosted `/v1/verify` would. It +downloads the live model, checks it against reference verdicts from the +service before using it, switches to newer models as they go live, and, while +auditing is on, uploads each decision without the texts so audits and risk +monitoring keep working. Falls back to the hosted API until a model is ready. + +`CacheVerifier.verify()` takes `similarity_score`, `cached_query` and +`runner_up_similarity_score` (all optional), which a fine-tuned model's joint +rules use; before this the client couldn't send them. `feedback()` takes +`cached_query` and `verification_id`. `VerifyResult` gains `decision_rule`, +`audit` and `verification_id`, which the API already returned. + +The offline Health Check now truncates long inputs exactly as the service +does since its verifier-core 0.7.0: a long query next to a short answer keeps +the room the answer doesn't need, instead of being cut to half the budget. + ## 0.5.0 New LangChain integration: `cacheverifier.integrations.langchain.VerifiedCache` diff --git a/README.md b/README.md index 47e6181..45f764c 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,8 @@ pip install cacheverifier pip install "cacheverifier[gptcache]" # with the offline Health Check (adds torch + sentence-transformers): pip install "cacheverifier[healthcheck]" +# to score with your fine-tuned model on your own machines (same deps): +pip install "cacheverifier[local]" ``` Requires Python 3.9+. The only runtime dependency is `httpx` — the extras above @@ -163,6 +165,40 @@ if job.get("result_model_version"): `cv.dry_run([...])` reports the same baseline-vs-tuned AUC on examples you pass directly, without writing anything or deploying a model. +## Local scoring (Managed plans) + +`LocalVerifier` runs the model CacheVerifier fine-tuned for you on your own +machines: no network round trip on your request path, and your query and +answer texts stay with you. Fine-tuning, drift monitoring, audits and +certification keep running on the service. + +```python +from cacheverifier import LocalVerifier + +cv = LocalVerifier(api_key="cv_...") # downloads your live model in the background +r = cv.verify(query, cached_answer, similarity_score=0.87, cached_query=cached_question) +if r.approved: + ... # serve from cache +else: + answer = call_llm(query) + if r.audit: # a randomized audit: report whether the cached answer was right + cv.feedback(query, cached_answer, was_correct=same_answer(answer, cached_answer), result=r) +``` + +- It decides exactly as the hosted `/v1/verify` would with that model: every download + carries reference verdicts from the service, and a model that doesn't reproduce them + locally is not used. +- When a newer model goes live (after a fine-tune or a scheduled recalibration), it is + downloaded, checked, and switched to without a restart. +- While auditing is on, each decision is uploaded as scores, verdicts and a keyed hash of + the pair, never the texts, so audits and `/v1/monitor/risk` work as with the hosted API. +- Until a model is ready, calls go to the hosted API (`fallback="hosted"`, the default) + or are rejected (`fallback="reject"`). Pass `wait=True` to block until the first + download finishes (a model is ~330MB). +- The audit draw happens in this client. The risk bound assumes it is independent of + whether the answer is right, which the client guarantees unless it is modified. +- Not supported yet: the online ACI threshold (local scoring uses the model's fixed one). + ## Local Health Check (offline) `cv.dry_run()` still uploads your examples to the API. If that's a blocker — a @@ -224,7 +260,7 @@ verdict | method | endpoint | |---|---| -| `verify(query, candidate_answer)` | `POST /v1/verify` | +| `verify(query, candidate_answer, similarity_score=..., cached_query=...)` | `POST /v1/verify` | | `verify_batch(pairs)` | `POST /v1/verify/batch` | | `feedback(...)` / `feedback_batch(items)` | `POST /v1/feedback` / `/batch` | | `finetune(...)` / `dry_run(examples, ...)` | `POST /v1/finetune/jobs` / `/dry-run` | @@ -233,6 +269,8 @@ verdict | `drift_status()` | `GET /v1/monitor/drift-status` | | `gray_zone_threshold()` | `GET /v1/monitor/gray-zone-threshold` | | `usage()` / `savings()` | `GET /v1/usage/status` / `/savings` | +| `LocalVerifier.verify(...)` / `verify_batch(...)` | local; decisions to `POST /v1/verify/decisions` | +| `LocalVerifier.refresh()` | `GET /v1/models/active` + `/v1/models/{id}/files/{name}` | Non-2xx responses raise `CacheVerifierError` (`.status_code`, `.detail`). diff --git a/cacheverifier/__init__.py b/cacheverifier/__init__.py index a3edd5b..1aede6c 100644 --- a/cacheverifier/__init__.py +++ b/cacheverifier/__init__.py @@ -17,6 +17,15 @@ """ from cacheverifier.client import CacheVerifier, CacheVerifierError, ShadowVerdict, VerifyResult +from cacheverifier.local import LocalVerifier, ModelCheckFailed -__version__ = "0.5.0" -__all__ = ["CacheVerifier", "CacheVerifierError", "ShadowVerdict", "VerifyResult", "__version__"] +__version__ = "0.6.0" +__all__ = [ + "CacheVerifier", + "CacheVerifierError", + "LocalVerifier", + "ModelCheckFailed", + "ShadowVerdict", + "VerifyResult", + "__version__", +] diff --git a/cacheverifier/_healthcheck/_finetune.py b/cacheverifier/_healthcheck/_finetune.py index d164097..31a35b8 100644 --- a/cacheverifier/_healthcheck/_finetune.py +++ b/cacheverifier/_healthcheck/_finetune.py @@ -25,6 +25,7 @@ from torch.utils.data import DataLoader from cacheverifier._healthcheck._base_models import DEFAULT_BASE_MODEL +from cacheverifier._inference import MAX_SEQUENCE_LENGTH, smart_truncate_pair # The four third-party imports above are from the `healthcheck` extra. This module is # only ever imported from inside `cacheverifier._healthcheck.cli.run()`, @@ -36,13 +37,11 @@ """`cross-encoder/nli-*` models' class order, used when the loaded model doesn't expose an `id2label` naming them.""" -MAX_SEQUENCE_LENGTH = 128 -"""Token cap applied identically at train and score time. Matches the -hosted service; BERT-style attention memory is O(n^2) in sequence length, -and this keeps a CPU run well-behaved.""" - -_CONTENT_TOKEN_BUDGET = MAX_SEQUENCE_LENGTH - 3 # [CLS] + 2x [SEP] -_MIN_QUERY_TOKENS = 32 +# Token cap and truncation shared with local scoring, so a Health Check +# trains with exactly what the service and `LocalVerifier` score with. Before +# 0.6.0 this module kept its own copy of the truncation, which had fallen +# behind the service's 0.7.0 change (a long query next to a short answer +# was cut to half the budget). TRAIN_BATCH_SIZE = 4 TRAIN_EPOCHS = 3 @@ -102,34 +101,6 @@ def auc_delta(self) -> float: return self.auc_tuned - self.auc_baseline -def _head_tail_truncate(ids: list[int], budget: int, head_ratio: float = 0.6) -> list[int]: - if len(ids) <= budget: - return ids - head_n = int(budget * head_ratio) - tail_n = budget - head_n - return ids[:head_n] + (ids[-tail_n:] if tail_n > 0 else []) - - -def smart_truncate_pair(query: str, candidate_answer: str, tokenizer: Any) -> tuple[str, str]: - """Tokenizer-aware head+tail truncation, applied identically at train - and score time (a model trained on one truncation policy and scored - with another sees a different token window than it was optimized - against). A no-op whenever the pair already fits.""" - query_ids = tokenizer.encode(query, add_special_tokens=False) - answer_ids = tokenizer.encode(candidate_answer, add_special_tokens=False) - if len(query_ids) + len(answer_ids) <= _CONTENT_TOKEN_BUDGET: - return query, candidate_answer - - query_budget = min(len(query_ids), max(_MIN_QUERY_TOKENS, _CONTENT_TOKEN_BUDGET // 2)) - answer_budget = _CONTENT_TOKEN_BUDGET - query_budget - query_ids = _head_tail_truncate(query_ids, query_budget) - answer_ids = _head_tail_truncate(answer_ids, answer_budget) - return ( - tokenizer.decode(query_ids, skip_special_tokens=True), - tokenizer.decode(answer_ids, skip_special_tokens=True), - ) - - def roc_auc(scores: np.ndarray, labels: np.ndarray) -> float: """Rank-based AUC, no sklearn dependency. NaN if either class is absent.""" order = np.argsort(scores) diff --git a/cacheverifier/_inference.py b/cacheverifier/_inference.py new file mode 100644 index 0000000..f9ffd59 --- /dev/null +++ b/cacheverifier/_inference.py @@ -0,0 +1,162 @@ +"""The scoring and decision steps the hosted service applies to one +(query, candidate_answer) pair, for running a downloaded model locally. + +A port of the hosted service's inference path: input composition with the +`cached_query` diff, tokenizer-aware truncation, and the threshold / joint +decision rules. Training, certification and monitoring stay on the service. +Pure Python (the tokenizer is passed in), so it imports without the `local` +extra. + +Kept in step with the service by two checks: the service's own test suite +compares these functions with its implementation, and every model download +carries reference verdicts that `LocalVerifier` reproduces before it decides +anything (see `cacheverifier.local`). +""" + +from __future__ import annotations + +import difflib +import math +from dataclasses import dataclass +from typing import Any + +MAX_SEQUENCE_LENGTH = 128 +"""Token cap applied at train and score time, as on the service.""" + +CONTENT_TOKEN_BUDGET = MAX_SEQUENCE_LENGTH - 3 # [CLS] + 2x [SEP] +MIN_QUERY_TOKENS = 32 +QUERY_DIFF_MAX_WORDS = 12 + + +def query_diff_summary(current_query: str, cached_query: str) -> str: + """Word-level diff of the cache entry's own query against the request's, + e.g. `removed: pause | added: cancel`.""" + old, new = cached_query.split(), current_query.split() + sm = difflib.SequenceMatcher(a=old, b=new, autojunk=False) + removed: list[str] = [] + added: list[str] = [] + for tag, i1, i2, j1, j2 in sm.get_opcodes(): + if tag in ("replace", "delete"): + removed.extend(old[i1:i2]) + if tag in ("replace", "insert"): + added.extend(new[j1:j2]) + parts = [] + if removed: + parts.append("removed: " + " ".join(removed[:QUERY_DIFF_MAX_WORDS])) + if added: + parts.append("added: " + " ".join(added[:QUERY_DIFF_MAX_WORDS])) + return " | ".join(parts) if parts else "no change" + + +def compose_query_input(current_query: str, cached_query: str | None) -> str: + if not cached_query: + return current_query + return f"{current_query} [qdiff] {query_diff_summary(current_query, cached_query)}" + + +def _head_tail_truncate(ids: list[int], budget: int, head_ratio: float = 0.6) -> list[int]: + if len(ids) <= budget: + return ids + head_n = int(budget * head_ratio) + tail_n = budget - head_n + return ids[:head_n] + (ids[-tail_n:] if tail_n > 0 else []) + + +def smart_truncate_pair(query: str, candidate_answer: str, tokenizer: Any) -> tuple[str, str]: + """Tokenizer-aware head+tail truncation, a no-op whenever the pair fits. + The answer gets at most half the budget while the query wants more; the + query gets whatever the answer doesn't need.""" + query_ids = tokenizer.encode(query, add_special_tokens=False) + answer_ids = tokenizer.encode(candidate_answer, add_special_tokens=False) + if len(query_ids) + len(answer_ids) <= CONTENT_TOKEN_BUDGET: + return query, candidate_answer + half = CONTENT_TOKEN_BUDGET // 2 + query_budget = min(len(query_ids), max(MIN_QUERY_TOKENS, half, CONTENT_TOKEN_BUDGET - len(answer_ids))) + answer_budget = CONTENT_TOKEN_BUDGET - query_budget + query_ids = _head_tail_truncate(query_ids, query_budget) + answer_ids = _head_tail_truncate(answer_ids, answer_budget) + return ( + tokenizer.decode(query_ids, skip_special_tokens=True), + tokenizer.decode(answer_ids, skip_special_tokens=True), + ) + + +def _sigmoid(x: float) -> float: + if x >= 0.0: + return 1.0 / (1.0 + math.exp(-x)) + e = math.exp(x) + return e / (1.0 + e) + + +@dataclass(frozen=True) +class JointRule: + """A fitted joint decision rule, as the service stores it: approve iff + the verifier score reaches `verifier_floor` (when set) and the fused + probability over the standardized similarity, verifier score (and, with + `uses_gap`, the gap to the runner-up match) reaches `threshold`.""" + + coef_similarity: float + coef_verifier: float + intercept: float + similarity_mean: float + similarity_std: float + verifier_mean: float + verifier_std: float + threshold: float + coef_gap: float = 0.0 + gap_mean: float = 0.0 + gap_std: float = 1.0 + uses_gap: bool = False + verifier_floor: float | None = None + + def approves( + self, verifier_score: float, similarity_score: float, runner_up_similarity_score: float | None = None + ) -> bool: + if self.verifier_floor is not None and verifier_score < self.verifier_floor: + return False + z_sim = (similarity_score - self.similarity_mean) / self.similarity_std + z_ver = (verifier_score - self.verifier_mean) / self.verifier_std + logit = self.coef_similarity * z_sim + self.coef_verifier * z_ver + self.intercept + if self.uses_gap: + if runner_up_similarity_score is None: + raise ValueError("this joint rule uses the similarity gap; runner_up_similarity_score is required") + z_gap = ((similarity_score - runner_up_similarity_score) - self.gap_mean) / self.gap_std + logit += self.coef_gap * z_gap + return _sigmoid(logit) >= self.threshold + + @classmethod + def from_dict(cls, d: dict[str, Any]) -> JointRule: + floor = d.get("verifier_floor") + return cls( + coef_similarity=float(d["coef_similarity"]), + coef_verifier=float(d["coef_verifier"]), + intercept=float(d["intercept"]), + similarity_mean=float(d["similarity_mean"]), + similarity_std=float(d["similarity_std"]), + verifier_mean=float(d["verifier_mean"]), + verifier_std=float(d["verifier_std"]), + threshold=float(d["threshold"]), + coef_gap=float(d.get("coef_gap", 0.0)), + gap_mean=float(d.get("gap_mean", 0.0)), + gap_std=float(d.get("gap_std", 1.0)), + uses_gap=bool(d.get("uses_gap", False)), + verifier_floor=float(floor) if floor is not None else None, + ) + + +def decide( + score: float, + threshold: float, + joint_rule: JointRule | None, + joint_rule_gap: JointRule | None, + similarity_score: float | None, + runner_up_similarity_score: float | None, +) -> tuple[bool, str]: + """(approved, decision_rule), with the service's precedence: the gap + rule when both similarity scores are given, then the 2-feature rule + when the similarity is, else the plain threshold.""" + if joint_rule_gap is not None and similarity_score is not None and runner_up_similarity_score is not None: + return joint_rule_gap.approves(score, similarity_score, runner_up_similarity_score), "joint_gap" + if joint_rule is not None and similarity_score is not None: + return joint_rule.approves(score, similarity_score), "joint" + return score >= threshold, "threshold" diff --git a/cacheverifier/client.py b/cacheverifier/client.py index 890b154..9d8e95b 100644 --- a/cacheverifier/client.py +++ b/cacheverifier/client.py @@ -105,6 +105,14 @@ class VerifyResult: - `shadow`: on a cold-start call, what the shared stock model would have decided (a `ShadowVerdict`); None otherwise, and on servers that don't send it. + - `decision_rule`: `"threshold"`, `"joint"` / `"joint_gap"` (a fitted rule + that also used your similarity scores), `"aci"`, or `"cold_start"`. + - `audit`: this approval was drawn for a randomized audit, so `approved` + is False: call your LLM, serve its answer, and report whether the + cached answer matched with `feedback(..., verification_id=...)`. + - `verification_id`: set while auditing is on; pass it to `feedback()`. + From `LocalVerifier` it is None until the decision has been uploaded: + pass the result itself as `feedback(..., result=r)` instead. """ approved: bool @@ -115,6 +123,11 @@ class VerifyResult: degraded: bool = False notice: str | None = None shadow: ShadowVerdict | None = None + decision_rule: str | None = None + audit: bool = False + verification_id: int | None = None + local_decision: str | None = None + """`LocalVerifier` only: its handle for this decision until upload.""" @classmethod def _from_json(cls, d: dict[str, Any]) -> VerifyResult: @@ -127,6 +140,9 @@ def _from_json(cls, d: dict[str, Any]) -> VerifyResult: latency_ms=float(d["latency_ms"]), notice=d.get("notice"), shadow=ShadowVerdict._from_json(shadow) if shadow else None, + decision_rule=d.get("decision_rule"), + audit=bool(d.get("audit", False)), + verification_id=d.get("verification_id"), ) @@ -194,15 +210,36 @@ def __exit__(self, *_exc: object) -> None: # -- core: verify ---------------------------------------------------- - def verify(self, query: str, candidate_answer: str) -> VerifyResult: + def verify( + self, + query: str, + candidate_answer: str, + *, + similarity_score: float | None = None, + cached_query: str | None = None, + runner_up_similarity_score: float | None = None, + ) -> VerifyResult: """Approve or reject one gray-zone cache hit. `POST /v1/verify`. + The optional arguments are what your cache lookup already knows: + the hit's similarity score, the cached entry's own original query, + and the second-best match's similarity. A fine-tuned model uses them + when it learned a rule from them; they never hurt otherwise. + On a timeout / connection error / 5xx / rate-limited 429 this returns a synthetic `VerifyResult` (`degraded=True`, `approved` per `fail_open`) instead of raising -- the verifier being unreachable should degrade to a normal cache miss, not an exception on your request path. """ - data = self._verify_call("/v1/verify", {"query": query, "candidate_answer": candidate_answer}) + payload = {"query": query, "candidate_answer": candidate_answer} + payload.update( + _drop_none( + similarity_score=similarity_score, + cached_query=cached_query, + runner_up_similarity_score=runner_up_similarity_score, + ) + ) + data = self._verify_call("/v1/verify", payload) if data is None: return self._degraded_result() return VerifyResult._from_json(data) @@ -273,6 +310,8 @@ def feedback( was_correct: bool, *, similarity_score: float | None = None, + cached_query: str | None = None, + verification_id: int | None = None, stale: bool = False, idempotency_key: str | None = None, ) -> int: @@ -286,6 +325,9 @@ def feedback( - `similarity_score`: your cache backend's own score for this candidate, if you still have it. Only used by `gray_zone_threshold()`; safe to omit. + - `cached_query`: the cached entry's own original query, if you have it. + - `verification_id`: from the `VerifyResult` this feedback is about + (always for an `audit` result) -- it ties the label to that verdict. - `stale`: set instead of a bare `was_correct=False` when the answer is wrong ONLY because a fact changed (price, date, ...), not a semantic mismatch. Stale rows are excluded from training and drift. @@ -297,8 +339,9 @@ def feedback( "was_correct": was_correct, "stale": stale, } - if similarity_score is not None: - payload["similarity_score"] = similarity_score + payload.update( + _drop_none(similarity_score=similarity_score, cached_query=cached_query, verification_id=verification_id) + ) headers = {"Idempotency-Key": idempotency_key} if idempotency_key else None return int(self._post("/v1/feedback", json=payload, headers=headers)["id"]) diff --git a/cacheverifier/local.py b/cacheverifier/local.py new file mode 100644 index 0000000..1efd215 --- /dev/null +++ b/cacheverifier/local.py @@ -0,0 +1,502 @@ +"""Score gray-zone cache hits on your own machines with your fine-tuned model. + +`LocalVerifier` downloads the model CacheVerifier fine-tuned for your account +(Managed plans), decides locally with the same threshold and rules the hosted +`/v1/verify` would use, and keeps the hosted side of the loop running: + +- every few minutes it checks which model is live, and switches to a newer + one once it has downloaded and checked it; +- while auditing is on, it draws audits locally and uploads each decision + (scores, verdicts and a keyed hash of the pair; never the texts), so audits, + `/v1/monitor/risk` and feedback linking work as with the hosted API; +- feedback goes to the service as usual, for fine-tuning and drift monitoring. + + from cacheverifier.local import LocalVerifier + + cv = LocalVerifier(api_key="cv_...") + r = cv.verify(query, candidate_answer, similarity_score=0.87) + if r.approved: + ... # serve from cache + else: + ... # call your LLM; if r.audit, report: cv.feedback(query, candidate_answer, ok, result=r) + +Before a model is ready (first download, no fine-tune yet, or a model that +fails its check), calls go to the hosted API instead (`fallback="hosted"`, +counted against your verify quota) or are rejected (`fallback="reject"`). + +Needs the `local` extra: `pip install "cacheverifier[local]"`. +""" + +from __future__ import annotations + +import hashlib +import hmac +import logging +import os +import random +import shutil +import threading +import time +import uuid +from collections import OrderedDict +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + +import httpx + +from cacheverifier._inference import JointRule, compose_query_input, decide, smart_truncate_pair +from cacheverifier.client import ( + DEFAULT_BASE_URL, + VERIFY_UNAVAILABLE, + CacheVerifier, + CacheVerifierError, + VerifyResult, +) + +_log = logging.getLogger(__name__) + +SUPPORTED_BUNDLE_FORMAT = 1 +DEFAULT_REFRESH_INTERVAL = 600.0 +"""Seconds between checks for a newer live model.""" +DEFAULT_UPLOAD_INTERVAL = 30.0 +"""Seconds between decision uploads (sooner when a batch fills up).""" +UPLOAD_BATCH = 1000 +"""The service's per-request cap on `POST /v1/verify/decisions`.""" +MAX_PENDING_DECISIONS = 100_000 +"""Decisions kept while the service can't be reached; the oldest are dropped +(and logged) beyond this, so an outage can't grow memory without bound.""" +MAX_REMEMBERED_IDS = 200_000 +KEEP_MODEL_VERSIONS = 2 +"""Downloaded versions kept on disk: the live one and the one before it.""" +REFERENCE_SCORE_TOLERANCE = 1e-3 +"""Allowed gap between a local score and the service's on a reference pair +(relative above 1). CPU builds of the same model agree far closer than this; +a mismatch means different inputs or a different model, not rounding.""" + +__all__ = ["LocalVerifier", "ModelCheckFailed"] + + +class ModelCheckFailed(RuntimeError): + """A downloaded model didn't reproduce the service's reference verdicts.""" + + +def _default_cache_dir() -> Path: + base = os.environ.get("XDG_CACHE_HOME") or os.path.join(os.path.expanduser("~"), ".cache") + return Path(base) / "cacheverifier" / "models" + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +@dataclass +class _Model: + manifest: dict[str, Any] + encoder: Any + joint_rule: JointRule | None + joint_rule_gap: JointRule | None + lock: threading.Lock + + @property + def version_id(self) -> int: + return int(self.manifest["model_version_id"]) + + def scores(self, queries: Sequence[str], answers: Sequence[str], cached: Sequence[str | None]) -> list[float]: + use_cached = bool(self.manifest["uses_cached_query"]) + with self.lock: # a Hugging Face fast tokenizer is not thread-safe + pairs = [ + smart_truncate_pair(compose_query_input(q, c if use_cached else None), a, self.encoder.tokenizer) + for q, a, c in zip(queries, answers, cached) + ] + return [float(s) for s in self.encoder.predict(pairs, show_progress_bar=False)] + + def decide(self, score: float, sim: float | None, runner_up: float | None) -> tuple[bool, str]: + return decide(score, float(self.manifest["threshold"]), self.joint_rule, self.joint_rule_gap, sim, runner_up) + + +class LocalVerifier: + """See the module docstring. Thread-safe; create one per process and + reuse it. `close()` (or a `with` block) uploads what's pending.""" + + def __init__( + self, + api_key: str, + *, + base_url: str = DEFAULT_BASE_URL, + cache_dir: str | os.PathLike[str] | None = None, + fallback: str = "hosted", + refresh_interval: float = DEFAULT_REFRESH_INTERVAL, + upload_interval: float = DEFAULT_UPLOAD_INTERVAL, + wait: bool = False, + timeout: float = 60.0, + transport: httpx.BaseTransport | None = None, + background: bool = True, + ) -> None: + """- `fallback`: `"hosted"` sends calls to `/v1/verify` while no local + model is ready; `"reject"` returns a rejected, `degraded` result. + - `wait`: block until the first model is downloaded and checked (a + checkpoint is ~330MB) instead of starting on the fallback. + - `background`: run the refresh/upload thread. With False, call + `refresh()` and `flush()` yourself. + """ + if fallback not in ("hosted", "reject"): + raise ValueError('fallback must be "hosted" or "reject"') + self._fallback = fallback + self._hosted = CacheVerifier(api_key, base_url=base_url, timeout=timeout, transport=transport) + host = urlparse(base_url).netloc or "default" + self._cache_dir = Path(cache_dir) if cache_dir is not None else _default_cache_dir() / host + self._refresh_interval = refresh_interval + self._upload_interval = upload_interval + self._model: _Model | None = None + self._state_lock = threading.Lock() + self._refresh_lock = threading.Lock() + self._pending: list[tuple[str, int, dict[str, Any]]] = [] + self._ids: OrderedDict[str, int] = OrderedDict() + self._stop = threading.Event() + self._wake = threading.Event() + self._thread: threading.Thread | None = None + if wait: + self.refresh() + if background: + self._thread = threading.Thread(target=self._run, name="cacheverifier-local", daemon=True) + self._thread.start() + + # -- lifecycle ------------------------------------------------------- + + @property + def model_version(self) -> str | None: + """`"v"` of the model deciding locally, or None while on the fallback.""" + model = self._model + return f"v{model.version_id}" if model is not None else None + + def close(self) -> None: + self._stop.set() + self._wake.set() + if self._thread is not None: + self._thread.join(timeout=30) + try: + self.flush() + finally: + self._hosted.close() + + def __enter__(self) -> LocalVerifier: + return self + + def __exit__(self, *_exc: object) -> None: + self.close() + + def _run(self) -> None: + next_refresh = 0.0 if self._model is None else time.monotonic() + self._refresh_interval + while not self._stop.is_set(): + if time.monotonic() >= next_refresh: + try: + self.refresh() + except Exception as exc: # noqa: BLE001 -- keep the loop alive; the current model keeps deciding + _log.warning("cacheverifier: model refresh failed (%s); will retry", exc) + # Retry sooner while there's no local model at all. + next_refresh = time.monotonic() + (self._refresh_interval if self._model else 60.0) + try: + self.flush() + except Exception as exc: # noqa: BLE001 -- pending decisions stay queued for the next try + _log.warning("cacheverifier: decision upload failed (%s); will retry", exc) + self._wake.wait(self._upload_interval) + self._wake.clear() + + # -- model download, check, switch ----------------------------------- + + def refresh(self) -> bool: + """Switch to the live model if it's newer than the local one. Returns + whether a new model was loaded. Raises `ModelCheckFailed` when the + new model doesn't reproduce the reference verdicts (the old one, or + the fallback, keeps deciding).""" + with self._refresh_lock: + try: + manifest = self._hosted._get("/v1/models/active") + except CacheVerifierError as exc: + if exc.status_code in (402, 404): + _log.warning("cacheverifier: no model to run locally (%s): %s", exc.status_code, exc.detail) + return False + raise + current = self._model + if current is not None and current.version_id == manifest["model_version_id"]: + current.manifest.update(audit_rate=manifest["audit_rate"], pair_key=manifest["pair_key"]) + return False + if manifest.get("format") != SUPPORTED_BUNDLE_FORMAT: + raise ModelCheckFailed( + f"model format {manifest.get('format')} is newer than this client; upgrade cacheverifier" + ) + if manifest.get("aci_enabled"): + _log.warning( + "cacheverifier: ACI is on for this account, but local scoring decides with the model's fixed " + "threshold; turn ACI off or use the hosted API for the online threshold" + ) + path = self._download(manifest) + model = self._load(manifest, path) + self._check(model) + with self._state_lock: + self._model = model + self._prune(keep=path) + _log.info("cacheverifier: now deciding locally with v%s", model.version_id) + return True + + def _version_dir(self, manifest: dict[str, Any]) -> Path: + files_digest = hashlib.sha256("".join(f["sha256"] for f in manifest["files"]).encode()).hexdigest()[:12] + return self._cache_dir / f"v{manifest['model_version_id']}-{files_digest}" + + def _download(self, manifest: dict[str, Any]) -> Path: + target = self._version_dir(manifest) + target.mkdir(parents=True, exist_ok=True) + for f in manifest["files"]: + dest = target / f["name"] + if dest.exists() and dest.stat().st_size == f["size"] and _sha256(dest) == f["sha256"]: + continue + part = target / (f["name"] + ".part") + url = f"/v1/models/{manifest['model_version_id']}/files/{f['name']}" + for _attempt in range(3): # resume an interrupted transfer with a Range request + done = part.stat().st_size if part.exists() else 0 + if done >= f["size"]: + break + headers = {"Range": f"bytes={done}-"} if done else {} + with self._hosted._client.stream("GET", url, headers=headers, timeout=None) as resp: + if resp.status_code == 429: + raise CacheVerifierError(429, "model downloads busy", resp.headers.get("Retry-After")) + if resp.status_code not in (200, 206): + resp.read() + CacheVerifier._unwrap(resp) + mode = "ab" if resp.status_code == 206 else "wb" + with part.open(mode) as out: + for chunk in resp.iter_bytes(1 << 20): + out.write(chunk) + if part.stat().st_size != f["size"] or _sha256(part) != f["sha256"]: + part.unlink(missing_ok=True) + raise ModelCheckFailed(f"downloaded {f['name']} doesn't match its sha256; will retry") + part.replace(dest) + return target + + @staticmethod + def _load(manifest: dict[str, Any], path: Path) -> _Model: + try: + from sentence_transformers import CrossEncoder + except ImportError as exc: # pragma: no cover - depends on the environment + raise ImportError('local scoring needs the local extra: pip install "cacheverifier[local]"') from exc + encoder = CrossEncoder(str(path), max_length=int(manifest["max_sequence_length"])) + rule = manifest.get("joint_rule") + rule_gap = manifest.get("joint_rule_gap") + return _Model( + manifest=dict(manifest), + encoder=encoder, + joint_rule=JointRule.from_dict(rule) if rule else None, + joint_rule_gap=JointRule.from_dict(rule_gap) if rule_gap else None, + lock=threading.Lock(), + ) + + @staticmethod + def _check(model: _Model) -> None: + refs = model.manifest["reference"] + scores = model.scores( + [r["query"] for r in refs], [r["candidate_answer"] for r in refs], [r.get("cached_query") for r in refs] + ) + for ref, score in zip(refs, scores): + expected = float(ref["score"]) + if abs(score - expected) > REFERENCE_SCORE_TOLERANCE * max(1.0, abs(expected)): + raise ModelCheckFailed( + f"v{model.version_id}: local score {score:.6f} != service {expected:.6f} on a reference pair; " + "not using this model (check that torch / sentence-transformers are supported versions)" + ) + # The decision logic, on the service's own score: catches a rule + # this client reads differently even where the scores agree. + verdict = model.decide(expected, ref.get("similarity_score"), ref.get("runner_up_similarity_score")) + if verdict != (ref["approved"], ref["decision_rule"]): + raise ModelCheckFailed( + f"v{model.version_id}: decision rule mismatch on a reference pair; upgrade cacheverifier" + ) + + def _prune(self, keep: Path) -> None: + dirs = sorted( + (d for d in self._cache_dir.iterdir() if d.is_dir() and d != keep), + key=lambda d: d.stat().st_mtime, + reverse=True, + ) + for old in dirs[KEEP_MODEL_VERSIONS - 1 :]: + shutil.rmtree(old, ignore_errors=True) + + # -- verify ------------------------------------------------------------ + + def verify( + self, + query: str, + candidate_answer: str, + *, + similarity_score: float | None = None, + cached_query: str | None = None, + runner_up_similarity_score: float | None = None, + ) -> VerifyResult: + """Same arguments and result as `CacheVerifier.verify`.""" + return self.verify_batch( + [(query, candidate_answer)], + similarity_scores=[similarity_score], + cached_queries=[cached_query], + runner_up_similarity_scores=[runner_up_similarity_score], + )[0] + + def verify_batch( + self, + pairs: Sequence[tuple[str, str]], + *, + similarity_scores: Sequence[float | None] | None = None, + cached_queries: Sequence[str | None] | None = None, + runner_up_similarity_scores: Sequence[float | None] | None = None, + ) -> list[VerifyResult]: + n = len(pairs) + sims = list(similarity_scores) if similarity_scores is not None else [None] * n + cached = list(cached_queries) if cached_queries is not None else [None] * n + runner_ups = list(runner_up_similarity_scores) if runner_up_similarity_scores is not None else [None] * n + model = self._model + if model is None: + return [self._fallback_verify(p, s, c, r) for p, s, c, r in zip(pairs, sims, cached, runner_ups)] + + start = time.perf_counter() + scores = model.scores([q for q, _ in pairs], [a for _, a in pairs], cached) + latency_ms = (time.perf_counter() - start) * 1000.0 / max(1, n) + audit_rate = float(model.manifest["audit_rate"]) + results = [] + for (query, answer), score, sim, runner_up in zip(pairs, scores, sims, runner_ups): + ok, rule = model.decide(score, sim, runner_up) + audited = ok and audit_rate > 0.0 and random.random() < audit_rate + token = self._queue(model, query, answer, score, rule, ok, audited, audit_rate) if audit_rate > 0 else None + results.append( + VerifyResult( + approved=ok and not audited, + score=score, + threshold=float(model.manifest["threshold"]), + model_version=f"v{model.version_id}", + latency_ms=latency_ms, + decision_rule=rule, + audit=audited, + local_decision=token, + ) + ) + return results + + def _fallback_verify( + self, pair: tuple[str, str], sim: float | None, cached: str | None, runner_up: float | None + ) -> VerifyResult: + if self._fallback == "hosted": + return self._hosted.verify( + pair[0], pair[1], similarity_score=sim, cached_query=cached, runner_up_similarity_score=runner_up + ) + return VerifyResult( + approved=False, score=0.0, threshold=0.0, model_version=VERIFY_UNAVAILABLE, latency_ms=0.0, degraded=True + ) + + # -- decision upload --------------------------------------------------- + + def _queue( + self, + model: _Model, + query: str, + answer: str, + score: float, + rule: str, + approved: bool, + audited: bool, + audit_rate: float, + ) -> str: + key = bytes.fromhex(model.manifest["pair_key"]) + digest = hmac.new(key, f"{query}\x00{answer}".encode(), hashlib.sha256).hexdigest() + token = uuid.uuid4().hex + decision = { + "score": score, + "decision_rule": rule, + "verifier_approved": approved, + "audited": audited, + "audit_probability": audit_rate, + "pair_digest": digest, + } + with self._state_lock: + self._pending.append((token, model.version_id, decision)) + if len(self._pending) > MAX_PENDING_DECISIONS: + dropped = len(self._pending) - MAX_PENDING_DECISIONS + del self._pending[:dropped] + _log.warning("cacheverifier: dropped %d un-uploaded decisions (service unreachable?)", dropped) + full = len(self._pending) >= UPLOAD_BATCH + if full: + self._wake.set() + return token + + def flush(self) -> None: + """Upload pending decisions now.""" + while True: + with self._state_lock: + if not self._pending: + return + version = self._pending[0][1] + batch = [p for p in self._pending[:UPLOAD_BATCH] if p[1] == version] + resp = self._hosted._post( + "/v1/verify/decisions", + json={"model_version_id": version, "decisions": [d for _, _, d in batch]}, + ) + with self._state_lock: + for (token, _, _), vid in zip(batch, resp["verification_ids"]): + self._ids[token] = int(vid) + while len(self._ids) > MAX_REMEMBERED_IDS: + self._ids.popitem(last=False) + sent = {token for token, _, _ in batch} + self._pending = [p for p in self._pending if p[0] not in sent] + + def verification_id(self, result: VerifyResult) -> int | None: + """The service's `verification_id` for a local result (uploads it + first if it's still pending). None when auditing was off.""" + if result.verification_id is not None or result.local_decision is None: + return result.verification_id + with self._state_lock: + vid = self._ids.get(result.local_decision) + if vid is None: + self.flush() + with self._state_lock: + vid = self._ids.get(result.local_decision) + return vid + + # -- feedback ---------------------------------------------------------- + + def feedback( + self, + query: str, + candidate_answer: str, + was_correct: bool, + *, + result: VerifyResult | None = None, + similarity_score: float | None = None, + cached_query: str | None = None, + stale: bool = False, + idempotency_key: str | None = None, + ) -> int: + """`CacheVerifier.feedback`, linked to `result` when given (always + pass it for an `audit` result: that's what makes the audit count).""" + verification_id = self.verification_id(result) if result is not None else None + return self._hosted.feedback( + query, + candidate_answer, + was_correct, + similarity_score=similarity_score, + cached_query=cached_query, + verification_id=verification_id, + stale=stale, + idempotency_key=idempotency_key, + ) + + @property + def hosted(self) -> CacheVerifier: + """The hosted client this uses, for fine-tuning, monitoring and usage calls.""" + return self._hosted + + def __repr__(self) -> str: + return f"LocalVerifier(model_version={self.model_version!r}, cache_dir={str(self._cache_dir)!r})" + diff --git a/pyproject.toml b/pyproject.toml index 3e94a1c..7744bcb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,8 +4,8 @@ build-backend = "hatchling.build" [project] name = "cacheverifier" -version = "0.5.0" -description = "Python client for the hosted CacheVerifier semantic-cache verification API" +version = "0.6.0" +description = "Python client for the CacheVerifier semantic-cache verification API, hosted or scored locally" readme = "README.md" requires-python = ">=3.9" license = { text = "MIT" } @@ -55,6 +55,12 @@ healthcheck = [ "torch>=2.2", "sentence-transformers>=3.0,<4", ] +# `LocalVerifier`: the same scoring stack as the healthcheck extra. +local = [ + "numpy>=1.26,<2", + "torch>=2.2", + "sentence-transformers>=3.0,<4", +] dev = ["pytest>=7", "mypy>=1.5", "ruff>=0.4", "langchain-core>=0.3"] [tool.hatch.build.targets.wheel] diff --git a/tests/test_local.py b/tests/test_local.py new file mode 100644 index 0000000..7d8efa6 --- /dev/null +++ b/tests/test_local.py @@ -0,0 +1,251 @@ +"""Offline tests for `LocalVerifier`: the service is an httpx.MockTransport +and the model a word-overlap stand-in, so nothing here downloads a real +checkpoint or needs torch.""" + +from __future__ import annotations + +import hashlib +import hmac +import json +import threading +from pathlib import Path +from typing import Any + +import httpx +import pytest + +from cacheverifier import _inference +from cacheverifier.local import LocalVerifier, ModelCheckFailed, _Model + +PAIR_KEY = "ab" * 32 +FILES = {"config.json": b'{"fake": true}', "model.safetensors": bytes(range(256)) * 400} + + +class FakeTokenizer: + def __init__(self) -> None: + self.vocab: dict[str, int] = {} + self.words: dict[int, str] = {} + + def encode(self, text: str, add_special_tokens: bool = False) -> list[int]: + ids = [] + for w in text.split(): + if w not in self.vocab: + self.vocab[w] = len(self.vocab) + self.words[self.vocab[w]] = w + ids.append(self.vocab[w]) + return ids + + def decode(self, ids: list[int], skip_special_tokens: bool = True) -> str: + return " ".join(self.words[i] for i in ids) + + +class FakeEncoder: + """Scores a pair by word overlap, minus a little per [qdiff] word.""" + + tokenizer = FakeTokenizer() + + def predict(self, pairs: list[tuple[str, str]], show_progress_bar: bool = False) -> list[float]: + out = [] + for a, b in pairs: + wa, wb = set(a.lower().split()), set(b.lower().split()) + out.append(4.0 * len(wa & wb) / max(1, len(wa | wb)) - 1.0 - 0.1 * a.count("[qdiff]")) + return out + + +REF_PAIRS = [ + {"query": "when will my parcel arrive", "candidate_answer": "your parcel will arrive in 3 days"}, + {"query": "where is the depot", "candidate_answer": "your parcel will arrive in 3 days"}, + { + "query": "how long does my parcel take", + "candidate_answer": "your parcel will arrive in 3 days", + "cached_query": "when will my parcel arrive", + "similarity_score": 0.86, + "runner_up_similarity_score": 0.7, + }, +] + + +def _manifest(version: int = 50, threshold: float = 0.0, audit_rate: float = 0.0, tamper: bool = False) -> dict: + enc = FakeEncoder() + refs = [] + for r in REF_PAIRS: + text_a = _inference.compose_query_input(r["query"], r.get("cached_query")) + score = enc.predict([(text_a, r["candidate_answer"])])[0] + (0.5 if tamper else 0.0) + ok, rule = _inference.decide(score, threshold, None, None, r.get("similarity_score"), None) + refs.append({**r, "score": score, "approved": ok, "decision_rule": rule}) + return { + "model_version_id": version, + "model_version": f"v{version}", + "format": 1, + "files": [{"name": n, "size": len(b), "sha256": hashlib.sha256(b).hexdigest()} for n, b in FILES.items()], + "score_kind": "logit", + "max_sequence_length": 128, + "uses_cached_query": True, + "threshold": threshold, + "joint_rule": None, + "joint_rule_gap": None, + "reference": refs, + "audit_rate": audit_rate, + "pair_key": PAIR_KEY, + "aci_enabled": False, + } + + +class FakeService: + def __init__(self, manifest: dict | None, status: int = 200) -> None: + self.manifest, self.status = manifest, status + self.calls: list[httpx.Request] = [] + self.decision_bodies: list[dict] = [] + self.feedback_bodies: list[dict] = [] + self.next_id = 1000 + + def __call__(self, request: httpx.Request) -> httpx.Response: + self.calls.append(request) + path = request.url.path + if path == "/v1/models/active": + if self.manifest is None: + return httpx.Response(self.status, json={"detail": "no model"}) + return httpx.Response(200, json=self.manifest) + if path.startswith("/v1/models/") and "/files/" in path: + data = FILES[path.rsplit("/", 1)[1]] + rng = request.headers.get("Range") + if rng: + start = int(rng.split("=")[1].rstrip("-")) + return httpx.Response(206, content=data[start:]) + return httpx.Response(200, content=data) + if path == "/v1/verify/decisions": + body = json.loads(request.content) + self.decision_bodies.append(body) + ids = list(range(self.next_id, self.next_id + len(body["decisions"]))) + self.next_id += len(ids) + return httpx.Response(200, json={"verification_ids": ids}) + if path == "/v1/feedback": + self.feedback_bodies.append(json.loads(request.content)) + return httpx.Response(200, json={"id": 1}) + if path == "/v1/verify": + return httpx.Response( + 200, json={"approved": True, "score": 9.0, "threshold": 0.0, "model_version": "v50", "latency_ms": 20} + ) + return httpx.Response(404, json={"detail": path}) + + +@pytest.fixture(autouse=True) +def _fake_model(monkeypatch): + def load(manifest: dict[str, Any], path: Path) -> _Model: + for f in manifest["files"]: # the real loader reads these; make sure they're there + assert (path / f["name"]).read_bytes() == FILES[f["name"]] + return _Model(manifest=dict(manifest), encoder=FakeEncoder(), joint_rule=None, joint_rule_gap=None, + lock=threading.Lock()) + + monkeypatch.setattr(LocalVerifier, "_load", staticmethod(load)) + + +def _local(tmp_path: Path, service: FakeService, **kwargs: Any) -> LocalVerifier: + return LocalVerifier("cv_test", cache_dir=tmp_path, transport=httpx.MockTransport(service), background=False, + **kwargs) + + +def _paths(service: FakeService) -> list[str]: + return [r.url.path for r in service.calls] + + +def test_downloads_checks_and_decides_locally(tmp_path): + service = FakeService(_manifest()) + cv = _local(tmp_path, service, wait=True) + assert cv.model_version == "v50" + r = cv.verify("when will my parcel arrive", "your parcel will arrive in 3 days") + assert r.approved and r.model_version == "v50" and r.decision_rule == "threshold" and not r.degraded + assert cv.verify("where is the depot", "your parcel will arrive in 3 days").approved is False + assert "/v1/verify" not in _paths(service) + # A second refresh with the same live version downloads nothing. + n = len(service.calls) + assert cv.refresh() is False + assert _paths(service)[n:] == ["/v1/models/active"] + + +def test_an_interrupted_download_resumes(tmp_path): + service = FakeService(_manifest()) + cv = _local(tmp_path, service) + target = cv._version_dir(service.manifest) + target.mkdir(parents=True) + (target / "model.safetensors.part").write_bytes(FILES["model.safetensors"][:5000]) + cv.refresh() + ranges = [r.headers.get("Range") for r in service.calls if r.url.path.endswith("model.safetensors")] + assert ranges == ["bytes=5000-"] + assert (target / "model.safetensors").read_bytes() == FILES["model.safetensors"] + + +def test_a_model_that_misses_the_reference_verdicts_is_not_used(tmp_path): + service = FakeService(_manifest(tamper=True)) + cv = _local(tmp_path, service) + with pytest.raises(ModelCheckFailed): + cv.refresh() + assert cv.model_version is None + r = cv.verify("q", "a") # falls back to the hosted API + assert r.score == 9.0 and "/v1/verify" in _paths(service) + + +def test_without_a_plan_or_fine_tune_it_stays_on_the_fallback(tmp_path): + service = FakeService(None, status=402) + cv = _local(tmp_path, service, fallback="reject") + assert cv.refresh() is False + r = cv.verify("q", "a") + assert r.degraded and not r.approved and "/v1/verify" not in _paths(service) + + +def test_audits_upload_without_texts_and_feedback_links(tmp_path, monkeypatch): + service = FakeService(_manifest(audit_rate=1.0)) + cv = _local(tmp_path, service, wait=True) + q, a = "when will my parcel arrive", "your parcel will arrive in 3 days" + r = cv.verify(q, a) + assert r.audit and not r.approved and r.verification_id is None and r.local_decision + + cv.feedback(q, a, True, result=r) # uploads the pending decision first, then links + [body] = service.decision_bodies + assert body["model_version_id"] == 50 + [d] = body["decisions"] + assert d["audited"] and d["verifier_approved"] and d["audit_probability"] == 1.0 + expected = hmac.new(bytes.fromhex(PAIR_KEY), f"{q}\x00{a}".encode(), hashlib.sha256).hexdigest() + assert d["pair_digest"] == expected + raw = service.calls[[c.url.path for c in service.calls].index("/v1/verify/decisions")].content.decode() + assert q not in raw and a not in raw + assert service.feedback_bodies[-1]["verification_id"] == 1000 + + +def test_no_uploads_while_auditing_is_off(tmp_path): + service = FakeService(_manifest(audit_rate=0.0)) + cv = _local(tmp_path, service, wait=True) + r = cv.verify("when will my parcel arrive", "your parcel will arrive in 3 days") + assert r.local_decision is None + cv.flush() + cv.feedback("when will my parcel arrive", "x", False, result=r) + assert service.decision_bodies == [] and "verification_id" not in service.feedback_bodies[-1] + + +def test_a_newer_live_version_replaces_the_old_and_old_downloads_are_pruned(tmp_path): + service = FakeService(_manifest(version=50)) + cv = _local(tmp_path, service, wait=True) + for version in (51, 52): + service.manifest = _manifest(version=version, threshold=-0.5) + assert cv.refresh() is True + assert cv.model_version == f"v{version}" + kept = sorted(d.name.split("-")[0] for d in tmp_path.iterdir()) + assert kept == ["v51", "v52"] + + +def test_decide_follows_the_service_precedence(): + rule = _inference.JointRule(1.0, 1.0, 0.0, 0.5, 0.1, 0.0, 1.0, 0.5) + gap = _inference.JointRule(1.0, 1.0, 0.0, 0.5, 0.1, 0.0, 1.0, 0.5, coef_gap=5.0, uses_gap=True) + assert _inference.decide(0.2, 0.0, rule, gap, 0.9, 0.85)[1] == "joint_gap" + assert _inference.decide(0.2, 0.0, rule, gap, 0.9, None)[1] == "joint" + assert _inference.decide(0.2, 0.0, rule, gap, None, None) == (True, "threshold") + floored = _inference.JointRule(1.0, 1.0, 0.0, 0.5, 0.1, 0.0, 1.0, 0.5, verifier_floor=1.0) + assert floored.approves(0.5, 0.99) is False + + +def test_truncation_gives_a_long_query_the_room_a_short_answer_leaves(): + tok = FakeTokenizer() + query = " ".join(f"q{i}" for i in range(200)) + q, a = _inference.smart_truncate_pair(query, "short answer", tok) + assert a == "short answer" + assert len(q.split()) == _inference.CONTENT_TOKEN_BUDGET - 2