From c91fae2b54cf5ff8832f16d65ad7f16affe51aac Mon Sep 17 00:00:00 2001 From: Abhyudaya Baiju Tharakan Date: Fri, 9 Oct 2026 07:53:52 +0530 Subject: [PATCH] feat(cache): add opt-in scoped semantic generation cache Behind SEMANTIC_CACHE_ENABLED and an explicit route or key opt-in in SEMANTIC_CACHE_POLICY_JSON, managed nonstream chat requests without tools can reuse a complete answer whose last user message has cosine similarity of at least 0.98. Every other part of the request, the model, sampling settings and policy revisions must match exactly, and a hit is refused when numbers, dates, negations or quoted text differ. Embeddings use an explicitly configured, priced model capped at 1,024 tokens and $0.001 per lookup. Entries are scoped to the principal, kept for 300 seconds, 256 per principal and 1 MiB each, in D1 (migration 0026) and the existing media bucket. Hits report semantic-hit provenance with no provider call; zero-content retention bypasses the cache. --- docs/semantic-generation-cache.md | 134 ++++++ .../0026_semantic_cache.sql | 23 ++ package.json | 2 +- routes/chat_cache.py | 42 +- services/cache_service.py | 12 + services/semantic_generation_cache.py | 380 ++++++++++++++++++ tests/fixtures/secret_firewall_coverage.json | 2 + tests/test_semantic_generation_cache.py | 259 ++++++++++++ .../test_semantic_generation_cache_worker.mjs | 195 +++++++++ worker/container-env.mjs | 2 + worker/d1-schema.mjs | 2 +- worker/semantic-cache-d1.mjs | 146 +++++++ worker/semantic-generation-cache.mjs | 210 ++++++++++ 13 files changed, 1405 insertions(+), 4 deletions(-) create mode 100644 docs/semantic-generation-cache.md create mode 100644 intelligence-migrations/0026_semantic_cache.sql create mode 100644 services/semantic_generation_cache.py create mode 100644 tests/test_semantic_generation_cache.py create mode 100644 tests/test_semantic_generation_cache_worker.mjs create mode 100644 worker/semantic-cache-d1.mjs create mode 100644 worker/semantic-generation-cache.mjs diff --git a/docs/semantic-generation-cache.md b/docs/semantic-generation-cache.md new file mode 100644 index 00000000..d204dc61 --- /dev/null +++ b/docs/semantic-generation-cache.md @@ -0,0 +1,134 @@ +# Scoped semantic generation cache + +Semantic reuse is off by default. `SEMANTIC_CACHE_ENABLED` defaults to `false`, and +`SEMANTIC_CACHE_POLICY_JSON` defaults to `{}`, which opts in no routes or keys. +Empty environment values mean these defaults. Invalid configuration disables the +feature and emits one warning without the supplied value. The exact response cache +keeps its existing opt-in and behavior when semantic reuse is disabled or unscoped. + +To disable semantic reuse: + +```text +SEMANTIC_CACHE_ENABLED=false +SEMANTIC_CACHE_POLICY_JSON={} +``` + +To opt in the authenticated Chat Completions route: + +```text +SEMANTIC_CACHE_ENABLED=true +SEMANTIC_CACHE_POLICY_JSON={"routes":["/v1/chat/completions"],"embedding_model":"openai:text-embedding-3-small","revision":"1","model_revision":"1","allow_tools":false,"allow_streams":false} +MODEL_PRICING_USD_PER_MILLION={"openai:text-embedding-3-small":{"input":0.02,"output":0}} +``` + +Pricing values are operator-supplied USD per million tokens, not a price catalog. +Preserve the other model entries in an existing pricing configuration. The embedding +model must be an explicit `provider:model` already configured in the gateway. No new +provider credentials, origins or bindings are introduced. Use `keys:["account-id"]` +to opt in an authenticated account ID, or its username when it has no ID. These are +server-authenticated identities, not raw API keys. Route and key opt-ins are combined +with OR. Worker routes use their full route name, such as +`/codex-easy/v1/chat/completions`, rather than the upstream path. + +Only `allow_tools:false` and `allow_streams:false` are supported. Either option set to +`true` is invalid configuration and disables semantic reuse. Tool definitions, tool +results, function calls, streams and requests for more than one choice are ineligible. +Only direct, explicitly selected Chat Completions models are supported; automatic and +cascade routes bypass semantic reuse. The last message must be a nonempty user text. +Dynamic state, provider-selection headers and ambiguous upstream credentials bypass +reuse. Existing exact caching remains independent for these requests. + +## Compatibility and fact checks + +Reuse requires exact matches for the authenticated principal and presented key, +provider, model, route, system prompt, earlier turns, every request field apart from +the final user text, sampling parameters, seed, response/output schema and policy +revisions. Grants, secret policy and retention participate in the compatibility +identity. Organization, project and protocol headers also partition responses. +Change `revision` after changing a compatibility policy, and `model_revision` when +the configured embedding model changes without changing its name. + +The final user text alone is embedded. Cosine similarity must be at least 0.98. +A matching vector is insufficient: the two texts must have the same ordered numbers, +dates, English month/weekday names and number words, negation words and exact quoted strings. Their protected facts are stored only +as a digest. Changing a number, a date, `not`, `never`, a negative contraction, or quoted +text prevents replay. Recognized volatile terms such as `today`, `now`, `current`, +`latest`, `weather` and `stock` bypass semantic reuse. These conservative lexical checks +cannot prove factual equivalence; opt in only routes whose answers tolerate paraphrases. + +## Spending and provenance + +Embedding input is limited to 1,024 estimated tokens (4,096 UTF-8 bytes). Oversized +inputs bypass lookup without truncating the user's question. A lookup is skipped if +embedding pricing is unknown, or if the conservative 1,024-token exposure including +a flat request price exceeds $0.001. Every dispatched embedding is separately recorded +through usage accounting, including misses and failed embeddings. Provider token counts +are used when present; otherwise accounting uses the conservative estimate. Embedding +permissions and budget admission apply before dispatch. No warmup generations run. + +A hit replays the complete stored JSON bytes with: + +```text +X-MultiLLM-Cache: semantic-hit +X-MultiLLM-Cache-Backend: semantic-d1-r2 +X-MultiLLM-Usage-Basis: cache-served +X-MultiLLM-Provider-Calls: 0 +Age: +``` + +The generation ledger records zero generation cost and cache provenance. The separate +embedding charge still applies. Stored provider token usage remains in the original +JSON response; the provenance headers explain that it describes the stored generation. +`Cache-Control:no-store` bypasses lookup and storage; `no-cache` and the existing +`X-MultiLLM-Cache:refresh` force a generation. `max-age` can further restrict replay age. + +## Storage and failures + +Apply the additive `0026_semantic_cache.sql` migration before enabling the feature. +D1 holds `semantic_generation_cache`, with scoped digests, a bounded vector, embedding +revision, expiry, integrity hash and body pointer. It scans at most 256 scoped entries +exactly; Vectorize is not used. The private transport returns the four best compatible +candidates to bound response size even for large vectors. A corrupt body among these +candidates can reduce reuse without changing normal dispatch. Complete successful single-choice JSON responses of at +most 1 MiB are stored in the existing `multillm_media` R2 bucket under `semantic-cache/`. +The exact cache's separate capacity policy does not provide semantic oldest-entry +replacement, so semantic bodies use their own prefix. + +Entries expire after 300 seconds. D1 transactions evict the oldest entries to keep at +most 256 per principal, across all partitions. Replaced or evicted bodies stay immutable +until expiry so concurrent readers remain safe. Bounded maintenance removes expired +metadata and R2 bodies without touching another prefix. Missing, corrupt, foreign or +expired bodies never replay. Request cancellation and policy changes prevent reuse +or storing an in-flight result. Responses with errors, truncation, tool calls or +`Cache-Control:no-store` are not stored. + +Zero-content retention bypasses semantic embedding, lookup and storage entirely. +Retention controls must be enabled for the gateway to honor a zero-retention request. +A missing enabled schema returns JSON HTTP 503 with `error:semantic_cache_schema_missing` +before embedding or generation. Ordinary cache or embedding failures fall through to +normal dispatch once; they never fabricate a success or grant a generation retry. + +## Runtime interfaces + +The Flask Chat Completions cache decorator handles semantic lookup and storage. Its +fixed private transport uses `/v1/state/semantic-cache` on `intelligence.internal`. +The private Worker state dispatcher must pass validated operations to +`handleSemanticCache`. This private handler is not a public route. + +Native Worker integrations use `prepareSemanticCache` after authentication, retention +and policy admission. Call `lookup` before generation and return any resulting response, +including a schema error. Call `store` only after classified successful finalization; +its required `canStore` callback must return `true` only for complete, classified success. +Missing classification prevents storage. A hit uses +`semanticCacheServedEvent` instead of provider generation telemetry. Supply named +`embed`, `embeddingAllowed`, `reserveEmbedding` and `accountEmbedding` collaborators +from the existing configured embedding transport, grant checks, budget reservations +and usage ledger. The embedding transport must honor its abort signal and must not +introduce retries or caller-selected origins. `accountEmbedding` settles the reservation +according to `submission_outcome` (`before-dispatch`, `response` or `unknown`); +unknown submissions retain their hold. All four collaborators are required; a missing +collaborator bypasses lookup. `reserveEmbedding` returns `false` to deny admission, +or a reservation (including `null` for an unbudgeted principal). Embedding and private +storage operations have two-second deadlines in the Worker. +Use `cleanupSemanticCache` in scheduled maintenance. The Worker module does not mount +public routes or automatically select a provider transport. diff --git a/intelligence-migrations/0026_semantic_cache.sql b/intelligence-migrations/0026_semantic_cache.sql new file mode 100644 index 00000000..00a099e9 --- /dev/null +++ b/intelligence-migrations/0026_semantic_cache.sql @@ -0,0 +1,23 @@ +-- Scoped semantic metadata; response content stays in the existing media bucket. +CREATE TABLE IF NOT EXISTS semantic_generation_cache ( + principal_hash TEXT NOT NULL, + entry_id TEXT NOT NULL, + partition_hash TEXT NOT NULL, + model_revision TEXT NOT NULL, + vector TEXT NOT NULL, + guard_hash TEXT NOT NULL, + created_at REAL NOT NULL, + expires_at REAL NOT NULL, + body_pointer TEXT NOT NULL, + body_bytes INTEGER NOT NULL CHECK (body_bytes BETWEEN 1 AND 1048576), + body_hash TEXT NOT NULL, + metadata TEXT NOT NULL, + PRIMARY KEY (principal_hash, entry_id), + CHECK (expires_at > created_at AND expires_at <= created_at + 300) +); +CREATE INDEX IF NOT EXISTS semantic_generation_cache_partition + ON semantic_generation_cache (principal_hash, partition_hash, model_revision, expires_at); +CREATE INDEX IF NOT EXISTS semantic_generation_cache_oldest + ON semantic_generation_cache (principal_hash, created_at, entry_id); +CREATE INDEX IF NOT EXISTS semantic_generation_cache_expiry + ON semantic_generation_cache (expires_at); diff --git a/package.json b/package.json index 4dc35c96..4fb55f9e 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "lint": "ruff check . --select E9,F63,F7,F82", "typecheck": "mypy --ignore-missing-imports --follow-imports=skip --disable-error-code import-untyped --disable-error-code var-annotated --disable-error-code arg-type --disable-error-code assignment --disable-error-code operator app.py routes services providers streaming", "test": "python -m pytest -q", - "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_managed_idempotency_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_gateway_alerts_worker.mjs tests/test_session_tiers_worker.mjs tests/test_usage_reservations_worker.mjs tests/test_shared_generation_cache_worker.mjs tests/test_deferred_tools_worker.mjs tests/test_generation_deadline_worker.mjs tests/test_wave4_worker.mjs", + "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_managed_idempotency_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_gateway_alerts_worker.mjs tests/test_session_tiers_worker.mjs tests/test_usage_reservations_worker.mjs tests/test_shared_generation_cache_worker.mjs tests/test_deferred_tools_worker.mjs tests/test_generation_deadline_worker.mjs tests/test_wave4_worker.mjs tests/test_semantic_generation_cache_worker.mjs", "db:migrate": "python scripts/validate_sqlite_schema.py --apply", "db:validate": "python scripts/validate_sqlite_schema.py", "security:secrets": "python scripts/check_static_secrets.py", diff --git a/routes/chat_cache.py b/routes/chat_cache.py index 1e4545c4..cb4e16b4 100644 --- a/routes/chat_cache.py +++ b/routes/chat_cache.py @@ -20,7 +20,7 @@ from flask import Response, g, request from route_helpers import request_api_key -from services import cache_policy +from services import cache_policy, semantic_generation_cache as semantic from services.retention_policy import request_policy from services.cache_service import ResponseCache, shared_response_cache from services import shared_generation_cache as shared_cache @@ -183,7 +183,7 @@ def cached_chat_completion(view): """Wrap an authenticated Chat Completions view; apply it below the auth decorator.""" @wraps(view) - def wrapper(*args, **kwargs): + def exact_wrapper(*args, **kwargs): retention = request_policy() mode, max_age = request_mode() settings = _settings() @@ -229,6 +229,44 @@ def wrapper(*args, **kwargs): _store_if_complete(response, key, settings, shared=shared, identity=identity) return _shared_headers(response, "miss") if use_shared else _mark(response, "miss") + @wraps(view) + def wrapper(*args, **kwargs): + if not semantic.settings().enabled: + return exact_wrapper(*args, **kwargs) + payload = request.get_json(silent=True) + try: + prepared = semantic.prepare_request(payload, _principal()) + except semantic.SemanticCacheSchemaMissing: + return Response(json.dumps({"error": "semantic_cache_schema_missing", + "message": "The semantic generation cache schema is missing. Apply its migration before enabling it."}), + status=503, content_type="application/json") + except Exception: + return exact_wrapper(*args, **kwargs) + if prepared is None: + return exact_wrapper(*args, **kwargs) + mode, max_age = request_mode() + if max_age is None: + match = _MAX_AGE.search(request.headers.get("Cache-Control", "").lower()) + max_age = int(match.group(1)) if match else None + if mode != "refresh" and "no-cache" not in request.headers.get("Cache-Control", "").lower(): + try: + entry = prepared.lookup() + except semantic.SemanticCacheSchemaMissing: + return Response(json.dumps({"error": "semantic_cache_schema_missing", + "message": "The semantic generation cache schema is missing. Apply its migration before enabling it."}), + status=503, content_type="application/json") + if entry is not None and (max_age is None or entry[2] <= max_age): + return semantic.semantic_hit(entry) + response = exact_wrapper(*args, **kwargs) + if isinstance(response, Response): + prepared.save(response, { + "content_type": "application/json", + "headers": {name: response.headers[name] for name in _STORED_HEADERS if name in response.headers}, + "provider": getattr(g, "multillm_provider", None), + "model": getattr(g, "multillm_model", None), + }) + return response + return wrapper diff --git a/services/cache_service.py b/services/cache_service.py index e5e20da0..1a1a446b 100644 --- a/services/cache_service.py +++ b/services/cache_service.py @@ -157,3 +157,15 @@ def shared_response_cache(): store = SharedGenerationCache() current_app.extensions["shared_generation_cache"] = store return store + + +def semantic_response_cache(): + """The fixed private semantic store, overridable by an application collaborator.""" + from flask import current_app + from services.semantic_generation_cache import SemanticGenerationCache + + store = current_app.extensions.get("semantic_generation_cache") + if store is None: + store = SemanticGenerationCache() + current_app.extensions["semantic_generation_cache"] = store + return store diff --git a/services/semantic_generation_cache.py b/services/semantic_generation_cache.py new file mode 100644 index 00000000..65c5fac2 --- /dev/null +++ b/services/semantic_generation_cache.py @@ -0,0 +1,380 @@ +"""Explicitly scoped semantic reuse with bounded paid embeddings and private storage.""" +from __future__ import annotations + +import base64 +import hashlib +import json +import logging +import math +import os +import queue +import re +import threading +import time +from dataclasses import dataclass +from functools import lru_cache +from typing import Any, Callable + +import requests +from requests.adapters import HTTPAdapter +from flask import Response, after_this_request, current_app, g, request + +from services.cost_service import CostService +from services import cache_policy, shared_generation_cache as exact +from services.retention_policy import request_policy + +MAX_TOKENS = 1024 +MAX_COST = 0.001 +MAX_RPC_BYTES = 2 * 1024 * 1024 +ENDPOINT = "http://intelligence.internal/v1/state/semantic-cache" +_SLOTS = threading.BoundedSemaphore(4) +_MODEL = re.compile(r"[a-z][a-z0-9-]*:[A-Za-z0-9@][A-Za-z0-9._:/+@-]{0,255}\Z") +_VOLATILE = re.compile(r"\b(?:today|tomorrow|yesterday|now|current|latest|live|weather|stock|real.?time)\b", re.I) +_FACTS = re.compile(r"[-+]?\d+(?:[.,:/-]\d+)*|\b(?:no|not|never|without|cannot|neither|nor|nothing|nobody|none)\b|\b\w+n['’]t\b", re.I) +_DATE_NUMBERS = re.compile(r"\b(?:jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|jun(?:e)?|jul(?:y)?|aug(?:ust)?|sep(?:t(?:ember)?)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?|mon(?:day)?|tue(?:s(?:day)?)?|wed(?:nesday)?|thu(?:rs(?:day)?)?|fri(?:day)?|sat(?:urday)?|sun(?:day)?|zero|one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve|thirteen|fourteen|fifteen|sixteen|seventeen|eighteen|nineteen|twenty|thirty|forty|fifty|sixty|seventy|eighty|ninety|hundred|thousand|million|billion|trillion|first|second|third|fourth|fifth|sixth|seventh|eighth|ninth|tenth)\b", re.I) +_QUOTES = re.compile(r'''"[^"\n]*"|(? 0) + + +def cosine(one, two): + if not vector_valid(one) or not vector_valid(two) or len(one) != len(two): + return -1 + return sum(a * b for a, b in zip(one, two)) / math.sqrt(sum(a * a for a in one)) / math.sqrt(sum(b * b for b in two)) + + +def partition(principal, path, payload, revisions, policy): + messages = [*payload["messages"][:-1], {**payload["messages"][-1], "content": None}] + invariant = {**payload, "messages": messages} + return {"principal_hash": digest(principal), + "partition_hash": digest(canonical([path, invariant, revisions, policy])), + "model_revision": digest(canonical([policy.get("embedding_model"), policy.get("model_revision", "1")]))} + + +class SemanticCacheSchemaMissing(Exception): + """Deployment has not provided the required durable schema.""" + + +def _exchange(encoded, results, stopped, deadline): + result = None + try: + with requests.Session() as session: + session.trust_env = False + session.mount("http://", HTTPAdapter(max_retries=0)) + with session.post(ENDPOINT, data=encoded, headers={"Content-Type": "application/json", "Accept-Encoding": "identity"}, + timeout=(1, 2), allow_redirects=False, stream=True) as response: + data = bytearray() + if response.headers.get("Content-Encoding", "identity") != "identity": + raise ValueError("cache_transport_unavailable") + for chunk in response.iter_content(chunk_size=4096): + if stopped.is_set() or time.monotonic() >= deadline or len(data) + len(chunk) > MAX_RPC_BYTES: + raise ValueError("cache_transport_unavailable") + data.extend(chunk) + value = json.loads(data) + if response.status_code == 503 and value.get("error") == "semantic_cache_schema_missing": + result = SemanticCacheSchemaMissing() + elif response.status_code == 200 and value.get("version") == 1: + result = value + except Exception: + pass + finally: + results.put_nowait(result) + _SLOTS.release() + + +def private_transport(value): + encoded = canonical({**value, "version": 1}).encode() + if len(encoded) > MAX_RPC_BYTES or not _SLOTS.acquire(blocking=False): + return None + results, stopped = queue.Queue(maxsize=1), threading.Event() + deadline = time.monotonic() + 3 + worker = threading.Thread(target=_exchange, args=(encoded, results, stopped, deadline), daemon=True) + try: + worker.start() + except Exception: + _SLOTS.release() + return None + try: + result = results.get(timeout=3) + if isinstance(result, SemanticCacheSchemaMissing): + raise result + return result + except queue.Empty: + return None + finally: + stopped.set() + + +class SemanticGenerationCache: + """D1 metadata and scoped R2 bodies over one fixed private operation set.""" + def __init__(self, transport=None): + self.transport = transport or private_transport + + def call(self, operation, **values): + result = self.transport({"version": 1, "operation": operation, **values}) + if not isinstance(result, dict) or result.get("version") != 1 or "error" in result: + raise ValueError("semantic_cache_unavailable") + return result + + def ready(self): + if self.call("ready").get("ready") is not True: + raise ValueError("semantic_cache_unavailable") + + def scan(self, identity, vector, guard): + rows = self.call("scan", **identity, vector=vector, guard_hash=guard).get("rows") + if not isinstance(rows, list) or len(rows) > 256: + raise ValueError("semantic_cache_unavailable") + return rows + + def body(self, identity, row): + entry = self.call("body", **identity, entry_id=row["entry_id"]).get("entry") + if not isinstance(entry, dict) or not exact.valid_metadata(entry.get("metadata")): + return None + age, encoded = entry.get("age"), entry.get("body") + if type(age) not in (int, float) or not 0 <= age < 300 or not isinstance(encoded, str) or len(encoded) > 1398104: + return None + body = base64.b64decode(encoded, validate=True) + if len(body) > exact.MAX_BODY_BYTES or not exact.complete_body(body): + return None + return body, entry["metadata"], age + + def put(self, identity, vector, guard, body, metadata): + return self.call("put", **identity, vector=vector, guard_hash=guard, + body=base64.b64encode(body).decode("ascii"), metadata=metadata).get("stored") is True + + +def embedding_allowed(model): + from services.key_controls import model_allowed + return model_allowed(getattr(g, "authenticated_user", {}) or {}, model) + + +def embedding_preflight(model, text): + from routes.media_audio import MediaRequest, OPERATIONS, validate_media_candidate + from services.auth_service import AuthService + media = MediaRequest(OPERATIONS["embeddings"], model, {"input": text, "encoding_format": "float"}) + validate_media_candidate(current_app, AuthService, media, model) + return media + + +def configured_embedding(model, text, media=None): + """Use the existing configured embedding route, without generation warmup.""" + from routes.media_audio import dispatch_media_candidate + from services.auth_service import AuthService + from services.metrics_service import MetricsService + from services.proxy_service import ProxyService + media = media or embedding_preflight(model, text) + response = dispatch_media_candidate(current_app, AuthService, MetricsService, ProxyService, media, model) + try: + body = bytearray() + for chunk in response.iter_encoded(): + body.extend(chunk) + if len(body) > 262144: + raise ValueError("embedding_response_too_large") + if response.status_code != 200: + raise ValueError("embedding_failed") + return json.loads(body) + finally: + response.close() + + +def embed_accounted(model, text, price): + from services import request_accounting as accounting + from services.prompt_cache_cost import CacheObservation + from services.usage_types import UsageObservation + embed = current_app.extensions.get("semantic_cache_embedding", configured_embedding) + media = embedding_preflight(model, text) if embed is configured_embedding else None + user = getattr(g, "authenticated_user", {}) or {} + context = accounting.UsageContext(kind="embeddings", models=[model], provider=model.split(":", 1)[0], + user=user, started=time.perf_counter(), start_ns=time.time_ns(), units=1, + trace=accounting.telemetry_export.trace_context(request.headers.get("traceparent"))) + context.path, context.method = "/v1/embeddings", "POST" + context.input_tokens, context.output_tokens = MAX_TOKENS, 0 + if accounting.budgeted(user): + decision = accounting.BudgetService.check_and_reserve(user, price) + if not decision.allowed: + return None + context.reservation = decision.reservation + usage, status = CacheObservation(MAX_TOKENS, 0, "estimated", ("request_estimate",)), 502 + try: + accounting.BudgetService.mark_dispatched(context.reservation) + result = embed(model, text, media) if media is not None else embed(model, text) + observed = result.get("usage") if isinstance(result, dict) else None + tokens = observed.get("prompt_tokens", observed.get("input_tokens")) if isinstance(observed, dict) else None + if type(tokens) is int and 0 <= tokens <= 2**53 - 1: + usage = UsageObservation(tokens, 0, "provider", ("provider",)) + vector = result["data"][0]["embedding"] if isinstance(result, dict) else result + status = 200 if vector_valid(vector) else 502 + return vector if vector_valid(vector) else None + finally: + accounting._record(context, status, usage, None) + + +@dataclass +class PreparedSemanticCache: + store: Any + identity: dict + vector: list + guard: str + current: Callable[[], dict | None] + + def lookup(self): + try: + rows = self.store.scan(self.identity, self.vector, self.guard) + candidates = [row for row in rows if row.get("guard_hash") == self.guard + and cosine(self.vector, row.get("vector")) >= 0.98] + candidates.sort(key=lambda row: cosine(self.vector, row["vector"]), reverse=True) + for row in candidates: + entry = self.store.body(self.identity, row) + if entry is not None and self.current() == self.identity: + return entry + except SemanticCacheSchemaMissing: + raise + except Exception: + _warn_once("storage_unavailable") + return None + + def save(self, response, metadata): + from services.managed_turn import cache_response_allowed + try: + if (response.status_code != 200 or response.is_streamed or response.direct_passthrough + or response.mimetype != "application/json" or not cache_response_allowed(response) + or "no-store" in response.headers.get("Cache-Control", "").lower()): + return + body = response.get_data() + if len(body) <= exact.MAX_BODY_BYTES and exact.complete_body(body) and self.current() == self.identity: + self.store.put(self.identity, self.vector, self.guard, body, metadata) + except Exception: + _warn_once("storage_unavailable") + + +def prepare_request(payload, principal): + config, retention = settings(), request_policy() + policy = config.policy or {} + user = getattr(g, "authenticated_user", {}) or {} + key_scope = str(user.get("id") or user.get("username") or "") + if (not config.enabled or not principal or request.method != "POST" or not retention.allows_content + or request.path not in policy.get("routes", []) and key_scope not in policy.get("keys", []) + or not eligible(payload) or "no-store" in request.headers.get("Cache-Control", "").lower()): + return None + model = policy["embedding_model"] + price = CostService.estimate(model, MAX_TOKENS, 0) + if price is None or not math.isfinite(price) or not 0 <= price <= MAX_COST or not embedding_allowed(model): + return None + def current(): + resolved = exact.shared_policy(payload) + if resolved is None or settings() != config or not embedding_allowed(model): + return None + return partition(principal, request.path, payload, + {"policy": cache_policy.policy_digest(resolved), "retention": retention.revision, + "retention_config": os.environ.get("CONTENT_RETENTION_POLICY_JSON", ""), + "retention_enabled": os.environ.get("CONTENT_RETENTION_ENABLED", ""), + "headers": {key: request.headers.get(key) for key in ("openai-organization", "openai-project", + "openai-beta", "anthropic-version", "anthropic-beta")}}, policy) + identity = current() + if identity is None: + return None + from services.cache_service import semantic_response_cache + store = semantic_response_cache() + try: + store.ready() + vector = embed_accounted(model, payload["messages"][-1]["content"], price) + if vector is None: + return None + return PreparedSemanticCache(store, identity, vector, guard_hash(payload["messages"][-1]["content"]), current) + except SemanticCacheSchemaMissing: + raise + except Exception: + _warn_once("storage_unavailable") + return None + + +def semantic_hit(entry): + """Accounting consumes the existing hit marker before Flask emits distinct provenance.""" + body, metadata, age = entry + response = Response(body, content_type="application/json") + for name, value in metadata["headers"].items(): + response.headers[name] = value + response.headers["X-MultiLLM-Cache"] = "hit" + response.headers["X-MultiLLM-Cache-Backend"] = "semantic-d1-r2" + response.headers["X-MultiLLM-Usage-Basis"] = "cache-served" + response.headers["X-MultiLLM-Provider-Calls"] = "0" + response.headers["Age"] = str(int(age)) + g.multillm_provider, g.multillm_model = metadata["provider"], metadata["model"] + g.multillm_route_decision = "semantic-cache-hit" + @after_this_request + def provenance(result): + result.headers["X-MultiLLM-Cache"] = "semantic-hit" + return result + return response diff --git a/tests/fixtures/secret_firewall_coverage.json b/tests/fixtures/secret_firewall_coverage.json index 3bc276f1..b784b58d 100644 --- a/tests/fixtures/secret_firewall_coverage.json +++ b/tests/fixtures/secret_firewall_coverage.json @@ -362,6 +362,8 @@ "worker/secret-digest.mjs", "worker/secret-firewall.mjs", "worker/secret-scan.mjs", + "worker/semantic-cache-d1.mjs", + "worker/semantic-generation-cache.mjs", "worker/shadow-eval-d1.mjs", "worker/shadow-eval-schedule.mjs", "worker/sse-heartbeat.mjs", diff --git a/tests/test_semantic_generation_cache.py b/tests/test_semantic_generation_cache.py new file mode 100644 index 00000000..082efe2b --- /dev/null +++ b/tests/test_semantic_generation_cache.py @@ -0,0 +1,259 @@ +"""Scoped semantic lookup and authenticated Chat Completions contracts.""" +import importlib +import json +import os +import sqlite3 +from pathlib import Path +from unittest.mock import patch + +import pytest +from flask import Flask, Response, g + +from tests.test_chat_cache import BODY, ADMIN, CACHED, chat_cache, completion +from tests.unified_api_test_case import UnifiedApiTestCase + + +def module(): + return importlib.import_module("services.semantic_generation_cache") + + +POLICY = {"routes": ["/v1/chat/completions"], "embedding_model": "openai:embed-test", "revision": "one"} +PRICES = json.dumps({"openai:embed-test": {"input": 0.1, "output": 0}}) +ANSWER = json.dumps({"choices": [{"message": {"content": "answer"}, "finish_reason": "stop"}]}).encode() + + +class Store: + def __init__(self): + self.rows = [] + self.ready_calls = 0 + self.missing = False + self.broken = False + + def ready(self): + self.ready_calls += 1 + if self.missing: + raise module().SemanticCacheSchemaMissing() + if self.broken: + raise RuntimeError("storage error") + + def scan(self, identity, vector, guard): + return [row for row in self.rows if row["identity"] == identity] + + def body(self, identity, row): + return row["body"], row["metadata"], 0 + + def put(self, identity, vector, guard, body, metadata): + self.rows.append(dict(identity=identity, vector=vector, guard_hash=guard, body=body, metadata=metadata)) + return True + + +@pytest.fixture +def env(monkeypatch): + monkeypatch.setenv("SEMANTIC_CACHE_ENABLED", "true") + monkeypatch.setenv("SEMANTIC_CACHE_POLICY_JSON", json.dumps(POLICY)) + monkeypatch.setenv("MODEL_PRICING_USD_PER_MILLION", PRICES) + return monkeypatch + + +def test_settings_defaults_empty_and_invalid_warn_once(caplog): + m = module() + m._warn_once.cache_clear() + for config in ({}, {"SEMANTIC_CACHE_ENABLED": "", "SEMANTIC_CACHE_POLICY_JSON": ""}): + assert not m.settings(config).enabled + assert m.settings({"SEMANTIC_CACHE_ENABLED": "true"}).policy == {} + for bad in ("broken", "[]", '{"allow_tools":true}', '{"allow_streams":true}'): + assert not m.settings({"SEMANTIC_CACHE_ENABLED": "true", "SEMANTIC_CACHE_POLICY_JSON": bad}).enabled + assert bad not in caplog.text + assert len(caplog.records) == 1 + + +@pytest.mark.parametrize("change", [{"stream": True}, {"n": 2}, {"tools": [{}], "tool_choice": "none"}, + {"functions": [{}]}, {"messages": [{"role": "tool", "content": "result"}, {"role": "user", "content": "question"}]}, + {"messages": [{"role": "user", "content": "latest weather today"}]}, + {"messages": [{"role": "user", "content": "x" * 4097}]}]) +def test_ineligible_requests(env, change): + assert not module().eligible({**BODY, **change}) + + +@pytest.mark.parametrize("a,b", [("buy 12 shares", "buy 13 shares"), ("on 2026-10-09", "on 2026-10-10"), + ("do it", "do not do it"), ('say "hello"', 'say "bye"'), ("use 'yes'", "use 'no'"), + ("can go", "can't go"), ("no need", "need"), ("October 9", "November 9"), + ("meet Monday", "meet Tuesday"), ("buy twelve shares", "buy thirteen shares")]) +def test_protected_facts(a, b): + assert module().guard_hash(a) != module().guard_hash(b) + + +def test_similarity_threshold_and_bad_vectors(): + m = module() + assert m.cosine([1, 0], [0.99, 0.01]) >= 0.98 + assert m.cosine([1, 0], [0.9, 0.44]) < 0.98 + for vector in ([], [0, 0], [float("nan"), 1], [True, 1], [1]): + assert m.cosine([1, 0], vector) == -1 + + +def test_full_partition_except_last_user_content(env): + m = module() + payload = {**BODY, "messages": [{"role": "system", "content": "rules"}, {"role": "user", "content": "explain caching"}]} + one = m.partition("owner", "/v1/chat/completions", payload, {"policy": "one"}, POLICY) + paraphrase = {**payload, "messages": [payload["messages"][0], {"role": "user", "content": "describe caching"}]} + assert one == m.partition("owner", "/v1/chat/completions", paraphrase, {"policy": "one"}, POLICY) + for field, value in (("temperature", 0.1), ("top_p", 0.5), ("seed", 7), ("model", "openai:other"), + ("response_format", {"type": "json_object"}), ("stop", ["done"])): + assert one != m.partition("owner", "/v1/chat/completions", {**payload, field: value}, {"policy": "one"}, POLICY) + assert one != m.partition("foreign", "/v1/chat/completions", payload, {"policy": "one"}, POLICY) + assert one != m.partition("owner", "/v1/chat/completions", payload, {"policy": "two"}, POLICY) + + +class SemanticRouteTests(UnifiedApiTestCase): + def setUp(self): + with patch("config.load_runtime_env"): + super().setUp() + os.environ.update(SEMANTIC_CACHE_ENABLED="true", SEMANTIC_CACHE_POLICY_JSON=json.dumps(POLICY), + MODEL_PRICING_USD_PER_MILLION=PRICES, GENERATION_CACHE_SHARED_ENABLED="false", + CONTENT_RETENTION_ENABLED="false", RESPONSE_CACHE_POLICY_REVISION="legacy") + self.store = Store() + self.embeddings = [] + self.app.extensions["semantic_generation_cache"] = self.store + self.app.extensions["semantic_cache_embedding"] = lambda model, text: self.embeddings.append((model, text)) or [1, 0] + chat_cache().clear() + + def post(self, text="explain caching", headers=ADMIN, **extra): + payload = {**BODY, "messages": [{"role": "user", "content": text}], **extra} + with patch.object(self.app_module.ProxyService, "make_request", return_value=completion("real answer")) as send: + result = self.client.post("/v1/chat/completions", json=payload, headers=headers) + return result, send.call_count + + def test_registered_hit_preserves_body_and_zero_generation_cost(self): + accounting = importlib.import_module("services.request_accounting") + rows = [] + with patch.object(accounting.usage_ledger.LEDGER, "record", side_effect=rows.append): + first, _ = self.post() + hit, calls = self.post("describe caching") + assert calls == 0 + assert first.data == hit.data + assert hit.headers["X-MultiLLM-Cache"] == "semantic-hit" + assert hit.headers["X-MultiLLM-Usage-Basis"] == "cache-served" + assert hit.headers["X-MultiLLM-Provider-Calls"] == "0" + assert rows[-1]["cost_usd"] == 0 and rows[-1]["cost_basis"] == "cache" + assert len(self.embeddings) == 2 + + def test_schema_missing_fails_before_embedding_and_provider(self): + self.store.missing = True + response, calls = self.post() + assert response.status_code == 503 and calls == 0 + assert response.get_json()["error"] == "semantic_cache_schema_missing" + assert self.embeddings == [] + + def test_outage_falls_through_once_and_no_synthetic_output(self): + self.store.broken = True + result, calls = self.post() + assert calls == 1 and result.get_json()["choices"][0]["message"]["content"] == "real answer" + assert self.embeddings == [] + + def test_protected_mismatch_and_incomplete_never_hit(self): + self.post("explain 12 caches") + assert self.post("describe 13 caches")[1] == 1 + self.store.rows.clear() + with patch.object(self.app_module.ProxyService, "make_request", return_value=completion(finish_reason="length")): + self.client.post("/v1/chat/completions", json={**BODY, "messages": [{"role": "user", "content": "explain caching"}]}, headers=ADMIN) + assert not self.store.rows + + def test_scope_price_retention_and_tools_skip_every_semantic_operation(self): + cases = [("SEMANTIC_CACHE_POLICY_JSON", "{}", {}), ("SEMANTIC_CACHE_ENABLED", "false", {}), + ("MODEL_PRICING_USD_PER_MILLION", "{}", {}), + ("MODEL_PRICING_USD_PER_MILLION", json.dumps({"openai:embed-test": {"input": 100, "output": 0}}), {}), + ("CONTENT_RETENTION_ENABLED", "true", {"X-MultiLLM-Retention": "zero"})] + for name, value, headers in cases: + with patch.dict(os.environ, {name: value}): + assert self.post(headers={**ADMIN, **headers})[1] == 1 + assert self.post(tools=[{}], tool_choice="none")[1] == 1 + assert self.store.ready_calls == 0 and self.embeddings == [] + + def test_disabled_exact_cache_bytes_and_headers_unchanged(self): + with patch.dict(os.environ, {"SEMANTIC_CACHE_ENABLED": ""}): + first, _ = self.post(headers=CACHED) + second, calls = self.post(headers=CACHED) + assert second.data == first.data and calls == 0 + assert second.headers["X-MultiLLM-Cache"] == "hit" and self.embeddings == [] + + def test_authentication_precedes_semantic_cache(self): + response, calls = self.post(headers={"Authorization": "Bearer invalid"}) + assert response.status_code == 401 and calls == 0 and not self.store.ready_calls + + def test_embedding_is_separately_charged_with_provider_or_estimated_usage(self): + accounting = importlib.import_module("services.request_accounting") + rows = [] + with patch.object(accounting.usage_ledger.LEDGER, "record", side_effect=rows.append): + self.post() + self.app.extensions["semantic_cache_embedding"] = lambda *_: {"data": [{"embedding": [1, 0]}], "usage": {"prompt_tokens": 4}} + self.post("describe caching") + embeddings = [row for row in rows if row["kind"] == "embeddings"] + assert [row["cost_basis"] for row in embeddings] == ["estimate", "usage"] + assert [row["input_tokens"] for row in embeddings] == [1024, 4] + assert embeddings[0]["cost_usd"] == pytest.approx(0.0001024) + assert embeddings[1]["cost_usd"] == pytest.approx(0.0000004) + + def test_security_headers_partition_and_max_age_without_exact_opt_in(self): + self.post(headers={**ADMIN, "OpenAI-Project": "one"}) + assert self.post("describe caching", headers={**ADMIN, "OpenAI-Project": "two"})[1] == 1 + with patch.object(self.store, "body", side_effect=lambda identity, row: (row["body"], row["metadata"], 10)): + assert self.post("describe caching", headers={**ADMIN, "OpenAI-Project": "one", "Cache-Control": "max-age=1"})[1] == 1 + + def test_budget_denial_and_embedding_failure_still_dispatch_once(self): + accounting = importlib.import_module("services.request_accounting") + from types import SimpleNamespace + with self.app.test_request_context("/v1/chat/completions", method="POST"): + g.authenticated_user = {"id": "owner"} + with patch.object(accounting, "budgeted", return_value=True), patch.object(accounting.BudgetService, "check_and_reserve", return_value=SimpleNamespace(allowed=False)): + assert module().embed_accounted("openai:embed-test", "hello", 0.0001) is None + assert not self.embeddings + self.app.extensions["semantic_cache_embedding"] = lambda *_: (_ for _ in ()).throw(RuntimeError("embedding failure")) + rows = [] + with patch.object(accounting.usage_ledger.LEDGER, "record", side_effect=rows.append): + assert self.post()[1] == 1 + assert rows[0]["kind"] == "embeddings" and rows[0]["status"] == 502 and rows[0]["cost_usd"] > 0 + + +def test_migration_is_additive_and_repeatable(): + db = sqlite3.connect(":memory:") + db.executescript("CREATE TABLE existing(id); INSERT INTO existing VALUES(7);") + sql = (Path(__file__).parents[1] / "intelligence-migrations/0026_semantic_cache.sql").read_text() + db.executescript(sql) + db.executescript(sql) + assert db.execute("SELECT id FROM existing").fetchone() == (7,) + assert db.execute("SELECT COUNT(*) FROM semantic_generation_cache").fetchone() == (0,) + + +def test_private_adapter_enforces_complete_bounded_bodies_and_identity(): + import base64 + calls = [] + metadata = {"content_type": "application/json", "headers": {}, "provider": "openai", "model": "chat"} + entry = {"body": base64.b64encode(ANSWER).decode(), "metadata": metadata, "age": 0} + def transport(value): + calls.append(value) + return {"version": 1, "entry": entry} + cache = module().SemanticGenerationCache(transport) + identity = {"principal_hash": "a"*64, "partition_hash": "b"*64, "model_revision": "c"*64} + assert cache.body(identity, {"entry_id": "d"*32}) == (ANSWER, metadata, 0) + assert all(calls[-1][key] == value for key, value in identity.items()) + for change in ({"age": 300}, {"body": base64.b64encode(b'{"error":"no"}').decode()}, {"metadata": {}}): + entry.update(body=base64.b64encode(ANSWER).decode(), metadata=metadata, age=0) + entry.update(change) + assert cache.body(identity, {"entry_id": "d"*32}) is None + + +def test_configured_embedding_preflight_and_transport_use_existing_interfaces(): + m = module() + media = importlib.import_module("routes.media_audio") + accounting = importlib.import_module("services.request_accounting") + app = Flask(__name__) + with app.test_request_context("/v1/chat/completions", method="POST"): + g.authenticated_user = {"id": "owner"} + with patch.object(media, "validate_media_candidate", side_effect=ValueError("unconfigured")), patch.object(accounting, "_record") as record: + with pytest.raises(ValueError): + m.embed_accounted("openai:embed-test", "hello", 0.0001) + record.assert_not_called() + response = Response(json.dumps({"data": [{"embedding": [1, 0]}], "usage": {"prompt_tokens": 4}}), content_type="application/json") + with patch.object(media, "validate_media_candidate") as validate, patch.object(media, "dispatch_media_candidate", return_value=response) as dispatch, patch.object(accounting, "_record") as record: + assert m.embed_accounted("openai:embed-test", "hello", 0.0001) == [1, 0] + assert validate.call_count == dispatch.call_count == record.call_count == 1 diff --git a/tests/test_semantic_generation_cache_worker.mjs b/tests/test_semantic_generation_cache_worker.mjs new file mode 100644 index 00000000..583e3301 --- /dev/null +++ b/tests/test_semantic_generation_cache_worker.mjs @@ -0,0 +1,195 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { readFileSync } from "node:fs"; +import { DatabaseSync } from "node:sqlite"; +import { SemanticCacheD1, handleSemanticCache, cleanupSemanticCache } from "../worker/semantic-cache-d1.mjs"; +import { semanticCacheSettings, semanticPartition, guardHash, prepareSemanticCache, semanticCacheServedEvent } from "../worker/semantic-generation-cache.mjs"; + +const policy = {routes: ["/fixture/v1/chat/completions"], embedding_model: "openai:embed-test", revision: "one"}; +const enabled = {SEMANTIC_CACHE_ENABLED: "true", SEMANTIC_CACHE_POLICY_JSON: JSON.stringify(policy), + MODEL_PRICING_USD_PER_MILLION: JSON.stringify({"openai:embed-test": {input: 0.1, output: 0}})}; +const payload = {model: "chat-test", messages: [{role: "user", content: "explain caching"}], temperature: 0}; +const body = JSON.stringify({choices: [{message: {content: "actual answer"}, finish_reason: "stop"}]}); +const metadata = {content_type: "application/json", headers: {}, provider: "fixture", model: "chat-test"}; +const authority = {provider: "fixture", route: policy.routes[0], principal: {id: "owner", keyHash: "a".repeat(64)}}; +const context = {retentionPolicy: {enabled: false, mode: "inherit"}, cacheRevisions: {policy: "one"}}; +const request = (changes = {}, headers = {}) => new Request("https://provider.invalid/v1/chat/completions", { + method: "POST", headers: {"content-type": "application/json", ...headers}, body: JSON.stringify({...payload, ...changes})}); +const identity = {principal_hash: "a".repeat(64), partition_hash: "b".repeat(64), model_revision: "c".repeat(64)}; + +function storage() { + const db = new DatabaseSync(":memory:"); + db.exec("CREATE TABLE old_rows(id); INSERT INTO old_rows VALUES(7);"); + const sql = readFileSync(new URL("../intelligence-migrations/0026_semantic_cache.sql", import.meta.url), "utf8"); + db.exec(sql); db.exec(sql); assert.equal(db.prepare("SELECT id FROM old_rows").get().id, 7); + const execute = (sql, args) => { + const stmt = db.prepare(sql.replace(/\?(\d+)/g, (_, n) => `$v${n}`)); + const rows = /\?[1-9]/.test(sql) ? stmt.all(Object.fromEntries(args.map((value, i) => [`v${i+1}`, value]))) : stmt.all(...args); + return {results: rows.map(row => ({...row})), meta: {changes: db.prepare("SELECT changes() AS n").get().n}}; + }; + const binding = {prepare(sql) {return {sql, args: [], bind(...args) {this.args=args; return this;}, + async all() {return execute(sql, this.args);}, async run() {return execute(sql, this.args);}, async first() {return execute(sql, this.args).results[0] ?? null;}};}, + async batch(items) {db.exec("BEGIN"); try {const results=items.map(item=>execute(item.sql,item.args)); db.exec("COMMIT"); return results;} + catch(error) {db.exec("ROLLBACK"); throw error;}}}; + const rows=new Map(); + const bucket={rows, async put(key,value,options) {rows.set(key,{value:new Uint8Array(value),customMetadata:options.customMetadata});}, + async get(key) {const row=rows.get(key); return row?{size:row.value.length, async arrayBuffer() {return row.value.slice().buffer;}}:null;}, + async delete(key) {rows.delete(key);}, async list() {return {objects:[...rows].map(([key,row])=>({key,customMetadata:row.customMetadata})),truncated:false};}}; + return {db, env:{...enabled, INTELLIGENCE_DB:binding,multillm_media:bucket}}; +} + +test("configuration defaults and unsupported tools/streams disable the feature", () => { + for(const env of [{},{SEMANTIC_CACHE_ENABLED:"",SEMANTIC_CACHE_POLICY_JSON:""}, + {...enabled,SEMANTIC_CACHE_POLICY_JSON:'{"allow_tools":true}'}, {...enabled,SEMANTIC_CACHE_POLICY_JSON:'{"allow_streams":true}'}, + {...enabled,SEMANTIC_CACHE_ENABLED:"bad"}]) assert.equal(semanticCacheSettings(env).enabled,false); + assert.equal(semanticCacheSettings(enabled).enabled,true); +}); + +test("last-message partition preserves every conversation and parameter invariant", async () => { + const one=await semanticPartition("owner", authority.provider,authority.route,payload,context,policy,enabled); + assert.deepEqual(one,await semanticPartition("owner",authority.provider,authority.route, + {...payload,messages:[{role:"user",content:"describe caching"}]},context,policy,enabled)); + for(const field of ["model","temperature","top_p","seed","response_format"]) { + assert.notDeepEqual(one,await semanticPartition("owner",authority.provider,authority.route,{...payload,[field]:"changed"},context,policy,enabled)); + } + assert.notDeepEqual(one,await semanticPartition("other",authority.provider,authority.route,payload,context,policy,enabled)); + for(const [a,b] of [["buy 1","buy 2"],["do it","do not do it"],['say "one"','say "two"'],["2026-10-09","2026-10-10"], + ["October 9","November 9"],["meet Monday","meet Tuesday"],["buy twelve shares","buy thirteen shares"]]) { + assert.notEqual(await guardHash(a),await guardHash(b)); + } +}); + +test("D1 exact scan is scoped with TTL, corruption checks and oldest eviction", async () => { + const {env,db}=storage(); let now=1000; + const store=new SemanticCacheD1(env,{clock:()=>now}); const guard="d".repeat(64); + for(let i=0;i<257;i++) { + await store.put(identity,[1,i/1000],guard,new TextEncoder().encode(body),metadata); now+=0.001; + } + assert.equal(db.prepare("SELECT COUNT(*) AS n FROM semantic_generation_cache").get().n,256); + const rows=await store.scan(identity); assert.equal(rows.length,256); + assert.equal(await store.scan({...identity,principal_hash:"e".repeat(64)}).then(x=>x.length),0); + assert.equal(new TextDecoder().decode((await store.body(identity,rows[0])).body),body); + const blob=env.multillm_media.rows.get(rows[0].body_pointer); blob.value[0]^=1; + assert.equal(await store.body(identity,rows[0]),null); + assert.equal(await store.put(identity,[1,0],guard,new Uint8Array(1024*1024+1),metadata),false); + now+=300; assert.equal((await store.scan(identity)).length,0); + await cleanupSemanticCache(env,{now,limit:1000}); assert.equal(env.multillm_media.rows.size,0); +}); + +test("native collaborator stores and replays complete responses with paid embedding accounting", async () => { + const {env}=storage(); const costs=[]; + const collaborators={embeddingAllowed:()=>true,reserveEmbedding:()=>null,embed:async()=>({data:[{embedding:[1,0]}],usage:{prompt_tokens:4}}),accountEmbedding:event=>costs.push(event)}; + const first=await prepareSemanticCache(request(),env,authority,context,collaborators); + assert.equal(await first.lookup(),null); + await first.store(new Response(body,{headers:{"content-type":"application/json"}}),{canStore:()=>true}); + const next=await prepareSemanticCache(request({messages:[{role:"user",content:"describe caching"}]}),env,authority,context,collaborators); + const hit=await next.lookup(); assert.equal(await hit.text(),body); + assert.equal(hit.headers.get("X-MultiLLM-Cache"),"semantic-hit"); + assert.equal(hit.headers.get("X-MultiLLM-Provider-Calls"),"0"); + assert.equal(costs.length,2); assert.ok(costs[0].cost_usd>0); + assert.equal(semanticCacheServedEvent(hit,{startedAt:performance.now()}).cost_usd,0); +}); + +test("schema failure returns JSON 503 before any provider; other storage errors miss", async () => { + const {env,db}=storage(); db.exec("DROP TABLE semantic_generation_cache"); let embeddings=0; + const cache=await prepareSemanticCache(request(),env,authority,context,{embed:async()=>{embeddings++;return [1,0];},accountEmbedding:()=>{}}); + const rejection=await cache.lookup(); assert.equal(rejection.status,503); assert.equal(embeddings,0); + assert.equal((await rejection.json()).error,"semantic_cache_schema_missing"); +}); + +test("tools streams n>1 retention no-store unknown price and non-opted scope bypass before storage", async () => { + let touched=0; + const env={...enabled,INTELLIGENCE_DB:{prepare(){touched++;throw Error();}}}; + const collaborator={embed:async()=>assert.fail("embedding called"),accountEmbedding:()=>{}}; + for(const changes of [{stream:true},{n:2},{tools:[{}],tool_choice:"none"},{messages:[{role:"tool",content:"result"},{role:"user",content:"hello"}]}, + {messages:[{role:"user",content:"latest weather today"}]},{messages:[{role:"user",content:"x".repeat(4097)}]}]) { + assert.equal(await prepareSemanticCache(request(changes),env,authority,context,collaborator),null); + } + for(const change of [{SEMANTIC_CACHE_POLICY_JSON:"{}"},{MODEL_PRICING_USD_PER_MILLION:"{}"}, + {MODEL_PRICING_USD_PER_MILLION:JSON.stringify({"openai:embed-test":{input:100,output:0}})}]) { + assert.equal(await prepareSemanticCache(request(),{...env,...change},authority,context,collaborator),null); + } + assert.equal(await prepareSemanticCache(request({}, {"Cache-Control":"no-store"}),env,authority,context,collaborator),null); + assert.equal(await prepareSemanticCache(request(),env,authority,{retentionPolicy:{mode:"zero",enabled:true}},collaborator),null); + assert.equal(touched,0); +}); + +test("protected mismatch and low similarity cannot hit; policy mutation and abort cannot store", async () => { + const {env}=storage(); let vector=[1,0]; let revision=1; + const ctx={...context,cacheRevisions:()=>({revision})}; + const collab={embeddingAllowed:()=>true,reserveEmbedding:()=>null,embed:async()=>({data:[{embedding:vector}]}),accountEmbedding:()=>{}}; + const first=await prepareSemanticCache(request({messages:[{role:"user",content:"explain 12 caches"}]}),env,authority,ctx,collab); + await first.lookup(); await first.store(new Response(body,{headers:{"content-type":"application/json"}}),{canStore:()=>true}); + for(const text of ["describe 13 caches","do not explain 12 caches"]) { + const next=await prepareSemanticCache(request({messages:[{role:"user",content:text}]}),env,authority,ctx,collab); assert.equal(await next.lookup(),null); + } + vector=[0.9,0.44]; const low=await prepareSemanticCache(request({messages:[{role:"user",content:"describe 12 caches"}]}),env,authority,ctx,collab); + assert.equal(await low.lookup(),null); revision++; await low.store(new Response(body,{headers:{"content-type":"application/json"}}),{canStore:()=>true}); + assert.equal((await env.INTELLIGENCE_DB.prepare("SELECT COUNT(*) AS n FROM semantic_generation_cache").first()).n,1); +}); + +test("private handler validates operations, forbids cross-scope bodies and sanitizes schema errors", async () => { + const {env}=storage(); + assert.equal((await handleSemanticCache(env.INTELLIGENCE_DB,{version:1,operation:"ready"},env)).status,200); + assert.equal(await handleSemanticCache(env.INTELLIGENCE_DB,{version:1,operation:"scan",...identity,principal_hash:"bad"},env),null); + const broken={prepare(){throw Error("no such table: semantic_generation_cache");}}; + const failed=await handleSemanticCache(broken,{version:1,operation:"ready"},env); + assert.equal(failed.status,503); assert.doesNotMatch(await failed.text(),/no such table/); +}); + +test("embedding grants and budget admission precede dispatch and failures retain accounting", async () => { + const {env}=storage(); let calls=0; const costs=[]; + const collaborators={embeddingAllowed:()=>true,reserveEmbedding:()=>null, + embed:async()=>{calls++;throw Error("upstream failed");},accountEmbedding:event=>costs.push(event)}; + for(const change of [{embeddingAllowed:()=>false},{reserveEmbedding:()=>false},{embeddingAllowed:undefined}]) { + assert.equal(await prepareSemanticCache(request(),env,authority,context,{...collaborators,...change}),null); + } + assert.equal(calls,0); assert.equal(costs.length,0); + assert.equal(await prepareSemanticCache(request(),env,authority,context,collaborators),null); + assert.equal(calls,1); assert.equal(costs[0].status,502); assert.equal(costs[0].provider_calls,1); + assert.ok(costs[0].cost_usd>0); assert.equal(costs[0].submission_outcome,"unknown"); +}); + +test("cancellation, unclassified success, incomplete output and outages never replay or store", async () => { + const {env,db}=storage(); const controller=new AbortController(); + const req=new Request(request(),{signal:controller.signal}); + const collaborators={embeddingAllowed:()=>true,reserveEmbedding:()=>null,embed:async()=>[1,0],accountEmbedding:()=>{}}; + const prepared=await prepareSemanticCache(req,env,authority,context,collaborators); + await prepared.store(new Response(body,{headers:{"content-type":"application/json"}})); + for(const invalid of [JSON.stringify({choices:[{message:{content:"cut"},finish_reason:"length"}]}), + JSON.stringify({choices:[{message:{tool_calls:[{}]},finish_reason:"stop"}]})]) { + await prepared.store(new Response(invalid,{headers:{"content-type":"application/json"}}),{canStore:()=>true}); + } + assert.equal(db.prepare("SELECT COUNT(*) AS n FROM semantic_generation_cache").get().n,0); + const reply=new Response(body,{headers:{"content-type":"application/json"}}); + controller.abort(); assert.equal(await prepared.store(reply,{canStore:()=>true}),reply); + assert.equal(await prepared.lookup(),null); + assert.equal(await prepareSemanticCache(req,env,authority,context,collaborators),null); + const failed={async ready(){},async scan(){throw Error("unavailable");},async put(){throw Error("unavailable");}}; + const unavailable=await prepareSemanticCache(request(),env,authority,context,{...collaborators,store:failed}); + assert.equal(await unavailable.lookup(),null); + const original=new Response(body,{headers:{"content-type":"application/json"}}); + assert.equal(await unavailable.store(original,{canStore:()=>true}),original); assert.equal(await original.text(),body); + const missing=await prepareSemanticCache(request(),env,authority,context, + {...collaborators,store:{...failed,async scan(){throw Error("no such table: semantic_generation_cache");}}}); + assert.equal((await missing.lookup()).status,503); +}); + +test("embedding timeout aborts its transport and records uncertain submission once", async () => { + const {env}=storage(); const costs=[]; let signal; + assert.equal(await prepareSemanticCache(request(),env,authority,context,{embeddingAllowed:()=>true,reserveEmbedding:()=>null, + embed:async(_model,_text,options)=>{signal=options.signal;return new Promise(()=>{});},accountEmbedding:event=>costs.push(event)}),null); + assert.equal(signal.aborted,true); assert.equal(costs.length,1); assert.equal(costs[0].submission_outcome,"unknown"); +}); + +test("parallel writes stay scoped and strictly within principal capacity", async () => { + const {env,db}=storage(); const store=new SemanticCacheD1(env); + await Promise.all(Array.from({length:270},()=>store.put(identity,[1,0],"d".repeat(64),new TextEncoder().encode(body),metadata))); + assert.equal(db.prepare("SELECT COUNT(*) AS n FROM semantic_generation_cache").get().n,256); + const row=(await store.scan(identity))[0]; + assert.equal(await store.body({...identity,principal_hash:"e".repeat(64)},row),null); + const scan=await handleSemanticCache(env.INTELLIGENCE_DB,{version:1,operation:"scan",...identity,vector:[1,0],guard_hash:"d".repeat(64)},env); + const matches=(await scan.json()).rows; assert.equal(matches.length,4); + const foreign=await handleSemanticCache(env.INTELLIGENCE_DB,{version:1,operation:"scan",...identity,vector:[1,0],guard_hash:"e".repeat(64)},env); + assert.equal((await foreign.json()).rows.length,0); +}); diff --git a/worker/container-env.mjs b/worker/container-env.mjs index 1a7994db..bebb199d 100644 --- a/worker/container-env.mjs +++ b/worker/container-env.mjs @@ -40,6 +40,8 @@ const DIRECT_ENV_KEYS = [ "CONNECTION_PROFILES_DB_PATH", "CONFIG_REVISION_SYNC_ENABLED", "CONFIG_SYNC_TTL_SECONDS", + "SEMANTIC_CACHE_ENABLED", + "SEMANTIC_CACHE_POLICY_JSON", "GATEWAY_ALERTS_ENABLED", "GATEWAY_ALERT_WEBHOOK_ALLOWLIST", "CONFIG_SECURITY_TTL_SECONDS", diff --git a/worker/d1-schema.mjs b/worker/d1-schema.mjs index 366c2e78..6b34619f 100644 --- a/worker/d1-schema.mjs +++ b/worker/d1-schema.mjs @@ -10,7 +10,7 @@ export const REQUIRED_D1_TABLES = Object.freeze(["intelligence_policy", "intelli "usage_events", "usage_daily", "usage_batches", "media_jobs", "media_job_items", "shadow_eval_samples", "shadow_eval_results", "shadow_eval_config", "shadow_eval_policy_backups", "prompt_templates", - "config_snapshot_revisions", "config_snapshots", "config_snapshot_applications", "control_revisions", "tool_grants", "generation_cache", "usage_reservation_budgets", "usage_reservations", "usage_reservation_transitions", "session_tiers", "gateway_alert_rules", "gateway_alert_events", "managed_idempotency"]); + "config_snapshot_revisions", "config_snapshots", "config_snapshot_applications", "control_revisions", "tool_grants", "generation_cache", "usage_reservation_budgets", "usage_reservations", "usage_reservation_transitions", "session_tiers", "gateway_alert_rules", "gateway_alert_events", "managed_idempotency", "semantic_generation_cache"]); /** * Readiness for the D1 schema. A deploy that skipped `wrangler d1 migrations apply` reports diff --git a/worker/semantic-cache-d1.mjs b/worker/semantic-cache-d1.mjs new file mode 100644 index 00000000..aa58ad7e --- /dev/null +++ b/worker/semantic-cache-d1.mjs @@ -0,0 +1,146 @@ +/** Scoped exact vector scans and immutable bodies in the existing media bucket. */ +import { digest, encodeBody, decodeBody, completeCacheBody, validCacheMetadata, MAX_BODY_BYTES } from "./generation-cache-d1.mjs"; +export const SEMANTIC_PREFIX = "semantic-cache/v1/"; +export const SEMANTIC_TTL = 300; +const HASH = /^[0-9a-f]{64}$/; +const ID = /^[0-9a-f]{32}$/; +const object = value => value && typeof value === "object" && !Array.isArray(value); +export const validSemanticIdentity = value => object(value) + && ["principal_hash", "partition_hash", "model_revision"].every(key => HASH.test(value[key])); +export function validVector(value) { + return Array.isArray(value) && value.length > 0 && value.length <= 8192 + && value.every(x => typeof x === "number" && Number.isFinite(x)) + && Number.isFinite(value.reduce((n, x) => n + x*x, 0)) && value.reduce((n, x) => n + x*x, 0) > 0; +} +export function vectorSimilarity(one, two) { + if (!validVector(one) || !validVector(two) || one.length !== two.length) return -1; + return one.reduce((n,x,i)=>n+x*two[i],0) / Math.sqrt(one.reduce((n,x)=>n+x*x,0)) / Math.sqrt(two.reduce((n,x)=>n+x*x,0)); +} +export function schemaMissing(error) { + return /no such table:\s*semantic_generation_cache\b/i.test(String(error?.message ?? "")); +} +export function schemaErrorResponse() { + return Response.json({error: "semantic_cache_schema_missing", + message: "The semantic generation cache schema is missing. Apply its migration before enabling it."}, + {status: 503, headers: {"cache-control": "no-store"}}); +} +const FIELDS = "entry_id, partition_hash, model_revision, vector, guard_hash, created_at, expires_at, body_pointer, body_bytes, body_hash, metadata"; +const INSERT = `INSERT INTO semantic_generation_cache + (principal_hash, entry_id, partition_hash, model_revision, vector, guard_hash, + created_at, expires_at, body_pointer, body_bytes, body_hash, metadata) + SELECT ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12 + WHERE (SELECT COUNT(*) FROM semantic_generation_cache WHERE principal_hash=?1) < 256`; + +export class SemanticCacheD1 { + constructor(env, {clock = () => Date.now()/1000} = {}) {this.env = env; this.clock = clock;} + async ready() { + await this.env.INTELLIGENCE_DB.prepare("SELECT entry_id FROM semantic_generation_cache LIMIT 0").all(); + if (!this.env.multillm_media) throw new Error("semantic_cache_storage_unavailable"); + } + async scan(identity) { + if (!validSemanticIdentity(identity)) return []; + const now = this.clock(); + const result = await this.env.INTELLIGENCE_DB.prepare(`SELECT ${FIELDS} FROM semantic_generation_cache + WHERE principal_hash=? AND partition_hash=? AND model_revision=? AND expires_at>? ORDER BY created_at DESC, entry_id DESC LIMIT 256`) + .bind(identity.principal_hash, identity.partition_hash, identity.model_revision, now).all(); + return result.results.flatMap(row => { + try { + const vector = JSON.parse(row.vector); + return validVector(vector) && HASH.test(row.guard_hash) && ID.test(row.entry_id) + && Number.isFinite(row.created_at) && row.created_at <= now && row.expires_at <= row.created_at + SEMANTIC_TTL + ? [{...row, vector}] : []; + } catch {return [];} + }); + } + async body(identity, row) { + if (!validSemanticIdentity(identity) || !ID.test(row?.entry_id)) return null; + // Never trust a caller's pointer or metadata: re-read the scoped row. + const saved = await this.env.INTELLIGENCE_DB.prepare(`SELECT ${FIELDS} FROM semantic_generation_cache + WHERE principal_hash=? AND entry_id=? AND partition_hash=? AND model_revision=? AND expires_at>?`) + .bind(identity.principal_hash, row.entry_id, identity.partition_hash, identity.model_revision, this.clock()).first(); + if (!saved || saved.created_at > this.clock() || saved.expires_at > saved.created_at + SEMANTIC_TTL + || !Number.isInteger(saved.body_bytes) || saved.body_bytes < 1 || saved.body_bytes > MAX_BODY_BYTES || !HASH.test(saved.body_hash)) return null; + const prefix = `${SEMANTIC_PREFIX}${identity.principal_hash}/${saved.entry_id}/`; + if (typeof saved.body_pointer !== "string" || !saved.body_pointer.startsWith(prefix) || !ID.test(saved.body_pointer.slice(prefix.length))) return null; + let metadata; + try {metadata = JSON.parse(saved.metadata);} catch {return null;} + if (!validCacheMetadata(metadata)) return null; + const object = await this.env.multillm_media.get(saved.body_pointer); + if (!object || object.size !== saved.body_bytes) return null; + const body = new Uint8Array(await object.arrayBuffer()); + if (body.length !== saved.body_bytes || await digest(body) !== saved.body_hash || !completeCacheBody(body)) return null; + const age = this.clock() - saved.created_at; + return age >= 0 && age < SEMANTIC_TTL ? {body, metadata, age} : null; + } + async put(identity, vector, guard, body, metadata) { + if (!validSemanticIdentity(identity) || !validVector(vector) || !HASH.test(guard) || !(body instanceof Uint8Array) + || body.length > MAX_BODY_BYTES || !completeCacheBody(body) || !validCacheMetadata(metadata)) return false; + await this.ready(); + const now = this.clock(), expiry = now + SEMANTIC_TTL; + const id = crypto.randomUUID().replaceAll("-", ""); + const pointer = `${SEMANTIC_PREFIX}${identity.principal_hash}/${id}/${crypto.randomUUID().replaceAll("-", "")}`; + await this.env.multillm_media.put(pointer, body, {httpMetadata: {contentType: "application/json"}, customMetadata: {expires_at: String(expiry)}}); + let stored = false; + try { + const results = await this.env.INTELLIGENCE_DB.batch([ + this.env.INTELLIGENCE_DB.prepare("DELETE FROM semantic_generation_cache WHERE principal_hash=? AND expires_at<=?").bind(identity.principal_hash, now), + this.env.INTELLIGENCE_DB.prepare(`DELETE FROM semantic_generation_cache WHERE principal_hash=?1 AND entry_id IN + (SELECT entry_id FROM semantic_generation_cache WHERE principal_hash=?1 ORDER BY created_at, entry_id + LIMIT MAX(0, (SELECT COUNT(*) FROM semantic_generation_cache WHERE principal_hash=?1) - 255))`).bind(identity.principal_hash), + this.env.INTELLIGENCE_DB.prepare(INSERT).bind(identity.principal_hash, id, identity.partition_hash, identity.model_revision, + JSON.stringify(vector), guard, now, expiry, pointer, body.length, await digest(body), JSON.stringify(metadata)), + ]); + stored = results[2].meta.changes === 1; + return stored; + } finally {if (!stored) await this.env.multillm_media.delete(pointer).catch(() => {});} + } +} + +export async function cleanupSemanticCache(env, {now = Date.now()/1000, limit = 100, cursor} = {}) { + if (!Number.isFinite(now) || !Number.isInteger(limit) || limit < 1 || limit > 1000) throw new TypeError("invalid_cleanup_bounds"); + const result = await env.INTELLIGENCE_DB.prepare(`DELETE FROM semantic_generation_cache WHERE (principal_hash, entry_id) IN + (SELECT principal_hash, entry_id FROM semantic_generation_cache WHERE expires_at<=? LIMIT ?)`).bind(now, limit).run(); + const page = await env.multillm_media.list({prefix: SEMANTIC_PREFIX, limit, cursor, include: ["customMetadata"]}); + let deleted = 0; + for (const object of page.objects) { + const expiry = Number(object.customMetadata?.expires_at); + if (object.key.startsWith(SEMANTIC_PREFIX) && object.customMetadata?.expires_at && Number.isFinite(expiry) && expiry <= now) { + await env.multillm_media.delete(object.key); deleted++; + } + } + return {rows: result.meta.changes, objects: deleted, cursor: page.truncated ? page.cursor : null}; +} + +const exactFields = (value, fields) => object(value) && Object.keys(value).length === fields.length && fields.every(k => Object.hasOwn(value, k)); +const IDENTITY = ["version", "operation", "principal_hash", "partition_hash", "model_revision"]; +export async function handleSemanticCache(db, value, env) { + if (!object(value) || value.version !== 1) return null; + const store = new SemanticCacheD1({...env, INTELLIGENCE_DB: db}); + const reply = value => Response.json({version: 1, ...value}, {headers: {"cache-control": "no-store"}}); + try { + if (value.operation === "ready" && exactFields(value, ["version", "operation"])) {await store.ready(); return reply({ready: true});} + if (!validSemanticIdentity(value)) return null; + if (value.operation === "scan" && exactFields(value, [...IDENTITY, "vector", "guard_hash"]) + && validVector(value.vector) && HASH.test(value.guard_hash)) { + const rows = await store.scan(value); + // Scan the full scoped set, but bound the private reply even for 8,192-dimensional vectors. + const candidates = rows.filter(row=>row.guard_hash === value.guard_hash && vectorSimilarity(value.vector,row.vector)>=0.98) + .sort((a,b)=>vectorSimilarity(value.vector,b.vector)-vectorSimilarity(value.vector,a.vector)).slice(0,4); + return reply({rows: candidates.map(({entry_id, vector, guard_hash}) => ({entry_id, vector, guard_hash}))}); + } + if (value.operation === "body" && exactFields(value, [...IDENTITY, "entry_id"]) && ID.test(value.entry_id)) { + const entry = await store.body(value, value); + return reply({entry: entry ? {...entry, body: encodeBody(entry.body)} : null}); + } + if (value.operation === "put" && exactFields(value, [...IDENTITY, "vector", "guard_hash", "body", "metadata"]) + && validVector(value.vector) && HASH.test(value.guard_hash) && validCacheMetadata(value.metadata)) { + const body = decodeBody(value.body); + if (!body || !completeCacheBody(body)) return null; + return reply({stored: await store.put(value, value.vector, value.guard_hash, body, value.metadata)}); + } + return null; + } catch (error) { + if (schemaMissing(error)) return schemaErrorResponse(); + return Response.json({error: "semantic_cache_unavailable", message: "Semantic cache storage is unavailable."}, {status: 503}); + } +} diff --git a/worker/semantic-generation-cache.mjs b/worker/semantic-generation-cache.mjs new file mode 100644 index 00000000..995791d0 --- /dev/null +++ b/worker/semantic-generation-cache.mjs @@ -0,0 +1,210 @@ +/** Native semantic lookup after authentication and policy, before generation dispatch. */ +import { digest, MAX_BODY_BYTES, completeCacheBody } from "./generation-cache-d1.mjs"; +import { SemanticCacheD1, validVector, vectorSimilarity, schemaMissing, schemaErrorResponse } from "./semantic-cache-d1.mjs"; +import { retentionAllowsContent } from "./retention-policy.mjs"; +import { cacheServedEvent } from "./exact-generation-cache.mjs"; +const object = value => value && typeof value === "object" && !Array.isArray(value); +const MODEL = /^[a-z][a-z0-9-]*:[A-Za-z0-9@][A-Za-z0-9._:/+@-]{0,255}$/; +const VOLATILE = /\b(?:today|tomorrow|yesterday|now|current|latest|live|weather|stock|real.?time)\b/i; +let invalidWarned = false; +const warned = new Set(); +function warn(event) {if (!warned.has(event)) {warned.add(event); console.warn(`Semantic generation cache ${event}`);}} +function canonical(value) { + if (Array.isArray(value)) return value.map(canonical); + if (object(value)) return Object.fromEntries(Object.keys(value).sort().map(key => [key, canonical(value[key])])); + return value; +} +export function semanticCacheSettings(env = {}) { + const flag = String(env.SEMANTIC_CACHE_ENABLED ?? "").trim().toLowerCase(); + if (["", "false", "0", "no", "off"].includes(flag)) return {enabled: false, policy: {}}; + try { + if (!["true", "1", "yes", "on"].includes(flag)) throw Error(); + const policy = JSON.parse(String(env.SEMANTIC_CACHE_POLICY_JSON ?? "").trim() || "{}"); + if (!object(policy) || Object.keys(policy).some(k => !["routes", "keys", "embedding_model", "revision", "model_revision", "allow_tools", "allow_streams"].includes(k)) + || ["allow_tools", "allow_streams"].some(k => Object.hasOwn(policy, k) && policy[k] !== false)) throw Error(); + for (const key of ["routes", "keys"]) if (!Array.isArray(policy[key] ?? []) + || (policy[key] ?? []).some(v => typeof v !== "string" || !v.length || v.length > 256)) throw Error(); + if (((policy.routes?.length ?? 0) || (policy.keys?.length ?? 0)) && !MODEL.test(policy.embedding_model)) throw Error(); + for (const key of ["revision", "model_revision"]) if (Object.hasOwn(policy, key) + && (typeof policy[key] !== "string" || !policy[key].length || policy[key].length > 256)) throw Error(); + return {enabled: true, policy}; + } catch { + if (!invalidWarned) {invalidWarned = true; console.warn("Semantic generation cache invalid_configuration");} + return {enabled: false, policy: {}}; + } +} +export function semanticEligible(payload) { + if (!object(payload) || typeof payload.model !== "string" || !payload.model.length || /^(auto|free|cascade):/.test(payload.model) + || payload.stream || (Object.hasOwn(payload,"n") ? payload.n : 1) !== 1 || Object.hasOwn(payload, "routing") + || ["tools", "functions", "tool_choice", "function_call"].some(k => payload[k] != null + && (!Array.isArray(payload[k]) || payload[k].length > 0))) return false; + const messages = payload.messages; + if (!Array.isArray(messages) || !messages.length || messages.some(m => !object(m) + || !["user", "system", "assistant", "developer"].includes(m.role) || m.tool_calls && (!Array.isArray(m.tool_calls) || m.tool_calls.length) || m.function_call || m.tool_call_id)) return false; + const last = messages.at(-1); + return last.role === "user" && typeof last.content === "string" && !!last.content.trim() + && new TextEncoder().encode(last.content).length <= 4096 && !VOLATILE.test(last.content); +} +export async function guardHash(text) { + const facts = text.toLowerCase().match(/[-+]?\d+(?:[.,:/-]\d+)*|\b(?:no|not|never|without|cannot|neither|nor|nothing|nobody|none)\b|\b\w+n['’]t\b/g) ?? []; + const datesAndNumbers = text.toLowerCase().match(/\b(?:jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|jun(?:e)?|jul(?:y)?|aug(?:ust)?|sep(?:t(?:ember)?)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?|mon(?:day)?|tue(?:s(?:day)?)?|wed(?:nesday)?|thu(?:rs(?:day)?)?|fri(?:day)?|sat(?:urday)?|sun(?:day)?|zero|one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve|thirteen|fourteen|fifteen|sixteen|seventeen|eighteen|nineteen|twenty|thirty|forty|fifty|sixty|seventy|eighty|ninety|hundred|thousand|million|billion|trillion|first|second|third|fourth|fifth|sixth|seventh|eighth|ninth|tenth)\b/g) ?? []; + const quotes = text.match(/"[^"\n]*"|(? Object.hasOwn(entry, k)); + const input = flatOnly ? 0 : entry.input ?? entry.input_cost_per_million; + const output = flatOnly ? 0 : entry.output ?? entry.output_cost_per_million; + const flat = Object.hasOwn(entry, "request") ? entry.request : 0; + if ([input, output, flat].some(v => !["string", "number"].includes(typeof v) || String(v).trim() === "" + || !Number.isFinite(Number(v)) || Number(v) < 0)) return null; + const cost = Number(input)*tokens/1_000_000+Number(flat); + return Number.isFinite(cost) ? cost : null; + } catch {return null;} +} +async function boundedBytes(stream, maximum) { + if (!stream) return null; + const reader = stream.getReader(), parts = []; let size = 0, timer; + const deadline = new Promise(resolve => {timer = setTimeout(() => resolve(null), 2000);}); + try { + while (true) { + const part = await Promise.race([reader.read(), deadline]); + if (!part) return null; + if (part.done) break; + size += part.value.length; if (size > maximum) return null; + parts.push(part.value); + } + const bytes = new Uint8Array(size); let offset = 0; + for (const part of parts) {bytes.set(part, offset); offset += part.length;} + return bytes; + } finally {clearTimeout(timer); void reader.cancel().catch(() => {});} +} +async function optional(operation) { + let timer; + try {return await Promise.race([operation, new Promise((_, reject) => {timer = setTimeout(() => reject(Error("semantic_cache_timeout")), 2000);})]);} + finally {clearTimeout(timer);} +} +function markHit(entry) { + return new Response(entry.body, {headers: {"content-type": "application/json", ...entry.metadata.headers, + "X-MultiLLM-Cache": "semantic-hit", "X-MultiLLM-Cache-Backend": "semantic-d1-r2", + "X-MultiLLM-Usage-Basis": "cache-served", "X-MultiLLM-Provider-Calls": "0", Age: String(Math.floor(entry.age))}}); +} +async function paidEmbedding(env, authority, request, policy, text, price, collaborators) { + if (["embed", "accountEmbedding", "embeddingAllowed", "reserveEmbedding"].some(key => typeof collaborators[key] !== "function")) return null; + if (await optional(collaborators.embeddingAllowed(policy.embedding_model, authority)) !== true) return null; + const reservation = await optional(collaborators.reserveEmbedding(policy.embedding_model, price, authority)); + if (reservation === false) return null; + const controller = new AbortController(), abort = () => controller.abort(request.signal.reason); + request.signal.addEventListener("abort", abort, {once: true}); + const started = performance.now(); let result, status = 502, dispatched = false; + try { + if (request.signal.aborted) {status = 499; return null;} + dispatched = true; + result = await optional(collaborators.embed(policy.embedding_model, text, {signal: controller.signal, reservation})); + const vector = object(result) ? result.data?.[0]?.embedding : result; + status = validVector(vector) ? 200 : 502; + return validVector(vector) ? vector : null; + } finally { + controller.abort(); request.signal.removeEventListener("abort", abort); + const actual = result?.usage?.prompt_tokens ?? result?.usage?.input_tokens; + const measured = Number.isSafeInteger(actual) && actual >= 0; + await optional(collaborators.accountEmbedding({provider: policy.embedding_model.split(":", 1)[0], model: policy.embedding_model, + principal: authority.principal.id, endpoint: "/v1/embeddings", status, input_tokens: !dispatched ? 0 : measured ? actual : 1024, + output_tokens: 0, cost_usd: !dispatched ? 0 : measured ? embeddingPrice(env, policy.embedding_model, actual) : price, + cost_basis: measured ? "usage" : "estimate", provider_calls: dispatched ? 1 : 0, + submission_outcome: !dispatched ? "before-dispatch" : result ? "response" : "unknown", + duration_ms: Math.round(performance.now()-started), reservation})); + } +} + +export async function prepareSemanticCache(request, env, authority, context, collaborators = {}) { + const config = semanticCacheSettings(env), policy = config.policy; + const source = authority.cacheRequest ?? request; + if (!config.enabled || request.signal.aborted || !retentionAllowsContent(context.retentionPolicy) || request.method !== "POST" + || !authority.route?.endsWith("/chat/completions") || !authority.principal?.id + || !(policy.routes?.includes(authority.route) || policy.keys?.includes(authority.principal.id)) + || source.headers.get("Cache-Control")?.toLowerCase().includes("no-store") || new URL(source.url).search + || ["x-api-key", "api-key", "x-multillm-api-key", "x-provider", "x-model", "x-multillm-provider", "idempotency-key"] + .some(key => source.headers.has(key))) return null; + const price = embeddingPrice(env, policy.embedding_model); + if (price === null || price > 0.001) return null; + let payload; + try { + const bytes = await boundedBytes(request.clone().body, MAX_BODY_BYTES); + if (!bytes) return null; + payload = JSON.parse(new TextDecoder("utf-8", {fatal: true}).decode(bytes)); + } catch {return null;} + if (!semanticEligible(payload)) return null; + const keyHash = authority.principal.keyHash ?? (env.ADMIN_API_KEY ? await digest(env.ADMIN_API_KEY) : null); + if (!keyHash) return null; + const principal = `${authority.principal.id}\0${keyHash}`; + const policyContext = () => ({...context, semanticSecurity: {grants: authority.principal.model_allowlist ?? null, + secret_policy: authority.principal.secret_scan_mode ?? null, + headers: Object.fromEntries(["openai-organization", "openai-project", "openai-beta", "anthropic-version", "anthropic-beta"] + .map(key => [key, source.headers.get(key)]))}}); + const current = () => semanticPartition(principal, authority.provider, authority.route, payload, policyContext(), + semanticCacheSettings(env).policy, env); + const identity = await current(), guard = await guardHash(payload.messages.at(-1).content); + const store = collaborators.store ?? new SemanticCacheD1(env); + try {await optional(store.ready());} + catch (error) { + if (schemaMissing(error)) return {async lookup() {return schemaErrorResponse();}, async store(response) {return response;}}; + warn("storage_unavailable"); return null; + } + let vector; + try {vector = await paidEmbedding(env, authority, request, policy, payload.messages.at(-1).content, price, collaborators);} + catch {warn("embedding_unavailable"); return null;} + if (!vector || request.signal.aborted) return null; + const unchanged = async () => semanticCacheSettings(env).enabled && retentionAllowsContent(context.retentionPolicy) + && JSON.stringify(await current()) === JSON.stringify(identity) && !request.signal.aborted + && await optional(collaborators.embeddingAllowed(policy.embedding_model, authority)) === true; + return { + async lookup() { + const control = source.headers.get("Cache-Control")?.toLowerCase() ?? ""; + if (control.includes("no-cache") || source.headers.get("X-MultiLLM-Cache")?.toLowerCase() === "refresh") return null; + try { + const rows = await optional(store.scan(identity)); + const candidates = rows.filter(row => row.guard_hash === guard && cosine(vector, row.vector) >= 0.98) + .sort((a,b) => cosine(vector,b.vector)-cosine(vector,a.vector)); + const age = control.match(/(?:^|,)\s*max-age\s*=\s*(\d{1,9})\s*(?:,|$)/); + for (const row of candidates) { + const entry = await optional(store.body(identity, row)); + if (entry && (!age || entry.age <= Number(age[1])) && await unchanged()) return markHit(entry); + } + } catch (error) {if (schemaMissing(error)) return schemaErrorResponse(); warn("storage_unavailable");} + return null; + }, + async store(response, {canStore = () => false} = {}) { + if (response.status !== 200 || response.headers.get("content-type")?.split(";",1)[0] !== "application/json" + || response.headers.get("Cache-Control")?.toLowerCase().includes("no-store") + || response.headers.get("content-encoding") && response.headers.get("content-encoding") !== "identity") return response; + try { + const body = await boundedBytes(response.clone().body, MAX_BODY_BYTES); + if (body && completeCacheBody(body) && canStore() === true && await unchanged()) { + await optional(store.put(identity, vector, guard, body, + {content_type:"application/json",headers:{},provider:authority.provider,model:payload.model})); + } + } catch {warn("storage_unavailable");} + return response; + }, + }; +} +export const semanticCacheServedEvent = cacheServedEvent;