diff --git a/cloudflare-worker.mjs b/cloudflare-worker.mjs index 2f7598ef..a2d1a31d 100644 --- a/cloudflare-worker.mjs +++ b/cloudflare-worker.mjs @@ -1,4 +1,4 @@ -import { nativeGenerationFetch, withForwardedCorrelation, withNativeMetrics } from "./worker/gateway-extensions.mjs"; +import { nativeGenerationFetch, withForwardedCorrelation, withNativeMetrics, nativeCacheHeader } from "./worker/gateway-extensions.mjs"; import { tickNativeRevisionSync } from "./worker/native-config-sync.mjs"; import { firewallFetch } from "./worker/secret-firewall.mjs"; import { Container, ContainerProxy, getContainer } from "@cloudflare/containers"; @@ -526,13 +526,13 @@ function buildLinkApiUpstreamHeaders(request, protocol, upstreamPathname, upstre return headers; } -function copyLinkApiResponseHeaders(headers) { +function copyLinkApiResponseHeaders(headers, env) { const responseHeaders = new Headers(); for (const [header, value] of headers.entries()) { const normalized = header.toLowerCase(); if ( - LINKAPI_RESPONSE_HEADER_WHITELIST.has(normalized) || + nativeCacheHeader(normalized, headers, env) || LINKAPI_RESPONSE_HEADER_WHITELIST.has(normalized) || LINKAPI_RESPONSE_HEADER_PREFIXES.some((prefix) => normalized.startsWith(prefix)) ) { responseHeaders.set(header, value); @@ -613,13 +613,13 @@ function buildCodexEasyUpstreamHeaders(request, upstreamToken) { return headers; } -function copyCodexEasyResponseHeaders(headers) { +function copyCodexEasyResponseHeaders(headers, env) { const responseHeaders = new Headers(); for (const [header, value] of headers.entries()) { const normalized = header.toLowerCase(); if ( - CODEX_EASY_RESPONSE_HEADER_WHITELIST.has(normalized) || + nativeCacheHeader(normalized, headers, env) || CODEX_EASY_RESPONSE_HEADER_WHITELIST.has(normalized) || CODEX_EASY_RESPONSE_HEADER_PREFIXES.some((prefix) => normalized.startsWith(prefix)) ) { responseHeaders.set(header, value); @@ -1497,7 +1497,7 @@ async function handleDirectOpencodeRequest(request, env, requestUrl, ctx) { ...(bodyAllowed && request.body ? { duplex: "half" } : {}), }); const upstreamResponse = await nativeGenerationFetch(upstreamRequest, env, ctx, { - route: requestUrl.pathname, retentionHeader: request.headers.get("X-MultiLLM-Retention"), principal: { id: env.ADMIN_USERNAME || "admin" }, + route: requestUrl.pathname, cacheRequest: request, retentionHeader: request.headers.get("X-MultiLLM-Retention"), principal: { id: env.ADMIN_USERNAME || "admin" }, provider: requestUrl.pathname.split("/")[1], }, async (upstreamRequest, env, authority) => await firewallFetch(upstreamRequest, env, authority)); @@ -1505,7 +1505,7 @@ async function handleDirectOpencodeRequest(request, env, requestUrl, ctx) { const downstreamResponse = new Response(upstreamResponse.body, { status: upstreamResponse.status, statusText: upstreamResponse.statusText, - headers: copyLinkApiResponseHeaders(upstreamResponse.headers), + headers: copyLinkApiResponseHeaders(upstreamResponse.headers, env), }); return applyCorsHeaders( request, @@ -1632,7 +1632,7 @@ async function handleDirectLinkApiRequest(request, env, requestUrl, ctx) { ...(bodyAllowed && normalizedRequest.body ? { duplex: "half" } : {}), }); const upstreamResponse = await nativeGenerationFetch(upstreamRequest, env, ctx, { - route: requestUrl.pathname, retentionHeader: request.headers.get("X-MultiLLM-Retention"), principal: { id: env.ADMIN_USERNAME || "admin" }, + route: requestUrl.pathname, cacheRequest: request, retentionHeader: request.headers.get("X-MultiLLM-Retention"), principal: { id: env.ADMIN_USERNAME || "admin" }, provider: requestUrl.pathname.split("/")[1], }, async (upstreamRequest, env, authority) => await firewallFetch(upstreamRequest, env, authority)); @@ -1641,7 +1641,7 @@ async function handleDirectLinkApiRequest(request, env, requestUrl, ctx) { new Response(upstreamResponse.body, { status: upstreamResponse.status, statusText: upstreamResponse.statusText, - headers: copyLinkApiResponseHeaders(upstreamResponse.headers), + headers: copyLinkApiResponseHeaders(upstreamResponse.headers, env), }), env, ); @@ -1690,7 +1690,7 @@ async function handleDirectCodexEasyRequest(request, env, requestUrl, ctx) { ...(bodyAllowed && normalizedRequest.body ? { duplex: "half" } : {}), }); const upstreamResponse = await nativeGenerationFetch(upstreamRequest, env, ctx, { - route: requestUrl.pathname, retentionHeader: request.headers.get("X-MultiLLM-Retention"), principal: { id: env.ADMIN_USERNAME || "admin" }, + route: requestUrl.pathname, cacheRequest: request, retentionHeader: request.headers.get("X-MultiLLM-Retention"), principal: { id: env.ADMIN_USERNAME || "admin" }, provider: requestUrl.pathname.split("/")[1], }, async (upstreamRequest, env, authority) => await firewallFetch(upstreamRequest, env, authority)); @@ -1699,7 +1699,7 @@ async function handleDirectCodexEasyRequest(request, env, requestUrl, ctx) { new Response(upstreamResponse.body, { status: upstreamResponse.status, statusText: upstreamResponse.statusText, - headers: copyCodexEasyResponseHeaders(upstreamResponse.headers), + headers: copyCodexEasyResponseHeaders(upstreamResponse.headers, env), }), env, ); diff --git a/docs/shared-generation-cache.md b/docs/shared-generation-cache.md new file mode 100644 index 00000000..38e84a78 --- /dev/null +++ b/docs/shared-generation-cache.md @@ -0,0 +1,35 @@ +# Shared exact generation cache + +The default settings keep the existing process-memory cache: + +```text +GENERATION_CACHE_BACKEND=memory +GENERATION_CACHE_SHARED_ENABLED=false +``` + +To share exact complete Chat Completions responses between Container instances and native Worker requests, apply `intelligence-migrations/0019_generation_cache.sql`, then set: + +```text +GENERATION_CACHE_BACKEND=d1-r2 +GENERATION_CACHE_SHARED_ENABLED=true +``` + +Each caller must also send `X-MultiLLM-Cache: on`. Both settings and the request header are required. Empty settings use the defaults; malformed settings disable shared caching and emit one content-free warning. `RESPONSE_CACHE_ENABLED=false` disables the Flask response cache. Native caching applies to supported native Chat Completions endpoints; other native protocols and raw requests without the header retain their normal transport. Flask and native caches have separate policy identities and do not reuse one another's entries. + +Authentication, key scopes, model permission checks and retention policy run before lookup. Identity includes the canonical full request, authenticated principal and key digest, route, model, effective server policy and workflow version. Native identity also includes the upstream endpoint, protocol organization/project headers and current configuration revisions when revision synchronization is enabled. Requests with ambiguous provider-selection headers, separate upstream credentials, session identifiers or query parameters bypass shared caching. Policy changes during lookup prevent replay; changes during dispatch prevent storage. + +Eligible requests are deterministic (`temperature: 0` or an integer seed), nonstreaming and single-choice. Tools or legacy functions must be absent or disabled with `tool_choice: none`. Only an HTTP 200 JSON response with a complete finish reason and no runnable tool output is stored. Errors, partial, filtered, malformed, streaming and oversized results are never stored. There is no semantic similarity matching or automatic request replay. + +An eligible response carries `X-MultiLLM-Cache: hit` or `miss` and `X-MultiLLM-Cache-Backend: d1-r2`. Hits also carry `Age`, `X-MultiLLM-Usage-Basis: cache-served` and `X-MultiLLM-Provider-Calls: 0`. The original body is preserved, including any original provider usage. Those original counts do not describe a new provider call. Accounting records the replay with a cache cost basis and zero new provider cost; it does not claim the original generation was free. Optional native ledger recording still requires `NATIVE_EDGE_METRICS_ENABLED=true`. + +`Cache-Control: no-store` bypasses reads and writes. `no-cache` or `X-MultiLLM-Cache: refresh` requests fresh dispatch; `max-age` can restrict acceptable entry age. Zero-content retention bypasses before content is read from or written to shared storage. It applies prospectively and does not erase entries from earlier requests. + +## Storage and failures + +Metadata lives in the `generation_cache` D1 table. Response bodies use the existing `multillm_media` R2 binding under `generation-cache/v1/`. No additional binding is required. Each principal is limited to 512 live entries and 16 MiB of live response bodies. One body may contain at most 1 MiB. TTL is 300 seconds. Transactional capacity checks reject new writes when full and allow replacement of an existing key within the byte budget. Full caches continue ordinary dispatch. + +R2 pointers are immutable and scoped to principal and exact key. Reads validate the body size, digest, metadata and completion shape. Missing or corrupt bodies are misses. Native storage reads and writes wait at most two seconds per operation; the Container private transport waits at most three seconds. The private `POST /v1/state/generation-cache` endpoint accepts only fixed `get`, `put` and bounded `prune` operations on the existing internal outbound transport. It is unavailable when shared caching is disabled. Missing bindings or schema return a sanitized JSON HTTP 503 from this private endpoint. Chat cache failures continue normal dispatch and preserve its actual response; no success is invented and no upstream retry is added. + +## Expired objects + +Schedule bounded calls to `cleanupGenerationCache(env, {limit: 100, cursor})` from the storage module, or the equivalent private `prune` operation. Continue with the returned cursor until it is null. Cleanup removes expired metadata and objects only under `generation-cache/v1/` with an expired `expires_at` custom metadata value. It never deletes unrelated media objects. Replaced and orphaned bodies expire after their original 300-second TTL, so physical bucket usage can temporarily exceed the live per-principal metadata budget until cleanup runs. R2 and D1 storage operations can incur their normal service charges. Migration deployment and scheduled cleanup are operator actions; enabling the feature does not apply SQL or install a schedule. diff --git a/intelligence-migrations/0019_generation_cache.sql b/intelligence-migrations/0019_generation_cache.sql new file mode 100644 index 00000000..159c6f56 --- /dev/null +++ b/intelligence-migrations/0019_generation_cache.sql @@ -0,0 +1,16 @@ +-- Shared exact generation metadata; complete response bodies live in the media bucket. +CREATE TABLE IF NOT EXISTS generation_cache ( + principal_hash TEXT NOT NULL, + cache_key TEXT NOT NULL, + policy_hash TEXT NOT NULL, + model TEXT NOT NULL, + created_at REAL NOT NULL, + expires_at REAL NOT NULL, + body_pointer TEXT NOT NULL, + body_bytes INTEGER NOT NULL CHECK (body_bytes >= 1 AND body_bytes <= 1048576), + body_hash TEXT NOT NULL, + metadata TEXT NOT NULL, + PRIMARY KEY (principal_hash, cache_key), + CHECK (expires_at > created_at AND expires_at <= created_at + 300) +); +CREATE INDEX IF NOT EXISTS generation_cache_expiry ON generation_cache (expires_at); diff --git a/package.json b/package.json index 9281d289..470abf5a 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "lint": "ruff check . --select E9,F63,F7,F82", "typecheck": "mypy --ignore-missing-imports --follow-imports=skip --disable-error-code import-untyped --disable-error-code var-annotated --disable-error-code arg-type --disable-error-code assignment --disable-error-code operator app.py routes services providers streaming", "test": "python -m pytest -q", - "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_deferred_tools_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs", + "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_shared_generation_cache_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_deferred_tools_worker.mjs", "db:migrate": "python scripts/validate_sqlite_schema.py --apply", "db:validate": "python scripts/validate_sqlite_schema.py", "security:secrets": "python scripts/check_static_secrets.py", diff --git a/route_helpers.py b/route_helpers.py index cd72a0b8..3d1e958d 100644 --- a/route_helpers.py +++ b/route_helpers.py @@ -50,7 +50,7 @@ "X-MultiLLM-Auto-Route, X-MultiLLM-Auto-Selected-Model, " "X-MultiLLM-Auto-Attempts, X-MultiLLM-Auto-Selected-Priority, " "X-MultiLLM-Auto-Ordering, X-MultiLLM-Auto-Failover-Reasons, " - "X-MultiLLM-Cache, X-MultiLLM-Retention, X-MultiLLM-Roleplay-Recovery, X-MultiLLM-Transport-Failure, X-MultiLLM-Token-Count, " + "X-MultiLLM-Cache, X-MultiLLM-Cache-Backend, X-MultiLLM-Usage-Basis, X-MultiLLM-Provider-Calls, Age, X-MultiLLM-Retention, X-MultiLLM-Roleplay-Recovery, X-MultiLLM-Transport-Failure, X-MultiLLM-Token-Count, " "X-MultiLLM-Latency-Ms, X-MultiLLM-Estimated-Cost-USD, " "X-MultiLLM-Cost-Basis, WWW-Authenticate, X-PAYMENT-RESPONSE, X-Poll-After, " "X-NanoGPT-Advisor-ID, X-NanoGPT-Data-Endpoint, " diff --git a/routes/chat_cache.py b/routes/chat_cache.py index 02e36db9..12625366 100644 --- a/routes/chat_cache.py +++ b/routes/chat_cache.py @@ -22,7 +22,8 @@ from route_helpers import request_api_key from services import cache_policy from services.retention_policy import request_policy -from services.cache_service import ResponseCache +from services.cache_service import ResponseCache, shared_response_cache +from services import shared_generation_cache as shared_cache CACHE_HEADER = "X-MultiLLM-Cache" _OPT_IN = frozenset({"on", "true", "1", "yes"}) @@ -140,20 +141,25 @@ def _hit(entry) -> Response: return response -def _store_if_complete(response: Response, key: str, settings: dict) -> None: +def _store_if_complete(response: Response, key: str, settings: dict, *, shared=None, identity=None) -> None: if (response.status_code != 200 or response.direct_passthrough or response.mimetype != "application/json"): return body = response.get_data() - if len(body) > settings["max_bytes"] // 8 or not _complete_answer(body): + if len(body) > (shared_cache.MAX_BODY_BYTES if shared else settings["max_bytes"] // 8) or not _complete_answer(body): return - _store.configure(max_entries=settings["max_entries"], max_bytes=settings["max_bytes"]) - _store.put(key, body, { + metadata = { "content_type": response.headers.get("Content-Type", "application/json"), "headers": {name: response.headers[name] for name in _STORED_HEADERS if name in response.headers}, "provider": getattr(g, "multillm_provider", None), "model": getattr(g, "multillm_model", None), - }, ttl_seconds=settings["ttl"]) + } + if shared is not None: + metadata["content_type"] = "application/json" + shared.put(key, body, metadata, **identity) + else: + _store.configure(max_entries=settings["max_entries"], max_bytes=settings["max_bytes"]) + _store.put(key, body, metadata, ttl_seconds=settings["ttl"]) def _mark(response, value: str): @@ -162,6 +168,16 @@ def _mark(response, value: str): return response +def _shared_headers(response, value): + response = _mark(response, value) + if isinstance(response, Response): + response.headers["X-MultiLLM-Cache-Backend"] = "d1-r2" + if value == "hit": + response.headers["X-MultiLLM-Usage-Basis"] = "cache-served" + response.headers["X-MultiLLM-Provider-Calls"] = "0" + return response + + def cached_chat_completion(view): """Wrap an authenticated Chat Completions view; apply it below the auth decorator.""" @@ -180,7 +196,12 @@ def wrapper(*args, **kwargs): return _mark(view(*args, **kwargs), "bypass") revision = cache_policy.policy_revision() policy = None - if revision == cache_policy.ISOLATED_REVISION: + use_shared = shared_cache.shared_enabled() + if use_shared and revision in {cache_policy.LEGACY_REVISION, cache_policy.ISOLATED_REVISION}: + policy = shared_cache.shared_policy(payload) + if policy is None: + return _mark(view(*args, **kwargs), "bypass") + elif revision == cache_policy.ISOLATED_REVISION: policy = cache_policy.resolve_policy(payload) if policy is None: return _mark(view(*args, **kwargs), "bypass") @@ -188,15 +209,24 @@ def wrapper(*args, **kwargs): return _mark(view(*args, **kwargs), "bypass") key = cache_key(principal, request.path, payload, policy=policy) policy_digest = cache_policy.policy_digest(policy) if policy is not None else None + shared = shared_response_cache() if use_shared else None + identity = shared_cache.identity(principal, payload, policy) if use_shared else None if mode == "on": - entry = _store.get(key, max_age=max_age) + entry = shared.get(key, max_age=max_age, **identity) if shared else _store.get(key, max_age=max_age) + if use_shared and entry is not None: + current = shared_cache.shared_policy(payload) + if current is None or cache_policy.policy_digest(current) != policy_digest: + entry = None if entry is not None: - return _hit(entry) + response = _hit(entry) + return _shared_headers(response, "hit") if use_shared else response response = view(*args, **kwargs) - if (isinstance(response, Response) and (policy_digest is None - or cache_policy.policy_is_current(payload, policy_digest))): - _store_if_complete(response, key, settings) - return _mark(response, "miss") + current = shared_cache.shared_policy(payload) if use_shared else None + unchanged = (current is not None and cache_policy.policy_digest(current) == policy_digest) if use_shared else ( + policy_digest is None or cache_policy.policy_is_current(payload, policy_digest)) + if isinstance(response, Response) and unchanged: + _store_if_complete(response, key, settings, shared=shared, identity=identity) + return _shared_headers(response, "miss") if use_shared else _mark(response, "miss") return wrapper diff --git a/services/cache_service.py b/services/cache_service.py index c1f83fd9..e5e20da0 100644 --- a/services/cache_service.py +++ b/services/cache_service.py @@ -145,3 +145,15 @@ def clear(self): def stats(self): with self._lock: return {"entries": len(self._entries), "bytes": self._bytes} + + +def shared_response_cache(): + """An explicit application collaborator, or the fixed private shared adapter.""" + from flask import current_app + from services.shared_generation_cache import SharedGenerationCache + + store = current_app.extensions.get("shared_generation_cache") + if store is None: + store = SharedGenerationCache() + current_app.extensions["shared_generation_cache"] = store + return store diff --git a/services/shared_generation_cache.py b/services/shared_generation_cache.py new file mode 100644 index 00000000..a9e64cd8 --- /dev/null +++ b/services/shared_generation_cache.py @@ -0,0 +1,201 @@ +"""Opt-in shared exact response adapter over the fixed private Worker endpoint.""" +from __future__ import annotations + +import base64 +import hashlib +import json +import logging +import os +import queue +import threading +import time +from functools import lru_cache + +import requests +from flask import g, request +from requests.adapters import HTTPAdapter + +from services import cache_policy +from services.retention_policy import request_policy + +MAX_BODY_BYTES = 1024 * 1024 +MAX_RPC_BYTES = 1_500_000 +TTL_SECONDS = 300 +ENDPOINT = "http://intelligence.internal/v1/state/generation-cache" +_HEADERS = frozenset({"X-MultiLLM-Auto-Route", "X-MultiLLM-Auto-Selected-Model", + "X-MultiLLM-Auto-Selected-Priority", "X-MultiLLM-Cascade", "X-MultiLLM-Auto-Ordering", + "X-MultiLLM-Prompt-Cache", "X-MultiLLM-Prompt-Cache-Mode"}) +_SLOTS = threading.BoundedSemaphore(4) +logger = logging.getLogger(__name__) + + +@lru_cache(maxsize=8) +def _warn_once(event): + logger.warning("Shared generation cache %s", event) + + +def shared_enabled(env=None): + env = os.environ if env is None else env + backend = env.get("GENERATION_CACHE_BACKEND", "").strip() or "memory" + flag = env.get("GENERATION_CACHE_SHARED_ENABLED", "").strip().lower() + if backend not in {"memory", "d1-r2"}: + _warn_once("invalid_backend") + return False + if flag not in {"", "false", "0", "no", "off", "true", "1", "yes", "on"}: + _warn_once("invalid_enabled_setting") + return False + return backend == "d1-r2" and flag in {"true", "1", "yes", "on"} + + +def shared_policy(payload): + """Resolve server policy even when legacy memory identity is selected.""" + retention = request_policy() + if not retention.allows_content: + return None + if not retention.enabled: + return cache_policy.resolve_policy(payload) + try: + # The legacy resolver deliberately rejects prospective retention modes. + # Build the same route/grant snapshot with this request's immutable policy. + from services.key_controls import model_patterns + from services.secret_firewall import scan_mode + from flask import current_app + if request.query_string or any(request.headers.get(name) for name in + ("X-Provider", "X-Model", "X-MultiLLM-Provider")): + return None + route = cache_policy._resolved_route(payload["model"]) + if route is None: + return None + route["workflow_settings"] = { + "config": {name: current_app.config.get(name) for name in cache_policy._WORKFLOW_CONFIG}, + "environment": {name: os.environ.get(name) for name in cache_policy._WORKFLOW_ENV}} + user = getattr(g, "authenticated_user", None) or {} + patterns = model_patterns(user) + workflow = getattr(g, "multillm_workflow_version", cache_policy.WORKFLOW_VERSION) + if not isinstance(workflow, str) or not workflow: + return None + policy = cache_policy.CachePolicy(route, sorted({p.lower() for p in patterns}) if patterns else None, + scan_mode(user), {"mode": retention.mode, "revision": retention.revision}, workflow) + cache_policy.policy_digest(policy) + return policy + except Exception: + return None + + +def valid_metadata(value): + return (isinstance(value, dict) and set(value) == {"content_type", "headers", "provider", "model"} + and value["content_type"] == "application/json" + and isinstance(value["headers"], dict) and not set(value["headers"]) - _HEADERS + and all(isinstance(v, str) and len(v) <= 1024 and not any(ord(c) < 32 or ord(c) == 127 for c in v) + for v in value["headers"].values()) + and all(value[k] is None or isinstance(value[k], str) and len(value[k]) <= 256 + and not any(ord(c) < 32 or ord(c) == 127 for c in value[k]) for k in ("provider", "model"))) + + +def complete_body(body): + try: + value = json.loads(body) + choices = value.get("choices") if isinstance(value, dict) and "error" not in value else None + return (isinstance(choices, list) and len(choices) == 1 + and isinstance(choices[0], dict) and choices[0].get("finish_reason") in {"stop", "end_turn", "stop_sequence", "eos"} + and isinstance(choices[0].get("message"), dict) and not choices[0]["message"].get("tool_calls") + and not choices[0]["message"].get("function_call")) + except (ValueError, UnicodeError, RecursionError): + return False + + +def _exchange(encoded, results, stopped, deadline): + try: + with requests.Session() as session: + session.trust_env = False + session.mount("http://", HTTPAdapter(max_retries=0)) + if stopped.is_set(): + return + with session.post(ENDPOINT, data=encoded, headers={"Content-Type": "application/json", + "Accept": "application/json", "Accept-Encoding": "identity"}, timeout=(1, 2), + allow_redirects=False, stream=True) as response: + if response.status_code != 200 or response.headers.get("Content-Encoding", "identity") != "identity": + raise ValueError("cache_unavailable") + if response.headers.get("Content-Type", "").split(";", 1)[0].strip() != "application/json": + raise ValueError("cache_unavailable") + data = bytearray() + for chunk in response.iter_content(chunk_size=4096): + if stopped.is_set() or time.monotonic() >= deadline or len(data) + len(chunk) > MAX_RPC_BYTES: + raise ValueError("cache_unavailable") + data.extend(chunk) + value = json.loads(data) + if not isinstance(value, dict) or value.get("version") != 1 or "error" in value: + raise ValueError("cache_unavailable") + results.put_nowait(value) + except Exception: + results.put_nowait(None) + finally: + _SLOTS.release() + + +def private_transport(value): + """Single submission with bounded wait, body, concurrency and no redirects.""" + encoded = json.dumps({**value, "version": 1}, separators=(",", ":"), allow_nan=False).encode() + if len(encoded) > MAX_RPC_BYTES or not _SLOTS.acquire(blocking=False): + return None + results, stopped = queue.Queue(maxsize=1), threading.Event() + deadline = time.monotonic() + 3 + worker = threading.Thread(target=_exchange, args=(encoded, results, stopped, deadline), daemon=True) + try: + worker.start() + except Exception: + _SLOTS.release() + return None + try: + return results.get(timeout=max(0, deadline - time.monotonic())) + except queue.Empty: + return None + finally: + stopped.set() + + +class SharedGenerationCache: + """Bodies cross the private transport; keys contain only digested identities.""" + def __init__(self, transport=None): + self.transport = transport or private_transport + + def _call(self, value): + try: + response = self.transport({"version": 1, **value}) + if isinstance(response, dict) and response.get("version") == 1 and "error" not in response: + return response + except Exception: + pass + _warn_once("storage_unavailable") + return {} + + def get(self, key, *, principal_hash, policy_hash, model, max_age=None): + response = self._call(dict(operation="get", cache_key=key, principal_hash=principal_hash, + policy_hash=policy_hash, model=model, max_age=max_age)) + entry = response.get("entry") + if not isinstance(entry, dict) or not valid_metadata(entry.get("metadata")): + return None + age, encoded = entry.get("age"), entry.get("body") + if (type(age) not in (int, float) or not 0 <= age < TTL_SECONDS or + max_age is not None and age > max_age or not isinstance(encoded, str) + or len(encoded) > ((MAX_BODY_BYTES + 2) // 3) * 4): + return None + try: + body = base64.b64decode(encoded, validate=True) + except (ValueError, UnicodeError): + return None + if len(body) > MAX_BODY_BYTES or not complete_body(body): + return None + return body, entry["metadata"], age + + def put(self, key, body, metadata, *, principal_hash, policy_hash, model): + if len(body) > MAX_BODY_BYTES or not valid_metadata(metadata) or not complete_body(body): + return False + response = self._call(dict(operation="put", cache_key=key, principal_hash=principal_hash, + policy_hash=policy_hash, model=model, body=base64.b64encode(body).decode("ascii"), metadata=metadata)) + return response.get("stored") is True + + +def identity(principal, payload, policy): + return {"principal_hash": hashlib.sha256(principal.encode()).hexdigest(), + "policy_hash": cache_policy.policy_digest(policy), "model": payload["model"]} diff --git a/tests/fixtures/secret_firewall_coverage.json b/tests/fixtures/secret_firewall_coverage.json index 92fe448d..d918327c 100644 --- a/tests/fixtures/secret_firewall_coverage.json +++ b/tests/fixtures/secret_firewall_coverage.json @@ -230,9 +230,11 @@ "worker/control-users-d1.mjs", "worker/cors-policy.mjs", "worker/d1-schema.mjs", + "worker/exact-generation-cache.mjs", "worker/fallback-page.mjs", "worker/gateway-extensions.mjs", "worker/gateway-lifecycle.mjs", + "worker/generation-cache-d1.mjs", "worker/gpt-image-moderation.mjs", "worker/health-schedule.mjs", "worker/intelligence-auth-d1.mjs", diff --git a/tests/test_shared_generation_cache.py b/tests/test_shared_generation_cache.py new file mode 100644 index 00000000..1d7ed009 --- /dev/null +++ b/tests/test_shared_generation_cache.py @@ -0,0 +1,167 @@ +"""Shared exact responses through authenticated Chat Completions and bounded storage.""" +import base64 +import importlib +import json +import os +import sqlite3 +from pathlib import Path +from unittest.mock import patch + +from tests.test_chat_cache import BODY, CACHED, chat_cache, completion +from tests.unified_api_test_case import UnifiedApiTestCase + +MIGRATION = "0019_generation_cache.sql" + + +class SharedCacheRouteTest(UnifiedApiTestCase): + def setUp(self): + for target in ("env_loader.load_runtime_env", "config.load_runtime_env"): + loader = patch(target) + loader.start() + self.addCleanup(loader.stop) + network = patch("requests.sessions.Session.request", side_effect=AssertionError("Unexpected network")) + network.start() + self.addCleanup(network.stop) + super().setUp() + os.environ.update(GENERATION_CACHE_BACKEND="d1-r2", GENERATION_CACHE_SHARED_ENABLED="true", + RESPONSE_CACHE_ENABLED="true", RESPONSE_CACHE_POLICY_REVISION="legacy", + CONTENT_RETENTION_ENABLED="false", SECRET_SCAN_DEFAULT="off") + module = importlib.import_module("services.shared_generation_cache") + self.rows, self.calls = {}, [] + + def transport(value): + self.calls.append(value) + key = (value["principal_hash"], value["cache_key"], value["policy_hash"], value["model"]) + if value["operation"] == "get": + return {"version": 1, "entry": self.rows.get(key)} + self.rows[key] = {"body": value["body"], "metadata": value["metadata"], "age": 0} + return {"version": 1, "stored": True} + + self.transport = transport + self.store_type = module.SharedGenerationCache + self.app.extensions["shared_generation_cache"] = self.store_type(transport) + chat_cache().clear() + + def tearDown(self): + chat_cache().clear() + super().tearDown() + + def post(self, body=BODY, headers=CACHED, reply=None): + with patch.object(self.app_module.ProxyService, "make_request", return_value=reply if reply is not None else completion()) as upstream: + response = self.client.post("/v1/chat/completions", headers=headers, json=body) + return response, upstream.call_count + + def test_two_runtime_instances_share_an_exact_complete_response_and_account_cache(self): + rows = [] + accounting = importlib.import_module("services.request_accounting") + with patch.object(accounting.usage_ledger.LEDGER, "record", side_effect=rows.append): + first, calls = self.post() + self.assertEqual(calls, 1) + self.app.extensions["shared_generation_cache"] = self.store_type(self.transport) + chat_cache().clear() + hit, calls = self.post(body=dict(reversed(list(BODY.items())))) + self.assertEqual(calls, 0) + self.assertEqual(hit.data, first.data) + self.assertEqual(hit.headers["X-MultiLLM-Cache"], "hit") + self.assertEqual(hit.headers["X-MultiLLM-Cache-Backend"], "d1-r2") + self.assertEqual((rows[-1]["cost_usd"], rows[-1]["cost_basis"]), (0, "cache")) + self.assertEqual(hit.headers["X-MultiLLM-Usage-Basis"], "cache-served") + self.assertEqual(hit.headers["X-MultiLLM-Provider-Calls"], "0") + + def test_policy_model_principal_and_authentication_isolation(self): + self.post() + os.environ["SECRET_SCAN_DEFAULT"] = "block" + self.assertEqual(self.post()[1], 1) + self.assertEqual(self.post(body={**BODY, "model": "opencode:glm-5.3"})[1], 1) + auth = self.app_module.AuthService + with patch.object(auth, "get_current_user", return_value={"username": "admin", "is_admin": True}): + key = auth.create_user("shared-reader", is_admin=False, scopes=["chat"])["api_key"] + self.assertEqual(self.post(headers={**CACHED, "Authorization": f"Bearer {key}"})[1], 1) + before = len(self.calls) + response, calls = self.post(headers={**CACHED, "Authorization": "Bearer wrong"}) + self.assertEqual((response.status_code, calls, len(self.calls)), (401, 0, before)) + + def test_retention_and_ineligible_requests_never_touch_storage(self): + os.environ["CONTENT_RETENTION_ENABLED"] = "true" + for body, headers in ((BODY, {**CACHED, "X-MultiLLM-Retention": "zero"}), + ({**BODY, "temperature": 1}, CACHED), + ({**BODY, "tools": [{"type": "function"}]}, CACHED), + (BODY, {**CACHED, "Cache-Control": "no-store"})): + self.post(body=body, headers=headers) + self.assertEqual(self.calls, []) + + def test_inherit_retention_can_cache_and_revision_invalidates(self): + os.environ["CONTENT_RETENTION_ENABLED"] = "true" + self.post() + self.assertEqual(self.post()[1], 0) + os.environ["CONTENT_RETENTION_POLICY_JSON"] = '{"routes":{"/other":"zero"}}' + self.assertEqual(self.post()[1], 1) + + def test_errors_incomplete_and_oversized_results_are_not_stored(self): + for bad in (completion(status=500), completion(finish_reason="length"), + completion(tool_calls=[{"id": "call"}]), completion("x" * (1024 * 1024))): + self.rows.clear() + self.post(reply=bad) + self.assertEqual(self.post()[1], 1) + + def test_storage_outage_is_an_ordinary_dispatch_and_policy_changes_do_not_store(self): + def broken(value): + raise RuntimeError("private storage detail") + self.app.extensions["shared_generation_cache"] = self.store_type(broken) + self.assertEqual(self.post()[1], 1) + self.assertEqual(self.post()[1], 1) + self.app.extensions["shared_generation_cache"] = self.store_type(self.transport) + original = self.app_module.ProxyService.make_request + def dispatch(*args, **kwargs): + os.environ["SECRET_SCAN_DEFAULT"] = "block" + return completion() + with patch.object(self.app_module.ProxyService, "make_request", side_effect=dispatch): + self.client.post("/v1/chat/completions", headers=CACHED, json=BODY) + self.assertFalse(self.rows) + self.assertIsNotNone(original) + + def test_policy_change_during_lookup_cannot_replay_old_response(self): + self.post() + def changed(value): + entry = self.transport(value) + if value["operation"] == "get": + os.environ["SECRET_SCAN_DEFAULT"] = "block" + return entry + self.app.extensions["shared_generation_cache"] = self.store_type(changed) + self.assertEqual(self.post()[1], 1) + + def test_default_empty_disabled_and_malformed_settings_preserve_memory_headers(self): + for backend, enabled in (("memory", "true"), ("", ""), ("d1-r2", "false"), ("bad", "true"), ("d1-r2", "bad")): + os.environ.update(GENERATION_CACHE_BACKEND=backend, GENERATION_CACHE_SHARED_ENABLED=enabled) + chat_cache().clear() + first, _ = self.post() + hit, calls = self.post() + self.assertEqual((hit.headers["X-MultiLLM-Cache"], calls), ("hit", 0)) + self.assertEqual(first.data, hit.data) + self.assertNotIn("X-MultiLLM-Cache-Backend", hit.headers) + self.assertFalse(self.calls) + + +def test_shared_adapter_validates_response_size_age_and_metadata(): + module = importlib.import_module("services.shared_generation_cache") + body = json.dumps({"choices": [{"message": {"content": "ok"}, "finish_reason": "stop"}]}).encode() + metadata = {"content_type": "application/json", "headers": {}, "provider": None, "model": None} + identity = dict(principal_hash="a" * 64, policy_hash="b" * 64, model="fixture:model") + result = {"version": 1, "entry": {"body": base64.b64encode(body).decode(), "metadata": metadata, "age": 2}} + store = module.SharedGenerationCache(lambda value: result) + assert store.get("c" * 64, **identity)[0] == body + assert store.get("c" * 64, max_age=0, **identity) is None + result["entry"]["metadata"]["headers"] = {"Set-Cookie": "private"} + assert store.get("c" * 64, **identity) is None + assert not store.put("c" * 64, b"x" * (1024 * 1024 + 1), metadata, **identity) + + +def test_additive_migration_rehearsal_preserves_old_rows(): + db = sqlite3.connect(":memory:") + db.execute("CREATE TABLE old_rows (id INTEGER)") + db.execute("INSERT INTO old_rows VALUES (7)") + sql = (Path(__file__).parents[1] / "intelligence-migrations" / MIGRATION).read_text() + db.executescript(sql) + db.executescript(sql) + assert db.execute("SELECT id FROM old_rows").fetchone() == (7,) + assert db.execute("SELECT COUNT(*) FROM generation_cache").fetchone() == (0,) diff --git a/tests/test_shared_generation_cache_worker.mjs b/tests/test_shared_generation_cache_worker.mjs new file mode 100644 index 00000000..574ee176 --- /dev/null +++ b/tests/test_shared_generation_cache_worker.mjs @@ -0,0 +1,217 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { readFileSync } from "node:fs"; +import { DatabaseSync } from "node:sqlite"; +import { GenerationCacheD1, cleanupGenerationCache, CACHE_PREFIX } from "../worker/generation-cache-d1.mjs"; +import { generationCacheSettings, exactCacheIdentity, eligibleRequest, prepareNativeCache } from "../worker/exact-generation-cache.mjs"; +import { completeCacheBody } from "../worker/generation-cache-d1.mjs"; +import { nativeGenerationFetch } from "../worker/gateway-extensions.mjs"; +import { loadWorkerModule } from "./helpers/load_cloudflare_worker.mjs"; +import { handleControlStateRequest } from "../worker/control-state-d1.mjs"; + +const MIGRATION = "0019_generation_cache.sql"; +function database() { + const db = new DatabaseSync(":memory:"); + db.exec(readFileSync(new URL(`../intelligence-migrations/${MIGRATION}`, import.meta.url), "utf8")); + const execute = (sql, args) => { + const numbered = /\?[1-9]/.test(sql); + const statement = db.prepare(sql.replace(/\?(\d+)/g, (_, number) => `$v${number}`)); + const rows = numbered ? statement.all(Object.fromEntries(args.map((value, index) => [`v${index + 1}`, value]))) : statement.all(...args); + return {results: rows.map(row => ({...row})), meta: {changes: db.prepare("SELECT changes() AS n").get().n}}; + }; + return { prepare(sql) { return { sql, args: [], bind(...args) {this.args = args; return this;}, + async first() {return execute(sql, this.args).results[0] ?? null;}, + async all() {return execute(sql, this.args);}, async run() {return execute(sql, this.args);} }; }, + async batch(items) { + db.exec("BEGIN IMMEDIATE"); + try {const result = items.map(item => execute(item.sql, item.args)); db.exec("COMMIT"); return result;} + catch (error) {db.exec("ROLLBACK"); throw error;} + } }; +} +function bucket() { + const rows = new Map(); let writes = 0; + return { rows, get writes() { return writes; }, async put(key, value, options) { + writes++; rows.set(key, { value: new Uint8Array(value), customMetadata: options.customMetadata }); + }, async get(key) { const row = rows.get(key); return row ? { size: row.value.length, + async arrayBuffer() { return row.value.slice().buffer; } } : null; }, + async delete(key) { rows.delete(key); }, async list() { return { objects: [...rows].map(([key, row]) => ({key, customMetadata: row.customMetadata})), truncated: false }; } }; +} +const enabled = { GENERATION_CACHE_BACKEND: "d1-r2", GENERATION_CACHE_SHARED_ENABLED: "true", ADMIN_USERNAME: "test", ADMIN_API_KEY: "synthetic" }; +const payload = { model: "fixture-model", temperature: 0, messages: [{ role: "user", content: "hello" }] }; +const body = JSON.stringify({ choices: [{ message: { role: "assistant", content: "ok" }, finish_reason: "stop" }], usage: { prompt_tokens: 3, completion_tokens: 1 } }); +const metadata = { content_type: "application/json", headers: {}, provider: "fixture", model: "fixture-model" }; +const identity = { principal_hash: "a".repeat(64), cache_key: "b".repeat(64), policy_hash: "c".repeat(64), model: "fixture-model" }; +function request(changes = {}, headers = {}) { return new Request("https://provider.invalid/v1/chat/completions", { + method: "POST", headers: { "content-type": "application/json", "X-MultiLLM-Cache": "on", ...headers }, body: JSON.stringify({ ...payload, ...changes }) }); } +const authority = { route: "/fixture/v1/chat/completions", provider: "fixture", principal: { id: "test" } }; + +// SQLite executes the actual fixed statements, including transaction bounds, without a Worker daemon. +test("D1 and R2 share exact entries across instances with TTL and dangling-pointer misses", async () => { + const env = { ...enabled, INTELLIGENCE_DB: database(), multillm_media: bucket() }; let now = 1000; + const one = new GenerationCacheD1(env, { clock: () => now }); + const two = new GenerationCacheD1(env, { clock: () => now }); + assert.equal(await one.put(identity, new TextEncoder().encode(body), metadata), true); + assert.equal(new TextDecoder().decode((await two.get(identity)).body), body); + assert.equal(await two.get({ ...identity, principal_hash: "d".repeat(64) }), null); + assert.equal(await two.get({ ...identity, policy_hash: "d".repeat(64) }), null); + assert.equal(await two.get({ ...identity, model: "other" }), null); + now += 1; assert.equal(await two.get(identity, { maxAge: 0 }), null); + const stored = [...env.multillm_media.rows][0]; env.multillm_media.rows.delete(stored[0]); + assert.equal(await two.get(identity), null); env.multillm_media.rows.set(...stored); + now = 1300; assert.equal(await two.get(identity), null); + env.multillm_media.rows.set("unrelated", { value: new Uint8Array(), customMetadata: { expires_at: "0" } }); + await cleanupGenerationCache(env, { now, limit: 100 }); + assert.equal(env.multillm_media.rows.has("unrelated"), true); + assert.equal([...env.multillm_media.rows.keys()].some(key => key.startsWith(CACHE_PREFIX)), false); +}); + +test("transaction limits reject over-budget writes and preserve other principals", async () => { + const env = { ...enabled, INTELLIGENCE_DB: database(), multillm_media: bucket() }; + const store = new GenerationCacheD1(env); + const size = new TextEncoder().encode(body).length; + const now = Date.now() / 1000; + await env.INTELLIGENCE_DB.batch(Array.from({length: 512}, (_, index) => env.INTELLIGENCE_DB.prepare( + "INSERT INTO generation_cache VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)").bind(identity.principal_hash, index.toString(16).padStart(64, "0"), identity.policy_hash, + identity.model, now, now + 300, CACHE_PREFIX + "fixture", size, "d".repeat(64), JSON.stringify(metadata)))); + assert.equal(await store.put(identity, new TextEncoder().encode(body), metadata), false); + assert.equal(await store.put({ ...identity, principal_hash: "d".repeat(64) }, new TextEncoder().encode(body), metadata), true); + assert.equal(await store.put(identity, new Uint8Array(1024 * 1024 + 1), metadata), false); + await env.INTELLIGENCE_DB.prepare("DELETE FROM generation_cache WHERE principal_hash = ?").bind(identity.principal_hash).run(); + await env.INTELLIGENCE_DB.batch(Array.from({length: 16}, (_, index) => env.INTELLIGENCE_DB.prepare( + "INSERT INTO generation_cache VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)").bind(identity.principal_hash, index.toString(16).padStart(64, "0"), identity.policy_hash, + identity.model, now, now + 300, CACHE_PREFIX + "fixture", 1024 * 1024, "d".repeat(64), JSON.stringify(metadata)))); + assert.equal(await store.put(identity, new TextEncoder().encode(body), metadata), false); +}); + +test("native registered lifecycle replays exact bytes without dispatch and finalizes cache accounting", async () => { + const env = { ...enabled, INTELLIGENCE_DB: database(), multillm_media: bucket() }; const events = []; + const hooks = [{ enabled: () => true, finalize: event => events.push(event) }]; let calls = 0; + const fetcher = async () => { calls++; return new Response(body, { headers: {"content-type": "application/json"} }); }; + const first = await nativeGenerationFetch(request(), env, null, authority, fetcher, hooks); assert.equal(await first.text(), body); + const hit = await nativeGenerationFetch(request(), {...env}, null, authority, fetcher, hooks); assert.equal(await hit.text(), body); + assert.equal(calls, 1); assert.equal(hit.headers.get("X-MultiLLM-Cache"), "hit"); + assert.equal(hit.headers.get("X-MultiLLM-Cache-Backend"), "d1-r2"); + assert.equal(events.at(-1).usage_basis, "cache-served"); assert.equal(events.at(-1).provider_calls, 0); + assert.equal(events.at(-1).cost_basis, "cache"); assert.equal(events.at(-1).cost_usd, 0); + assert.equal(JSON.parse(body).usage.prompt_tokens, 3); +}); + +test("native retention streams tools errors and partial results bypass before storage", async () => { + let touched = 0; + const env = { ...enabled, CONTENT_RETENTION_ENABLED: "true", INTELLIGENCE_DB: {prepare() {touched++; throw Error();}}, multillm_media: bucket() }; + for (const [changes, headers] of [[{}, {"X-MultiLLM-Retention": "zero"}], [{stream: true}, {}], [{tools: [{type: "function"}]}, {}], [{temperature: 1}, {}]]) { + const result = await nativeGenerationFetch(request(changes, headers), env, null, authority, + async () => new Response(body, {headers: {"content-type": "application/json"}})); + await result.text(); + } + assert.equal(touched, 0); assert.equal(env.multillm_media.writes, 0); + const valid = { ...enabled, INTELLIGENCE_DB: database(), multillm_media: bucket() }; + for (const response of [new Response(body, {status: 500}), new Response(body.replace('"stop"', '"length"'), {headers: {"content-type": "application/json"}}), + new Response('{"choices":[{"message":{"tool_calls":[{}]},"finish_reason":"stop"}]}', {headers: {"content-type": "application/json"}})]) { + await (await nativeGenerationFetch(request(), valid, null, authority, async () => response)).text(); + } + assert.equal(valid.multillm_media.writes, 0); +}); + +test("private dispatch is default off, validates inputs and returns sanitized 503 for missing schema", async () => { + const rpc = value => new Request("http://intelligence.internal/v1/state/generation-cache", {method: "POST", + headers: {"content-type": "application/json"}, body: JSON.stringify({version: 1, operation: "get", ...identity, max_age: null, ...value})}); + assert.equal((await handleControlStateRequest(rpc(), {})).status, 404); + const broken = { ...enabled, INTELLIGENCE_DB: { prepare() {throw Error("private storage detail");}}, multillm_media: bucket() }; + const failed = await handleControlStateRequest(rpc(), broken); assert.equal(failed.status, 503); + assert.doesNotMatch(await failed.text(), /private storage detail/); + assert.equal((await handleControlStateRequest(rpc({principal_hash: "invalid"}), broken)).status, 400); + const env = { ...enabled, INTELLIGENCE_DB: database(), multillm_media: bucket() }; + const saved = await handleControlStateRequest(rpc({operation: "put", max_age: undefined, body: Buffer.from(body).toString("base64"), metadata}), env); + assert.equal(saved.status, 200); assert.equal((await saved.json()).stored, true); + const hit = await handleControlStateRequest(rpc(), env); assert.equal(Buffer.from((await hit.json()).entry.body, "base64").toString(), body); + let calls = 0; + const result = await nativeGenerationFetch(request(), broken, null, authority, async () => {calls++; return new Response(body);}); + assert.equal(await result.text(), body); assert.equal(calls, 1); +}); + +test("empty malformed default settings preserve raw native response and do not inspect cache", async () => { + for (const changes of [{}, {GENERATION_CACHE_BACKEND: "", GENERATION_CACHE_SHARED_ENABLED: ""}, + {...enabled, GENERATION_CACHE_SHARED_ENABLED: "bad"}, {...enabled, GENERATION_CACHE_BACKEND: "bad"}]) { + const env = {...changes, INTELLIGENCE_DB: {prepare() {assert.fail("cache inspected");}}}; + const raw = new Response("unchanged", {headers: {"x-fixture": "original"}}); + const result = await nativeGenerationFetch(request(), env, null, authority, async () => raw); + assert.equal(await result.text(), "unchanged"); assert.equal(result.headers.get("x-fixture"), "original"); + assert.equal(result.headers.get("X-MultiLLM-Cache"), null); + } + assert.equal(generationCacheSettings(enabled).enabled, true); +}); + +test("native canonical identity isolates principal policy endpoint and model", async () => { + const context = {retentionPolicy: {mode: "inherit", enabled: false}}; + const one = await exactCacheIdentity(request(), enabled, authority, context); + assert.equal(one.cache_key, (await exactCacheIdentity(request(), enabled, authority, context)).cache_key); + for (const [req, env, auth] of [[request({model: "other"}), enabled, authority], [request(), {...enabled, SECRET_SCAN_DEFAULT: "block"}, authority], + [request(), {...enabled, ADMIN_API_KEY: "other-synthetic"}, authority], [request(), enabled, {...authority, principal: {id: "other"}}]]) { + assert.notDeepEqual(await exactCacheIdentity(req, env, auth, context), one); + } +}); + + +test("authorized hub routes expose provenance and a bad key cannot read an entry", async () => { + const worker = (await loadWorkerModule()).default; + const env = {...enabled, INTELLIGENCE_DB: database(), multillm_media: bucket(), CODEX_EASY_API_KEY: "synthetic-upstream"}; + let calls = 0; const original = globalThis.fetch; + globalThis.fetch = async () => {calls++; return new Response(body, {headers: {"content-type": "application/json"}});}; + const source = (token = env.ADMIN_API_KEY) => new Request("https://gateway.example/codex-easy/v1/chat/completions", { + method: "POST", headers: {Authorization: `Bearer ${token}`, "content-type": "application/json", "X-MultiLLM-Cache": "on"}, body: JSON.stringify(payload)}); + try { + const first = await worker.fetch(source(), env); assert.equal(await first.text(), body); + assert.equal(first.headers.get("X-MultiLLM-Cache"), "miss"); + const hit = await worker.fetch(source(), env); assert.equal(await hit.text(), body); + assert.equal(hit.headers.get("X-MultiLLM-Cache"), "hit"); assert.equal(hit.headers.get("X-MultiLLM-Provider-Calls"), "0"); + assert.equal((await worker.fetch(source("wrong"), env)).status, 401); assert.equal(calls, 1); + } finally {globalThis.fetch = original;} +}); + +test("concurrent writes respect capacity and corrupt R2 bytes never replay", async () => { + const env = {...enabled, INTELLIGENCE_DB: database(), multillm_media: bucket()}; const store = new GenerationCacheD1(env); + await Promise.all(Array.from({length: 20}, (_, index) => store.put({...identity, cache_key: index.toString(16).padStart(64, "0")}, new TextEncoder().encode(body), metadata))); + assert.equal((await env.INTELLIGENCE_DB.prepare("SELECT COUNT(*) AS n FROM generation_cache").first()).n, 20); + await store.put(identity, new TextEncoder().encode(body), metadata); + const row = await env.INTELLIGENCE_DB.prepare("SELECT body_pointer FROM generation_cache WHERE cache_key=?").bind(identity.cache_key).first(); + const saved = env.multillm_media.rows.get(row.body_pointer); saved.value[0] ^= 1; + assert.equal(await store.get(identity), null); +}); + +test("native cancellation and refresh never replay or authorize another dispatch", async () => { + const env = {...enabled, INTELLIGENCE_DB: database(), multillm_media: bucket()}; let calls = 0; + const fetcher = async () => {calls++; return new Response(body, {headers: {"content-type": "application/json"}});}; + await (await nativeGenerationFetch(request(), env, null, authority, fetcher)).text(); + await (await nativeGenerationFetch(request({}, {"Cache-Control": "no-cache"}), env, null, authority, fetcher)).text(); + assert.equal(calls, 2); + const controller = new AbortController(); controller.abort(); + const canceled = new Request(request(), {signal: controller.signal}); + await assert.rejects(nativeGenerationFetch(canceled, env, null, authority, fetcher)); assert.equal(calls, 2); +}); + +test("ambiguous upstream credentials and malformed tools never enter the cache", async () => { + const context = {retentionPolicy: {mode: "inherit", enabled: false}}; + for (const header of ["x-multillm-api-key", "x-opencode-session", "x-grok-conv-id"]) { + assert.equal(await exactCacheIdentity(request({}, {[header]: "synthetic"}), enabled, authority, context), null); + } + for (const tools of [{type: "function"}, "invalid", false]) { + assert.equal(eligibleRequest({...payload, tools}), false); + } + assert.equal(completeCacheBody(new TextEncoder().encode(JSON.stringify({choices: [{finish_reason: "stop", message: {tool_calls: {id: "tool"}}}]}))), false); +}); + +test("stalled lookup falls through and a changed policy cannot replay a stored entry", async () => { + const context = {retentionPolicy: {mode: "inherit", enabled: false}, cacheRevisions: {security: 1}}; + const stuck = {...enabled, INTELLIGENCE_DB: {prepare() {return {bind() {return this;}, first() {return new Promise(() => {});}};}}}; + const started = performance.now(); + assert.equal(await (await prepareNativeCache(request(), stuck, authority, context)).lookup(), null); + assert.ok(performance.now() - started < 4000); + const env = {...enabled, INTELLIGENCE_DB: database(), multillm_media: bucket()}; + const original = await exactCacheIdentity(request(), env, authority, context); + await new GenerationCacheD1(env).put(original, new TextEncoder().encode(body), metadata); + const cache = await prepareNativeCache(request(), env, authority, context); + const get = env.multillm_media.get; + env.multillm_media.get = async key => {context.cacheRevisions.security++; return get(key);}; + assert.equal(await cache.lookup(), null); +}); diff --git a/worker/container-env.mjs b/worker/container-env.mjs index 17a010fd..b3b27f75 100644 --- a/worker/container-env.mjs +++ b/worker/container-env.mjs @@ -28,6 +28,8 @@ const DIRECT_ENV_KEYS = [ "WORKBENCH_WORKER_URL", "MCP_CONTRACT_DIGESTS_ENABLED", "DEFERRED_TOOLS_ENABLED", + "GENERATION_CACHE_BACKEND", + "GENERATION_CACHE_SHARED_ENABLED", "CONNECTION_PROFILES_DB_PATH", "CONFIG_REVISION_SYNC_ENABLED", "CONFIG_SYNC_TTL_SECONDS", diff --git a/worker/control-state-d1.mjs b/worker/control-state-d1.mjs index b17b6c28..8550972f 100644 --- a/worker/control-state-d1.mjs +++ b/worker/control-state-d1.mjs @@ -5,6 +5,8 @@ * Container's private outbound handler at /v1/state/. Fixed statements, no * client SQL. */ +import { handleGenerationCache, MAX_RPC_BYTES } from "./generation-cache-d1.mjs"; +import { generationCacheSettings } from "./exact-generation-cache.mjs"; import { boundedBody } from "./control-users-d1.mjs"; import { logFailure } from "./log.mjs"; import { handlePromptTemplates, promptTemplatesEnabled } from "./prompt-templates-d1.mjs"; @@ -267,6 +269,7 @@ async function catalog(db, body, env) { } const DOMAINS = Object.freeze({ + "generation-cache": { handle: handleGenerationCache, maxBytes: MAX_RPC_BYTES, operations: ["get", "put", "prune"] }, "prompt-templates": { handle: handlePromptTemplates, maxBytes: 524288, operations: ["create", "get", "list"] }, limits: { handle: limits, maxBytes: 16384, operations: ["sync"] }, login: { handle: login, maxBytes: 4096, operations: ["check", "failure", "success"] }, @@ -282,6 +285,7 @@ export async function handleControlStateRequest(request, env) { const domain = Object.hasOwn(DOMAINS, name) ? DOMAINS[name] : null; if (request.method !== "POST" || url.origin !== "http://intelligence.internal" || !domain || url.search || url.hash || url.username || url.password) return reply({ error: "not_found" }, 404); + if (name === "generation-cache" && !generationCacheSettings(env).enabled) return reply({ error: "not_found" }, 404); if (name === "prompt-templates" && !promptTemplatesEnabled(env)) return reply({ error: "not_found" }, 404); if (request.headers.get("content-type")?.split(";", 1)[0].trim().toLowerCase() !== "application/json") { return reply({ error: "invalid_request" }, 400); @@ -297,7 +301,7 @@ export async function handleControlStateRequest(request, env) { try { return await domain.handle(env.INTELLIGENCE_DB, body, env) ?? reply({ error: "invalid_request" }, 400); } catch (error) { - logFailure("control_state_storage_failed", revisionSyncSettings(env).enabled ? new Error("Revision storage unavailable") : error, { domain: name, + logFailure("control_state_storage_failed", name === "generation-cache" || revisionSyncSettings(env).enabled ? new Error("Revision storage unavailable") : error, { domain: name, operation: domain.operations.includes(body.operation) ? body.operation : "unknown" }); return reply({ error: "storage_unavailable" }, 503); } diff --git a/worker/cors-policy.mjs b/worker/cors-policy.mjs index fbe48ac9..46f072bf 100644 --- a/worker/cors-policy.mjs +++ b/worker/cors-policy.mjs @@ -3,4 +3,4 @@ export const CORS_DEFAULT_HEADERS = "X-MultiLLM-Tool-Repair, X-MultiLLM-Image-QA, X-MultiLLM-Cascade, X-MultiLLM-Conversion-Report, X-MultiLLM-Retention, Authorization, X-Api-Key, X-Goog-Api-Key, X-MultiLLM-Api-Key, X-Roleplay-Session-ID, Anthropic-Version, Anthropic-Beta, Anthropic-Dangerous-Direct-Browser-Access, Content-Type, Accept, Origin, X-Requested-With, OpenAI-Beta, OpenAI-Organization, OpenAI-Project, Idempotency-Key, Moderation, Moderation-Model, Redaction, X-Client-Request-ID, X-App-Name, X-Billing-Mode, X-BYOK-Provider, X-Encryption-Key, X-Encryption-Passphrase, X-Fal-Object-Lifecycle-Preference, X-PAYMENT, X-Prompt-Caching-Cut-After, X-Provider, X-Team-ID, X-Use-BYOK, x-x402"; export const CORS_EXPOSE_HEADERS = "X-MultiLLM-Secret-Scan, X-MultiLLM-Retention, X-MultiLLM-Roleplay-Recovery, " + - "X-MultiLLM-Tool-Repair, X-MultiLLM-Image-QA, X-MultiLLM-Cascade, X-MultiLLM-Conversion-Fidelity, X-MultiLLM-Conversion-Fields, Retry-After, X-Request-ID, X-MultiLLM-RateLimit-Limit, X-MultiLLM-RateLimit-Remaining, X-MultiLLM-RateLimit-Reset, X-MultiLLM-Optimization, X-MultiLLM-Optimization-Mode, X-MultiLLM-Estimated-Input-Before, X-MultiLLM-Estimated-Input-After, X-MultiLLM-Image-Prompts-Compacted, X-MultiLLM-Messages-Summarized, X-MultiLLM-Optimization-Target-Met, X-MultiLLM-Summary, X-MultiLLM-Optimization-Cache-Hits, X-MultiLLM-Optimization-Cache-Misses, X-MultiLLM-Prompt-Cache, X-MultiLLM-Prompt-Cache-Mode, X-MultiLLM-Prompt-Cache-Estimated-Tokens, X-MultiLLM-Provider, X-MultiLLM-Model, X-MultiLLM-Credential-Attempts, X-MultiLLM-Route-Decision, X-MultiLLM-Circuit-State, X-MultiLLM-Auto-Route, X-MultiLLM-Auto-Selected-Model, X-MultiLLM-Auto-Attempts, X-MultiLLM-Auto-Selected-Priority, X-MultiLLM-Auto-Ordering, X-MultiLLM-Auto-Failover-Reasons, X-MultiLLM-Cache, X-MultiLLM-Transport-Failure, X-MultiLLM-Token-Count, X-MultiLLM-Latency-Ms, X-MultiLLM-Estimated-Cost-USD, X-MultiLLM-Cost-Basis, X-MultiLLM-Reasoning-Normalized, X-Roleplay-Session-ID, X-Roleplay-Trace-ID, X-Roleplay-Session-Source, X-Roleplay-Provider, X-Roleplay-Model, X-Roleplay-Selection, X-Roleplay-Memory, X-Roleplay-Estimated-Input-Tokens, X-Roleplay-Max-Output-Tokens, X-Roleplay-Fallback-Count, X-Roleplay-State-Cache, X-Roleplay-Credential-Check, Server-Timing, WWW-Authenticate, X-PAYMENT-RESPONSE, X-Poll-After, X-NanoGPT-Advisor-ID, X-NanoGPT-Data-Endpoint, X-NanoGPT-Direct-Endpoint, X-NanoGPT-Inline-Moderation-Cost-USD, X-NanoGPT-Inline-Moderation-Flagged, X-NanoGPT-Inline-Moderation-Model"; + "X-MultiLLM-Tool-Repair, X-MultiLLM-Image-QA, X-MultiLLM-Cascade, X-MultiLLM-Conversion-Fidelity, X-MultiLLM-Conversion-Fields, Retry-After, X-Request-ID, X-MultiLLM-RateLimit-Limit, X-MultiLLM-RateLimit-Remaining, X-MultiLLM-RateLimit-Reset, X-MultiLLM-Optimization, X-MultiLLM-Optimization-Mode, X-MultiLLM-Estimated-Input-Before, X-MultiLLM-Estimated-Input-After, X-MultiLLM-Image-Prompts-Compacted, X-MultiLLM-Messages-Summarized, X-MultiLLM-Optimization-Target-Met, X-MultiLLM-Summary, X-MultiLLM-Optimization-Cache-Hits, X-MultiLLM-Optimization-Cache-Misses, X-MultiLLM-Prompt-Cache, X-MultiLLM-Prompt-Cache-Mode, X-MultiLLM-Prompt-Cache-Estimated-Tokens, X-MultiLLM-Provider, X-MultiLLM-Model, X-MultiLLM-Credential-Attempts, X-MultiLLM-Route-Decision, X-MultiLLM-Circuit-State, X-MultiLLM-Auto-Route, X-MultiLLM-Auto-Selected-Model, X-MultiLLM-Auto-Attempts, X-MultiLLM-Auto-Selected-Priority, X-MultiLLM-Auto-Ordering, X-MultiLLM-Auto-Failover-Reasons, X-MultiLLM-Cache, X-MultiLLM-Cache-Backend, X-MultiLLM-Usage-Basis, X-MultiLLM-Provider-Calls, Age, X-MultiLLM-Transport-Failure, X-MultiLLM-Token-Count, X-MultiLLM-Latency-Ms, X-MultiLLM-Estimated-Cost-USD, X-MultiLLM-Cost-Basis, X-MultiLLM-Reasoning-Normalized, X-Roleplay-Session-ID, X-Roleplay-Trace-ID, X-Roleplay-Session-Source, X-Roleplay-Provider, X-Roleplay-Model, X-Roleplay-Selection, X-Roleplay-Memory, X-Roleplay-Estimated-Input-Tokens, X-Roleplay-Max-Output-Tokens, X-Roleplay-Fallback-Count, X-Roleplay-State-Cache, X-Roleplay-Credential-Check, Server-Timing, WWW-Authenticate, X-PAYMENT-RESPONSE, X-Poll-After, X-NanoGPT-Advisor-ID, X-NanoGPT-Data-Endpoint, X-NanoGPT-Direct-Endpoint, X-NanoGPT-Inline-Moderation-Cost-USD, X-NanoGPT-Inline-Moderation-Flagged, X-NanoGPT-Inline-Moderation-Model"; diff --git a/worker/d1-schema.mjs b/worker/d1-schema.mjs index 64b5799f..7ba673b5 100644 --- a/worker/d1-schema.mjs +++ b/worker/d1-schema.mjs @@ -10,7 +10,7 @@ export const REQUIRED_D1_TABLES = Object.freeze(["intelligence_policy", "intelli "usage_events", "usage_daily", "usage_batches", "media_jobs", "media_job_items", "shadow_eval_samples", "shadow_eval_results", "shadow_eval_config", "shadow_eval_policy_backups", "prompt_templates", - "config_snapshot_revisions", "config_snapshots", "config_snapshot_applications", "control_revisions", "tool_grants"]); + "config_snapshot_revisions", "config_snapshots", "config_snapshot_applications", "control_revisions", "tool_grants", "generation_cache"]); /** * Readiness for the D1 schema. A deploy that skipped `wrangler d1 migrations apply` reports diff --git a/worker/exact-generation-cache.mjs b/worker/exact-generation-cache.mjs new file mode 100644 index 00000000..24394348 --- /dev/null +++ b/worker/exact-generation-cache.mjs @@ -0,0 +1,127 @@ +/** Opt-in exact Chat Completions caching after native authentication and retention. */ +import { GenerationCacheD1, MAX_BODY_BYTES, digest, completeCacheBody } from "./generation-cache-d1.mjs"; +import { retentionAllowsContent } from "./retention-policy.mjs"; +const warned = new Set(); +function warn(event) { if (!warned.has(event)) { warned.add(event); console.warn(JSON.stringify({event: `generation_cache_${event}`})); } } +export function generationCacheSettings(env = {}) { + const backend = String(env.GENERATION_CACHE_BACKEND ?? "").trim() || "memory"; + const flag = String(env.GENERATION_CACHE_SHARED_ENABLED ?? "").trim().toLowerCase(); + if (!["memory", "d1-r2"].includes(backend) || !["", "false", "0", "no", "off", "true", "1", "yes", "on"].includes(flag)) { + warn("invalid_setting"); return {enabled: false, backend: "memory"}; + } + return {enabled: backend === "d1-r2" && ["true", "1", "yes", "on"].includes(flag), backend}; +} +function canonical(value) { + if (Array.isArray(value)) return value.map(canonical); + if (value && typeof value === "object") return Object.fromEntries(Object.keys(value).sort().map(key => [key, canonical(value[key])])); + return value; +} +export function eligibleRequest(value) { + if ([value?.tools, value?.functions].some(tools => tools != null && !Array.isArray(tools))) return false; + return value && typeof value === "object" && !Array.isArray(value) && !value.stream && typeof value.model === "string" + && value.model !== "auto:intelligence" && !Object.hasOwn(value, "routing") && (value.n ?? 1) === 1 + && (!(value.tools?.length || value.functions?.length) || value.tool_choice === "none") + && (value.temperature === 0 || Number.isSafeInteger(value.seed)); +} +async function boundedBytes(stream, maximum) { + if (!stream) return new Uint8Array(); + const reader = stream.getReader(); const parts = []; let size = 0, timer; + const expired = new Promise(resolve => {timer = setTimeout(() => resolve(null), 2000);}); + try { + while (true) { + const item = await Promise.race([reader.read(), expired]); + if (!item) return null; + if (item.done) break; + size += item.value.length; if (size > maximum) return null; + parts.push(item.value); + } + const bytes = new Uint8Array(size); let offset = 0; + for (const part of parts) {bytes.set(part, offset); offset += part.length;} + return bytes; + } finally {clearTimeout(timer); void reader.cancel().catch(() => {});} +} +export async function exactCacheIdentity(request, env, authority, context) { + if (!retentionAllowsContent(context.retentionPolicy) || request.method !== "POST" + || !authority.route?.endsWith("/chat/completions") || new URL(request.url).search) return null; + const source = authority.cacheRequest ?? request; + if (new URL(source.url).search || ["x-api-key", "api-key", "x-multillm-api-key", "x-opencode-session", "x-grok-conv-id", + "x-provider", "x-model", "x-multillm-provider", "idempotency-key"] + .some(key => source.headers.has(key))) return null; + if (!authority.principal?.id || !env.ADMIN_API_KEY) return null; + const bytes = await boundedBytes(request.clone().body, MAX_BODY_BYTES); + if (!bytes) return null; + let payload; + try {payload = JSON.parse(new TextDecoder("utf-8", {fatal: true}).decode(bytes));} catch {return null;} + if (!eligibleRequest(payload)) return null; + const principal = await digest(`${authority.principal.id}\0${await digest(env.ADMIN_API_KEY)}`); + const policyHash = await nativePolicyHash(request, env, authority, context); + const key = await digest(JSON.stringify([principal, authority.route, policyHash, canonical(payload)])); + return {principal_hash: principal, cache_key: key, policy_hash: policyHash, model: payload.model}; +} +async function nativePolicyHash(request, env, authority, context) { + const source = authority.cacheRequest ?? request; + const revisions = typeof context.cacheRevisions === "function" ? context.cacheRevisions() : context.cacheRevisions ?? {}; + const policy = {version: "native-exact-v1", endpoint: request.url, provider: authority.provider, route: authority.route, + retention: context.retentionPolicy, retention_config: env.CONTENT_RETENTION_POLICY_JSON ?? "", + retention_enabled: env.CONTENT_RETENTION_ENABLED ?? "", revisions, + secret_policy: env.SECRET_SCAN_DEFAULT ?? "redact", secret_enabled: env.SECRET_SCAN_ENABLED ?? "", + principal_secret_policy: authority.principal.secret_scan_mode ?? null, + headers: Object.fromEntries(["openai-organization", "openai-project", "openai-beta", "anthropic-version", "anthropic-beta"] + .map(key => [key, source.headers.get(key)]))}; + return digest(JSON.stringify(canonical(policy))); +} +function mode(request) { + const value = request.headers.get("X-MultiLLM-Cache")?.trim().toLowerCase(); + const control = request.headers.get("Cache-Control")?.toLowerCase() ?? ""; + if (!["on", "true", "1", "yes", "refresh"].includes(value) || control.includes("no-store")) return null; + const age = control.match(/(?:^|,)\s*max-age\s*=\s*(\d{1,9})\s*(?:,|$)/); + return {refresh: value === "refresh" || control.includes("no-cache"), maxAge: age ? Number(age[1]) : null}; +} +function mark(response, value) { + const headers = new Headers(response.headers); + headers.set("X-MultiLLM-Cache", value); headers.set("X-MultiLLM-Cache-Backend", "d1-r2"); + headers.delete("X-MultiLLM-Usage-Basis"); headers.delete("X-MultiLLM-Provider-Calls"); + if (value === "hit") {headers.set("X-MultiLLM-Usage-Basis", "cache-served"); headers.set("X-MultiLLM-Provider-Calls", "0");} + return new Response(response.body, {status: response.status, statusText: response.statusText, headers}); +} +async function optionalStorage(operation) { + let timer; + const deadline = new Promise((_, reject) => {timer = setTimeout(() => reject(new Error("cache_timeout")), 2000);}); + try {return await Promise.race([operation, deadline]);} + finally {clearTimeout(timer);} +} +export async function prepareNativeCache(request, env, authority, context) { + if (!generationCacheSettings(env).enabled || !retentionAllowsContent(context.retentionPolicy)) return null; + const selected = mode(authority.cacheRequest ?? request); if (!selected) return null; + let identity; + try {identity = await exactCacheIdentity(request, env, authority, context);} catch {warn("identity_unavailable"); return null;} + if (!identity) return null; + const store = new GenerationCacheD1(env); + return { async lookup() { + if (selected.refresh || request.signal.aborted) return null; + try { + const entry = await optionalStorage(store.get(identity, selected)); + if (!entry || request.signal.aborted || await nativePolicyHash(request, env, authority, context) !== identity.policy_hash) return null; + const response = new Response(entry.body, {headers: {"content-type": entry.metadata.content_type, ...entry.metadata.headers, Age: String(Math.floor(entry.age))}}); + return mark(response, "hit"); + } catch {warn("storage_unavailable"); return null;} + }, async store(response) { + if (request.signal.aborted || response.status !== 200 || response.headers.get("content-type")?.split(";", 1)[0] !== "application/json" + || response.headers.get("content-encoding") && response.headers.get("content-encoding") !== "identity") return mark(response, "miss"); + try { + const length = response.headers.get("content-length"); + if (length !== null && (!/^\d+$/.test(length) || Number(length) > MAX_BODY_BYTES)) return mark(response, "miss"); + const body = await boundedBytes(response.clone().body, MAX_BODY_BYTES); + const current = await nativePolicyHash(request, env, authority, context); + if (body && completeCacheBody(body) && !request.signal.aborted && current === identity.policy_hash) { + await optionalStorage(store.put(identity, body, + {content_type: "application/json", headers: {}, provider: authority.provider, model: identity.model})); + } + } catch {warn("storage_unavailable");} + return mark(response, "miss"); + } }; +} +export function cacheServedEvent(response, context) { + return {...context, status: response.status, outcome: "success", duration_ms: Math.max(0, Math.round(performance.now() - context.startedAt)), + ttft_ms: null, input_tokens: null, output_tokens: null, usage_basis: "cache-served", provider_calls: 0, cost_usd: 0, cost_basis: "cache"}; +} diff --git a/worker/gateway-extensions.mjs b/worker/gateway-extensions.mjs index c22c7a5f..f05f05ca 100644 --- a/worker/gateway-extensions.mjs +++ b/worker/gateway-extensions.mjs @@ -1,4 +1,5 @@ /** Static registration boundary for native dispatch; later integrations supply named hooks. */ +import { generationCacheSettings, prepareNativeCache, cacheServedEvent } from "./exact-generation-cache.mjs"; import { createGatewayLifecycle } from "./gateway-lifecycle.mjs"; import { observeNativeResponse, appendNativeMetrics, PARSER_LIMIT } from "./request-telemetry.mjs"; import { recordNativeUsage } from "./usage-ledger-d1.mjs"; @@ -54,10 +55,13 @@ async function nativeContext(request, env, authority, metrics) { if (retentionPolicy.enabled) retentionPolicy = resolveRetentionPolicy(env, { ...retentionIdentity, keyHash: await retentionRequestId(env.ADMIN_API_KEY) }); const model = await requestedModel(request); + const revision = nativeRevisionConsumer(env); + const cacheRevisions = () => revision ? Object.fromEntries(Object.entries(revision.status().domains) + .map(([name, state]) => [name, state.revision])) : {}; return Object.freeze({ provider: authority.provider, principal, model: typeof model === "string" && /^[A-Za-z0-9][A-Za-z0-9._:/+@-]{0,223}$/.test(model) ? model : null, modelGroup: admissionModelGroup(authority.provider, model), - requestId: crypto.randomUUID(), endpoint: authority.route, startedAt, retentionPolicy }); + requestId: crypto.randomUUID(), endpoint: authority.route, startedAt, retentionPolicy, cacheRevisions }); } export async function nativeGenerationFetch(request, env, ctx, authority, fetcher, collaborators = []) { @@ -73,10 +77,11 @@ export async function nativeGenerationFetch(request, env, ctx, authority, fetche if (rejection) return rejection; const metrics = nativeMetricsEnabled(env); const admission = admissionSettings(env).enabled; + const cacheEnabled = generationCacheSettings(env).enabled; const registrations = collaborators.filter(hook => typeof hook.enabled === "function" ? hook.enabled(env) : hook.flag ? ["true", "1", "yes", "on"].includes(String(env[hook.flag] ?? "").trim().toLowerCase()) : metrics); const owner = new UpstreamCancellation({ signal: request.signal, onOutcome: authority.onCancellationOutcome }); - if (!metrics && !admission && !revision && !resolveRetentionPolicy(env).enabled && !registrations.length) { + if (!metrics && !admission && !revision && !resolveRetentionPolicy(env).enabled && !registrations.length && !cacheEnabled) { return dispatchNative(request, env, authority, fetcher, owner); } const context = await nativeContext(request, env, authority, metrics); @@ -91,12 +96,21 @@ export async function nativeGenerationFetch(request, env, ctx, authority, fetche try { await lifecycle.authorize(context); await lifecycle.admit(context); + const cache = cacheEnabled ? await prepareNativeCache(request, env, authority, context) : null; + const hit = await cache?.lookup(); + owner.throwIfAborted(); + if (hit) { + owner.complete(); + await lifecycle.finalize(cacheServedEvent(hit, context)); + return hit; + } const identity = admission ? { principal_hash: await principalHash((env.ADMIN_USERNAME || "admin").trim()), model_group: context.modelGroup, request_id: context.requestId, deadline_ms: Date.now() + 86_400_000 } : null; const dispatch = async () => { await lifecycle.before_dispatch(context); - const response = await dispatchNative(request, env, authority, fetcher, owner); + let response = await dispatchNative(request, env, authority, fetcher, owner); + if (cache) response = await cache.store(response); await lifecycle.observe(context); return response; }; @@ -123,3 +137,8 @@ export function withForwardedCorrelation(headers, env) { export function withNativeMetrics(response, env) { return nativeMetricsEnabled(env) ? appendNativeMetrics(response) : response; } + +const CACHE_RESPONSE_HEADERS = new Set(["x-multillm-cache", "x-multillm-cache-backend", "x-multillm-usage-basis", "x-multillm-provider-calls", "age"]); +export function nativeCacheHeader(name, headers, env) { + return generationCacheSettings(env).enabled && headers.get("X-MultiLLM-Cache-Backend") === "d1-r2" && CACHE_RESPONSE_HEADERS.has(name); +} diff --git a/worker/generation-cache-d1.mjs b/worker/generation-cache-d1.mjs new file mode 100644 index 00000000..f1564928 --- /dev/null +++ b/worker/generation-cache-d1.mjs @@ -0,0 +1,149 @@ +/** Fixed, scoped D1 metadata and immutable R2 bodies for exact generation replay. */ +export const CACHE_PREFIX = "generation-cache/v1/"; +export const MAX_BODY_BYTES = 1024 * 1024; +export const MAX_RPC_BYTES = 1_500_000; +export const TTL_SECONDS = 300; +const MAX_ENTRIES = 512; +const MAX_PRINCIPAL_BYTES = 16 * 1024 * 1024; +const HASH = /^[0-9a-f]{64}$/; +const MODEL = /^[A-Za-z0-9][A-Za-z0-9._:/+@-]{0,255}$/; +const HEADERS = new Set(["X-MultiLLM-Auto-Route", "X-MultiLLM-Auto-Selected-Model", "X-MultiLLM-Auto-Selected-Priority", + "X-MultiLLM-Cascade", "X-MultiLLM-Auto-Ordering", "X-MultiLLM-Prompt-Cache", "X-MultiLLM-Prompt-Cache-Mode"]); +const object = value => value !== null && typeof value === "object" && !Array.isArray(value); +const text = value => typeof value === "string" && value.length <= 256 && !/[\x00-\x1f\x7f]/.test(value); +export const validIdentity = value => object(value) && ["principal_hash", "cache_key", "policy_hash"].every(key => HASH.test(value[key])) + && typeof value.model === "string" && MODEL.test(value.model); +export function validCacheMetadata(value) { + return object(value) && Object.keys(value).length === 4 && value.content_type === "application/json" + && object(value.headers) && Object.entries(value.headers).every(([key, item]) => HEADERS.has(key) + && typeof item === "string" && item.length <= 1024 && !/[\x00-\x1f\x7f]/.test(item)) + && ["provider", "model"].every(key => value[key] === null || text(value[key])); +} +export function completeCacheBody(bytes) { + try { + const value = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes)); + if (!object(value) || Object.hasOwn(value, "error") || !Array.isArray(value.choices) || value.choices.length !== 1) return false; + const choice = value.choices[0]; + return object(choice) && ["stop", "end_turn", "stop_sequence", "eos"].includes(choice.finish_reason) + && object(choice.message) && (choice.message.tool_calls == null || Array.isArray(choice.message.tool_calls) + && choice.message.tool_calls.length === 0) && !choice.message.function_call; + } catch { return false; } +} +export async function digest(value) { + const bytes = typeof value === "string" ? new TextEncoder().encode(value) : value; + const result = await crypto.subtle.digest("SHA-256", bytes); + return Array.from(new Uint8Array(result), byte => byte.toString(16).padStart(2, "0")).join(""); +} +export function encodeBody(bytes) { + let binary = ""; + for (let offset = 0; offset < bytes.length; offset += 8192) binary += String.fromCharCode(...bytes.subarray(offset, offset + 8192)); + return btoa(binary); +} +export function decodeBody(value) { + if (typeof value !== "string" || value.length > Math.ceil(MAX_BODY_BYTES / 3) * 4 + || !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value)) return null; + try { const binary = atob(value); return Uint8Array.from(binary, character => character.charCodeAt(0)); } catch { return null; } +} + +// D1 serializes the transaction: simultaneous instances cannot both pass a capacity check. +const INSERT = `INSERT INTO generation_cache + (principal_hash, cache_key, policy_hash, model, created_at, expires_at, body_pointer, body_bytes, body_hash, metadata) + SELECT ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10 + WHERE (SELECT COUNT(*) FROM generation_cache WHERE principal_hash=?1 AND cache_key != ?2) < ${MAX_ENTRIES} + AND (SELECT COALESCE(SUM(body_bytes), 0) FROM generation_cache WHERE principal_hash=?1 AND cache_key != ?2) + ?8 <= ${MAX_PRINCIPAL_BYTES} + ON CONFLICT(principal_hash, cache_key) DO UPDATE SET policy_hash=excluded.policy_hash, model=excluded.model, + created_at=excluded.created_at, expires_at=excluded.expires_at, body_pointer=excluded.body_pointer, + body_bytes=excluded.body_bytes, body_hash=excluded.body_hash, metadata=excluded.metadata`; + +export class GenerationCacheD1 { + constructor(env, { clock = () => Date.now() / 1000 } = {}) { this.env = env; this.clock = clock; } + async get(identity, { maxAge = null } = {}) { + if (!validIdentity(identity)) return null; + const now = this.clock(); + const row = await this.env.INTELLIGENCE_DB.prepare(`SELECT created_at, expires_at, body_pointer, body_bytes, body_hash, metadata + FROM generation_cache WHERE principal_hash=? AND cache_key=? AND policy_hash=? AND model=? AND expires_at>?`) + .bind(identity.principal_hash, identity.cache_key, identity.policy_hash, identity.model, now).first(); + if (!row || !Number.isFinite(row.created_at) || row.created_at > now || row.expires_at > row.created_at + TTL_SECONDS + || !Number.isInteger(row.body_bytes) || row.body_bytes < 1 || row.body_bytes > MAX_BODY_BYTES || !HASH.test(row.body_hash)) return null; + const age = now - row.created_at; + if (maxAge !== null && age > maxAge) return null; + const prefix = `${CACHE_PREFIX}${identity.principal_hash}/${identity.cache_key}/`; + if (typeof row.body_pointer !== "string" || !row.body_pointer.startsWith(prefix) || !/^[0-9a-f]{32}$/.test(row.body_pointer.slice(prefix.length))) return null; + let metadata; + try { metadata = JSON.parse(row.metadata); } catch { return null; } + if (!validCacheMetadata(metadata)) return null; + const object = await this.env.multillm_media.get(row.body_pointer); + if (!object || object.size !== row.body_bytes) return null; + const body = new Uint8Array(await object.arrayBuffer()); + if (body.length !== row.body_bytes || await digest(body) !== row.body_hash || !completeCacheBody(body)) return null; + return { body, metadata, age }; + } + async put(identity, body, metadata) { + if (!validIdentity(identity) || !(body instanceof Uint8Array) || body.length > MAX_BODY_BYTES + || !validCacheMetadata(metadata) || !completeCacheBody(body)) return false; + // Check the deployed schema before creating any blob. + await this.env.INTELLIGENCE_DB.prepare("SELECT cache_key FROM generation_cache LIMIT 0").all(); + const now = this.clock(), expires = now + TTL_SECONDS; + const pointer = `${CACHE_PREFIX}${identity.principal_hash}/${identity.cache_key}/${crypto.randomUUID().replaceAll("-", "")}`; + const hash = await digest(body); + await this.env.multillm_media.put(pointer, body, { httpMetadata: {contentType: "application/json"}, + customMetadata: { expires_at: String(expires) } }); + let stored = false; + try { + const results = await this.env.INTELLIGENCE_DB.batch([ + this.env.INTELLIGENCE_DB.prepare("DELETE FROM generation_cache WHERE principal_hash=? AND expires_at<=?").bind(identity.principal_hash, now), + this.env.INTELLIGENCE_DB.prepare(INSERT).bind(identity.principal_hash, identity.cache_key, identity.policy_hash, + identity.model, now, expires, pointer, body.length, hash, JSON.stringify(metadata)), + ]); + stored = results[1].meta.changes === 1; + return stored; + } finally { + // Old/replaced pointers expire naturally; an active reader may still own one. + if (!stored) await this.env.multillm_media.delete(pointer).catch(() => {}); + } + } +} + +export async function cleanupGenerationCache(env, { now = Date.now() / 1000, limit = 100, cursor } = {}) { + if (!Number.isFinite(now) || !Number.isInteger(limit) || limit < 1 || limit > 1000) throw new TypeError("invalid_cleanup_bounds"); + const result = await env.INTELLIGENCE_DB.prepare(`DELETE FROM generation_cache WHERE (principal_hash, cache_key) IN + (SELECT principal_hash, cache_key FROM generation_cache WHERE expires_at<=? LIMIT ?)`).bind(now, limit).run(); + const page = await env.multillm_media.list({prefix: CACHE_PREFIX, limit, cursor, include: ["customMetadata"]}); + let deleted = 0; + for (const object of page.objects) { + const expiry = Number(object.customMetadata?.expires_at); + if (object.key.startsWith(CACHE_PREFIX) && object.customMetadata?.expires_at && Number.isFinite(expiry) && expiry <= now) { + await env.multillm_media.delete(object.key); deleted++; + } + } + return { rows: result.meta.changes, objects: deleted, cursor: page.truncated ? page.cursor : null }; +} + +const exactFields = (value, keys) => object(value) && Object.keys(value).length === keys.length && keys.every(key => Object.hasOwn(value, key)); +const IDENTITY_FIELDS = ["version", "operation", "principal_hash", "cache_key", "policy_hash", "model"]; +export async function handleGenerationCache(db, value, env) { + const store = new GenerationCacheD1({...env, INTELLIGENCE_DB: db}); + if (value.operation === "prune") { + if (!exactFields(value, ["version", "operation", "limit", "cursor"]) || !Number.isInteger(value.limit) || value.limit < 1 || value.limit > 1000 + || !(value.cursor === null || typeof value.cursor === "string" && value.cursor.length <= 2048)) return null; + if (!env.multillm_media) throw new Error("cache_storage_unavailable"); + return Response.json({version: 1, pruned: await cleanupGenerationCache(env, {limit: value.limit, cursor: value.cursor ?? undefined})}, + {headers: {"cache-control": "no-store"}}); + } + if (!validIdentity(value)) return null; + if (value.operation === "get") { + if (!exactFields(value, [...IDENTITY_FIELDS, "max_age"]) || !(value.max_age === null + || Number.isSafeInteger(value.max_age) && value.max_age >= 0 && value.max_age <= 999999999)) return null; + if (!env.multillm_media) throw new Error("cache_storage_unavailable"); + const entry = await store.get(value, {maxAge: value.max_age}); + return Response.json({version: 1, entry: entry ? {...entry, body: encodeBody(entry.body)} : null}, {headers: {"cache-control": "no-store"}}); + } + if (value.operation === "put") { + if (!exactFields(value, [...IDENTITY_FIELDS, "body", "metadata"]) || !validCacheMetadata(value.metadata)) return null; + const body = decodeBody(value.body); + if (!body || body.length > MAX_BODY_BYTES || !completeCacheBody(body)) return null; + if (!env.multillm_media) throw new Error("cache_storage_unavailable"); + return Response.json({version: 1, stored: await store.put(value, body, value.metadata)}, {headers: {"cache-control": "no-store"}}); + } + return null; +}