Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 11 additions & 11 deletions cloudflare-worker.mjs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { nativeGenerationFetch, withForwardedCorrelation, withNativeMetrics } from "./worker/gateway-extensions.mjs";
import { nativeGenerationFetch, withForwardedCorrelation, withNativeMetrics, nativeCacheHeader } from "./worker/gateway-extensions.mjs";
import { tickNativeRevisionSync } from "./worker/native-config-sync.mjs";
import { firewallFetch } from "./worker/secret-firewall.mjs";
import { Container, ContainerProxy, getContainer } from "@cloudflare/containers";
Expand Down Expand Up @@ -526,13 +526,13 @@ function buildLinkApiUpstreamHeaders(request, protocol, upstreamPathname, upstre
return headers;
}

function copyLinkApiResponseHeaders(headers) {
function copyLinkApiResponseHeaders(headers, env) {
const responseHeaders = new Headers();

for (const [header, value] of headers.entries()) {
const normalized = header.toLowerCase();
if (
LINKAPI_RESPONSE_HEADER_WHITELIST.has(normalized) ||
nativeCacheHeader(normalized, headers, env) || LINKAPI_RESPONSE_HEADER_WHITELIST.has(normalized) ||
LINKAPI_RESPONSE_HEADER_PREFIXES.some((prefix) => normalized.startsWith(prefix))
) {
responseHeaders.set(header, value);
Expand Down Expand Up @@ -613,13 +613,13 @@ function buildCodexEasyUpstreamHeaders(request, upstreamToken) {
return headers;
}

function copyCodexEasyResponseHeaders(headers) {
function copyCodexEasyResponseHeaders(headers, env) {
const responseHeaders = new Headers();

for (const [header, value] of headers.entries()) {
const normalized = header.toLowerCase();
if (
CODEX_EASY_RESPONSE_HEADER_WHITELIST.has(normalized) ||
nativeCacheHeader(normalized, headers, env) || CODEX_EASY_RESPONSE_HEADER_WHITELIST.has(normalized) ||
CODEX_EASY_RESPONSE_HEADER_PREFIXES.some((prefix) => normalized.startsWith(prefix))
) {
responseHeaders.set(header, value);
Expand Down Expand Up @@ -1497,15 +1497,15 @@ async function handleDirectOpencodeRequest(request, env, requestUrl, ctx) {
...(bodyAllowed && request.body ? { duplex: "half" } : {}),
});
const upstreamResponse = await nativeGenerationFetch(upstreamRequest, env, ctx, {
route: requestUrl.pathname, retentionHeader: request.headers.get("X-MultiLLM-Retention"), principal: { id: env.ADMIN_USERNAME || "admin" },
route: requestUrl.pathname, cacheRequest: request, retentionHeader: request.headers.get("X-MultiLLM-Retention"), principal: { id: env.ADMIN_USERNAME || "admin" },
provider: requestUrl.pathname.split("/")[1],
}, async (upstreamRequest, env, authority) => await firewallFetch(upstreamRequest, env, authority));

if (isOpencodeNativeRequest(requestUrl.pathname, request.method)) {
const downstreamResponse = new Response(upstreamResponse.body, {
status: upstreamResponse.status,
statusText: upstreamResponse.statusText,
headers: copyLinkApiResponseHeaders(upstreamResponse.headers),
headers: copyLinkApiResponseHeaders(upstreamResponse.headers, env),
});
return applyCorsHeaders(
request,
Expand Down Expand Up @@ -1632,7 +1632,7 @@ async function handleDirectLinkApiRequest(request, env, requestUrl, ctx) {
...(bodyAllowed && normalizedRequest.body ? { duplex: "half" } : {}),
});
const upstreamResponse = await nativeGenerationFetch(upstreamRequest, env, ctx, {
route: requestUrl.pathname, retentionHeader: request.headers.get("X-MultiLLM-Retention"), principal: { id: env.ADMIN_USERNAME || "admin" },
route: requestUrl.pathname, cacheRequest: request, retentionHeader: request.headers.get("X-MultiLLM-Retention"), principal: { id: env.ADMIN_USERNAME || "admin" },
provider: requestUrl.pathname.split("/")[1],
}, async (upstreamRequest, env, authority) => await firewallFetch(upstreamRequest, env, authority));

Expand All @@ -1641,7 +1641,7 @@ async function handleDirectLinkApiRequest(request, env, requestUrl, ctx) {
new Response(upstreamResponse.body, {
status: upstreamResponse.status,
statusText: upstreamResponse.statusText,
headers: copyLinkApiResponseHeaders(upstreamResponse.headers),
headers: copyLinkApiResponseHeaders(upstreamResponse.headers, env),
}),
env,
);
Expand Down Expand Up @@ -1690,7 +1690,7 @@ async function handleDirectCodexEasyRequest(request, env, requestUrl, ctx) {
...(bodyAllowed && normalizedRequest.body ? { duplex: "half" } : {}),
});
const upstreamResponse = await nativeGenerationFetch(upstreamRequest, env, ctx, {
route: requestUrl.pathname, retentionHeader: request.headers.get("X-MultiLLM-Retention"), principal: { id: env.ADMIN_USERNAME || "admin" },
route: requestUrl.pathname, cacheRequest: request, retentionHeader: request.headers.get("X-MultiLLM-Retention"), principal: { id: env.ADMIN_USERNAME || "admin" },
provider: requestUrl.pathname.split("/")[1],
}, async (upstreamRequest, env, authority) => await firewallFetch(upstreamRequest, env, authority));

Expand All @@ -1699,7 +1699,7 @@ async function handleDirectCodexEasyRequest(request, env, requestUrl, ctx) {
new Response(upstreamResponse.body, {
status: upstreamResponse.status,
statusText: upstreamResponse.statusText,
headers: copyCodexEasyResponseHeaders(upstreamResponse.headers),
headers: copyCodexEasyResponseHeaders(upstreamResponse.headers, env),
}),
env,
);
Expand Down
35 changes: 35 additions & 0 deletions docs/shared-generation-cache.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
# Shared exact generation cache

The default settings keep the existing process-memory cache:

```text
GENERATION_CACHE_BACKEND=memory
GENERATION_CACHE_SHARED_ENABLED=false
```

To share exact complete Chat Completions responses between Container instances and native Worker requests, apply `intelligence-migrations/0019_generation_cache.sql`, then set:

```text
GENERATION_CACHE_BACKEND=d1-r2
GENERATION_CACHE_SHARED_ENABLED=true
```

Each caller must also send `X-MultiLLM-Cache: on`. Both settings and the request header are required. Empty settings use the defaults; malformed settings disable shared caching and emit one content-free warning. `RESPONSE_CACHE_ENABLED=false` disables the Flask response cache. Native caching applies to supported native Chat Completions endpoints; other native protocols and raw requests without the header retain their normal transport. Flask and native caches have separate policy identities and do not reuse one another's entries.

Authentication, key scopes, model permission checks and retention policy run before lookup. Identity includes the canonical full request, authenticated principal and key digest, route, model, effective server policy and workflow version. Native identity also includes the upstream endpoint, protocol organization/project headers and current configuration revisions when revision synchronization is enabled. Requests with ambiguous provider-selection headers, separate upstream credentials, session identifiers or query parameters bypass shared caching. Policy changes during lookup prevent replay; changes during dispatch prevent storage.

Eligible requests are deterministic (`temperature: 0` or an integer seed), nonstreaming and single-choice. Tools or legacy functions must be absent or disabled with `tool_choice: none`. Only an HTTP 200 JSON response with a complete finish reason and no runnable tool output is stored. Errors, partial, filtered, malformed, streaming and oversized results are never stored. There is no semantic similarity matching or automatic request replay.

An eligible response carries `X-MultiLLM-Cache: hit` or `miss` and `X-MultiLLM-Cache-Backend: d1-r2`. Hits also carry `Age`, `X-MultiLLM-Usage-Basis: cache-served` and `X-MultiLLM-Provider-Calls: 0`. The original body is preserved, including any original provider usage. Those original counts do not describe a new provider call. Accounting records the replay with a cache cost basis and zero new provider cost; it does not claim the original generation was free. Optional native ledger recording still requires `NATIVE_EDGE_METRICS_ENABLED=true`.

`Cache-Control: no-store` bypasses reads and writes. `no-cache` or `X-MultiLLM-Cache: refresh` requests fresh dispatch; `max-age` can restrict acceptable entry age. Zero-content retention bypasses before content is read from or written to shared storage. It applies prospectively and does not erase entries from earlier requests.

## Storage and failures

Metadata lives in the `generation_cache` D1 table. Response bodies use the existing `multillm_media` R2 binding under `generation-cache/v1/`. No additional binding is required. Each principal is limited to 512 live entries and 16 MiB of live response bodies. One body may contain at most 1 MiB. TTL is 300 seconds. Transactional capacity checks reject new writes when full and allow replacement of an existing key within the byte budget. Full caches continue ordinary dispatch.

R2 pointers are immutable and scoped to principal and exact key. Reads validate the body size, digest, metadata and completion shape. Missing or corrupt bodies are misses. Native storage reads and writes wait at most two seconds per operation; the Container private transport waits at most three seconds. The private `POST /v1/state/generation-cache` endpoint accepts only fixed `get`, `put` and bounded `prune` operations on the existing internal outbound transport. It is unavailable when shared caching is disabled. Missing bindings or schema return a sanitized JSON HTTP 503 from this private endpoint. Chat cache failures continue normal dispatch and preserve its actual response; no success is invented and no upstream retry is added.

## Expired objects

Schedule bounded calls to `cleanupGenerationCache(env, {limit: 100, cursor})` from the storage module, or the equivalent private `prune` operation. Continue with the returned cursor until it is null. Cleanup removes expired metadata and objects only under `generation-cache/v1/` with an expired `expires_at` custom metadata value. It never deletes unrelated media objects. Replaced and orphaned bodies expire after their original 300-second TTL, so physical bucket usage can temporarily exceed the live per-principal metadata budget until cleanup runs. R2 and D1 storage operations can incur their normal service charges. Migration deployment and scheduled cleanup are operator actions; enabling the feature does not apply SQL or install a schedule.
16 changes: 16 additions & 0 deletions intelligence-migrations/0019_generation_cache.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
-- Shared exact generation metadata; complete response bodies live in the media bucket.
CREATE TABLE IF NOT EXISTS generation_cache (
principal_hash TEXT NOT NULL,
cache_key TEXT NOT NULL,
policy_hash TEXT NOT NULL,
model TEXT NOT NULL,
created_at REAL NOT NULL,
expires_at REAL NOT NULL,
body_pointer TEXT NOT NULL,
body_bytes INTEGER NOT NULL CHECK (body_bytes >= 1 AND body_bytes <= 1048576),
body_hash TEXT NOT NULL,
metadata TEXT NOT NULL,
PRIMARY KEY (principal_hash, cache_key),
CHECK (expires_at > created_at AND expires_at <= created_at + 300)
);
CREATE INDEX IF NOT EXISTS generation_cache_expiry ON generation_cache (expires_at);
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
"lint": "ruff check . --select E9,F63,F7,F82",
"typecheck": "mypy --ignore-missing-imports --follow-imports=skip --disable-error-code import-untyped --disable-error-code var-annotated --disable-error-code arg-type --disable-error-code assignment --disable-error-code operator app.py routes services providers streaming",
"test": "python -m pytest -q",
"test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_deferred_tools_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs",
"test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_shared_generation_cache_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_deferred_tools_worker.mjs",
"db:migrate": "python scripts/validate_sqlite_schema.py --apply",
"db:validate": "python scripts/validate_sqlite_schema.py",
"security:secrets": "python scripts/check_static_secrets.py",
Expand Down
2 changes: 1 addition & 1 deletion route_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@
"X-MultiLLM-Auto-Route, X-MultiLLM-Auto-Selected-Model, "
"X-MultiLLM-Auto-Attempts, X-MultiLLM-Auto-Selected-Priority, "
"X-MultiLLM-Auto-Ordering, X-MultiLLM-Auto-Failover-Reasons, "
"X-MultiLLM-Cache, X-MultiLLM-Retention, X-MultiLLM-Roleplay-Recovery, X-MultiLLM-Transport-Failure, X-MultiLLM-Token-Count, "
"X-MultiLLM-Cache, X-MultiLLM-Cache-Backend, X-MultiLLM-Usage-Basis, X-MultiLLM-Provider-Calls, Age, X-MultiLLM-Retention, X-MultiLLM-Roleplay-Recovery, X-MultiLLM-Transport-Failure, X-MultiLLM-Token-Count, "
"X-MultiLLM-Latency-Ms, X-MultiLLM-Estimated-Cost-USD, "
"X-MultiLLM-Cost-Basis, WWW-Authenticate, X-PAYMENT-RESPONSE, X-Poll-After, "
"X-NanoGPT-Advisor-ID, X-NanoGPT-Data-Endpoint, "
Expand Down
Loading
Loading