Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
87 commits
Select commit Hold shift + click to select a range
64037f4
feat(routing): add opt-in bandit route-order recommendations
ALikesToCode Oct 9, 2026
a15daf3
feat(context): add opt-in reversible context paging
ALikesToCode Oct 9, 2026
b636129
feat(providers): add opt-in local inference provider adapters
ALikesToCode Oct 9, 2026
ab5be97
feat(observability): add opt-in Langfuse and Helicone exports
ALikesToCode Oct 9, 2026
a7f8d76
feat(security): add opt-in prompt injection heuristics
ALikesToCode Oct 9, 2026
b0bf0bf
feat(routing): add opt-in canary and A/B auto-route cohorts
ALikesToCode Oct 9, 2026
a8bc284
Merge branch 'feat/context-pages' into feat/canary-traffic
ALikesToCode Oct 9, 2026
2e27ecc
feat(pools): add opt-in reset-aware account pool scheduling
ALikesToCode Oct 9, 2026
c91fae2
feat(cache): add opt-in scoped semantic generation cache
ALikesToCode Oct 9, 2026
5940bdc
Merge branch 'integrate/gateway-waves-1-4' into feat/context-pages
ALikesToCode Oct 9, 2026
6150133
Merge branch 'feat/context-pages' into feat/canary-traffic
ALikesToCode Oct 9, 2026
ddffeba
Merge branch 'feat/canary-traffic' into feat/semantic-cache
ALikesToCode Oct 9, 2026
585f5b7
feat(batches): add an opt-in gateway Batch API with explicit spillover
ALikesToCode Oct 9, 2026
ab9445f
Merge branch 'feat/semantic-cache' into feat/gateway-batches
ALikesToCode Oct 9, 2026
84d76ff
feat(pii): add opt-in PII redaction with streaming rehydration
ALikesToCode Oct 9, 2026
ae345fc
feat(responses): add opt-in hosted Responses continuation state
ALikesToCode Oct 9, 2026
a4630c5
Merge branch 'feat/gateway-batches' into feat/responses-state
ALikesToCode Oct 9, 2026
555c4dd
Merge branch 'feat/responses-state' into integrate/gateway-waves-1-5
ALikesToCode Oct 9, 2026
7d80d5d
Merge branch 'feat/observability-exports' into integrate/gateway-wave…
ALikesToCode Oct 9, 2026
3e28b16
Merge branch 'feat/local-inference' into integrate/gateway-waves-1-5
ALikesToCode Oct 9, 2026
2037c03
Merge branch 'feat/bandit-recommendations' into integrate/gateway-wav…
ALikesToCode Oct 9, 2026
2d0f2a6
Merge branch 'feat/prompt-injection' into integrate/gateway-waves-1-5
ALikesToCode Oct 9, 2026
c2d897a
Merge branch 'feat/pool-reset-scheduling' into integrate/gateway-wave…
ALikesToCode Oct 9, 2026
7198a88
Merge branch 'feat/pii-rehydration' into integrate/gateway-waves-1-5
ALikesToCode Oct 9, 2026
6c11a27
feat(gateway): wire the Wave 5 gateway features together
ALikesToCode Oct 9, 2026
6c67249
feat(routing): add opt-in latency SLO admission
ALikesToCode Oct 9, 2026
a4b2231
feat(pools): add opt-in learned provider cooldowns
ALikesToCode Oct 9, 2026
c3712d1
feat(realtime): add opt-in Worker Realtime WebSocket transport
ALikesToCode Oct 9, 2026
4af6c28
feat(security): add opt-in context-leak canary tokens
ALikesToCode Oct 9, 2026
5581f44
feat(usage): add opt-in signed usage receipts
ALikesToCode Oct 9, 2026
13231aa
feat(enterprise): add optional enterprise integration contracts
ALikesToCode Oct 9, 2026
1ef4abf
feat(routing): add opt-in bounded hedged auto requests
ALikesToCode Oct 9, 2026
ffb93c6
feat(streaming): add opt-in per-key stream cost breaker
ALikesToCode Oct 9, 2026
f0ceaf3
Merge branch 'feat/realtime' into feat/stream-cost-breaker
ALikesToCode Oct 9, 2026
46a1c40
Merge branch 'feat/stream-cost-breaker' into feat/learned-cooldown
ALikesToCode Oct 9, 2026
5402b2d
Merge branch 'feat/learned-cooldown' into feat/usage-receipts
ALikesToCode Oct 9, 2026
c1c3646
Merge branch 'feat/usage-receipts' into integrate/gateway-waves-1-6
ALikesToCode Oct 9, 2026
3c9ba9f
Merge branch 'feat/hedged-requests' into integrate/gateway-waves-1-6
ALikesToCode Oct 9, 2026
6514d0c
Merge branch 'feat/latency-slo' into integrate/gateway-waves-1-6
ALikesToCode Oct 9, 2026
12f83c9
Merge branch 'feat/context-canary' into integrate/gateway-waves-1-6
ALikesToCode Oct 9, 2026
b2f2fd7
Merge branch 'feat/enterprise-contracts' into integrate/gateway-waves…
ALikesToCode Oct 9, 2026
82270c6
test(d1): skip the stream cost column with its accounts table
ALikesToCode Oct 9, 2026
7f65c83
Merge branch 'feat/stream-cost-breaker' into feat/learned-cooldown
ALikesToCode Oct 9, 2026
fa431c3
Merge branch 'feat/learned-cooldown' into feat/usage-receipts
ALikesToCode Oct 9, 2026
4e5e3d0
Merge branch 'feat/usage-receipts' into integrate/gateway-waves-1-6
ALikesToCode Oct 9, 2026
470a092
test(d1): skip the stream cost column in the unmigrated integration s…
ALikesToCode Oct 9, 2026
ad04d06
Merge branch 'feat/stream-cost-breaker' into feat/learned-cooldown
ALikesToCode Oct 9, 2026
ec0f357
Merge branch 'feat/learned-cooldown' into feat/usage-receipts
ALikesToCode Oct 9, 2026
314c2fe
Merge branch 'feat/usage-receipts' into integrate/gateway-waves-1-6
ALikesToCode Oct 9, 2026
db0968d
feat(gateway): wire the Wave 6 gateway features together
ALikesToCode Oct 9, 2026
3d427ee
feat(auth): add opt-in SAML federation through a trusted broker
ALikesToCode Oct 9, 2026
5953994
feat(billing): add opt-in Stripe top-ups credited only from verified …
ALikesToCode Oct 9, 2026
248addd
feat(provisioning): add opt-in SCIM 2.0 user and group provisioning
ALikesToCode Oct 9, 2026
4d362a7
feat(credits): add an opt-in gateway credits ledger
ALikesToCode Oct 9, 2026
649f8f9
feat(governance): add opt-in hierarchical grants, budgets and attribu…
ALikesToCode Oct 9, 2026
048e2ff
feat(tenants): add opt-in organisations, teams and verified workspaces
ALikesToCode Oct 9, 2026
389f1c6
Merge branch 'feat/tenant-hierarchy' into feat/tenant-governance
ALikesToCode Oct 9, 2026
5083e2f
Merge branch 'feat/tenant-governance' into feat/saml-federation
ALikesToCode Oct 9, 2026
20f68a1
Merge branch 'feat/saml-federation' into feat/scim-provisioning
ALikesToCode Oct 9, 2026
7538aec
Merge branch 'feat/scim-provisioning' into feat/credits-ledger
ALikesToCode Oct 9, 2026
81629b4
Merge branch 'feat/credits-ledger' into feat/payment-billing
ALikesToCode Oct 9, 2026
0cfdf92
test(d1): start the integration Worker with the production nodejs_com…
ALikesToCode Oct 9, 2026
764de2a
Merge branch 'feat/payment-billing' into integrate/gateway-waves-1-7
ALikesToCode Oct 9, 2026
7559c87
Merge branch 'feat/tenant-hierarchy' into integrate/gateway-waves-1-7
ALikesToCode Oct 9, 2026
de9a471
Merge branch 'feat/tenant-hierarchy' into feat/tenant-governance
ALikesToCode Oct 9, 2026
0c9d36f
Merge branch 'feat/tenant-governance' into feat/saml-federation
ALikesToCode Oct 9, 2026
606bba8
Merge branch 'feat/saml-federation' into feat/scim-provisioning
ALikesToCode Oct 9, 2026
3a2a109
Merge branch 'feat/scim-provisioning' into feat/credits-ledger
ALikesToCode Oct 9, 2026
cc933b4
Merge branch 'feat/credits-ledger' into feat/payment-billing
ALikesToCode Oct 9, 2026
cee6ade
Merge branch 'feat/payment-billing' into integrate/gateway-waves-1-7
ALikesToCode Oct 9, 2026
83e9193
feat(gateway): wire the Wave 7 enterprise features together
ALikesToCode Oct 9, 2026
f96656f
Merge remote-tracking branch 'origin/integrate/gateway-waves-1-4' int…
ALikesToCode Oct 9, 2026
a532fae
Merge remote-tracking branch 'origin/integrate/gateway-waves-1-4' int…
ALikesToCode Oct 9, 2026
d570b2e
Merge branch 'integrate/gateway-waves-1-4' into integrate/gateway-wav…
ALikesToCode Oct 9, 2026
fb6ed66
Merge remote-tracking branch 'origin/feat/observability-exports' into…
ALikesToCode Oct 9, 2026
df64359
Merge branch 'integrate/gateway-waves-1-6' into integrate/gateway-wav…
ALikesToCode Oct 9, 2026
b4ec747
Merge branch 'integrate/gateway-waves-1-5' into integrate/gateway-wav…
ALikesToCode Oct 9, 2026
522f315
Merge remote-tracking branch 'origin/feat/prompt-injection' into inte…
ALikesToCode Oct 9, 2026
e83bec4
Merge branch 'integrate/gateway-waves-1-6' into integrate/gateway-wav…
ALikesToCode Oct 9, 2026
e7e2166
Merge branch 'integrate/gateway-waves-1-5' into integrate/gateway-wav…
ALikesToCode Oct 9, 2026
5fc1aa3
Merge branch 'integrate/gateway-waves-1-4' into integrate/gateway-wav…
ALikesToCode Oct 9, 2026
4510992
fix(ci): run the credits parity check with the CI interpreter
ALikesToCode Oct 9, 2026
27c5f1d
Merge branch 'integrate/gateway-waves-1-6' into integrate/gateway-wav…
ALikesToCode Oct 9, 2026
44519f4
Merge branch 'integrate/gateway-waves-1-5' into integrate/gateway-wav…
ALikesToCode Oct 9, 2026
dd44237
Merge branch 'integrate/gateway-waves-1-4' into integrate/gateway-wav…
ALikesToCode Oct 9, 2026
546cd41
Merge pull request #76 from ALikesToCode/integrate/gateway-waves-1-7
ALikesToCode Oct 9, 2026
3fee165
Merge pull request #69 from ALikesToCode/integrate/gateway-waves-1-6
ALikesToCode Oct 9, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,31 @@ store data name the D1 migration to apply before enabling them.
- [Deferred tool discovery](docs/deferred-tools.md): `DEFERRED_TOOLS_ENABLED` adds `multillm.tools.discover` to the MCP endpoints, with per-principal tool grants (migration 0018).
- [Protocol extras](docs/protocol-extras.md): `PROTOCOL_EXTRAS_ENABLED` keeps reviewed protocol-specific fields, such as Messages `top_k`, through translation; they reach a provider only if it admits the field.
- [Sticky session tiers](docs/session-tiers.md): `SESSION_TIER_MODE=sticky` keeps a session's approved model tier stable through tool calls in the main, delegation and aux lanes (migration 0021).
- [Hosted Responses state](docs/responses-state.md): `HOSTED_RESPONSES_ENABLED` lets a managed `/v1/responses` request with `gateway_state: true` keep its completed response under a gateway ID, continue from it with `previous_response_id`, and read or delete it later (migration 0028).
- [Gateway batches](docs/gateway-batches.md): `GATEWAY_BATCHES_ENABLED` adds OpenAI-compatible `/v1/files` and `/v1/batches` with a required spend budget, and `BATCH_SPILLOVER_ENABLED` turns a chat request sent with `X-MultiLLM-Priority: batch` and `Prefer: respond-async` into a one-item batch (migration 0027).
- [Observability exports](docs/observability-exports.md): `OBSERVABILITY_EXPORTERS_JSON` sends content-free request observations to the Langfuse or Helicone collectors you name.
- [Context pages](docs/context-pages.md): `CONTEXT_PAGING_ENABLED` lets a client that sends the `multillm_context_retrieve` capability replace older conversation turns with exact, owner-scoped page handles it can retrieve later (migration 0024).
- [Semantic generation cache](docs/semantic-generation-cache.md): `SEMANTIC_CACHE_ENABLED` with `SEMANTIC_CACHE_POLICY_JSON` reuses a stored answer for a near-identical final user message on opted-in routes or accounts, under strict matching and a per-lookup embedding spend cap (migration 0026).
- [Canary traffic](docs/canary-traffic.md): `CANARY_TRAFFIC_ENABLED` splits an auto route's sessions into baseline and candidate cohorts, in shadow or live mode, from a reviewed policy (migration 0025).
- [Bandit recommendations](docs/bandit-recommendations.md): `BANDIT_MODE` proposes a reviewed one-step route order change from stored evaluation results, without calling providers.
- [Local inference](docs/local-inference.md): `OLLAMA_BASE_URL`, `VLLM_BASE_URL`, `LM_STUDIO_BASE_URL`, `LLAMA_CPP_BASE_URL` and `SGLANG_BASE_URL` add explicitly configured OpenAI-compatible local servers as providers.
- [PII redaction](docs/pii-rehydration.md): `PII_REDACTION_ENABLED` with `PII_REDACTION_POLICY_JSON` replaces detected emails, phone numbers and card numbers with request-local placeholders before dispatch and restores them in JSON and streamed responses.
- [Prompt injection heuristics](docs/prompt-injection.md): `PROMPT_INJECTION_MODE=log` or `block` scores managed requests for injection and jailbreak patterns before dispatch.
- [Reset-aware pool scheduling](docs/pool-reset-scheduling.md): `POOL_RESET_SCHEDULING_ENABLED` skips pooled accounts whose quota is freshly exhausted and prefers the earliest reset, using only data the gateway already receives.
- [Latency SLO admission](docs/latency-slo.md): `LATENCY_SLO_MODE=reject` or `reroute` with `LATENCY_SLO_POLICY_JSON` predicts deadline risk from recent model measurements and can reject requests or reroute eligible automatic selections before dispatch.
- [Hedged auto requests](docs/hedged-requests.md): `HEDGED_REQUESTS_ENABLED` with `HEDGED_REQUESTS_POLICY_JSON` races at most two candidates on auto routes explicitly marked idempotent-safe, and accounts for every attempt.
- [Realtime WebSocket transport](docs/realtime.md): `REALTIME_ENABLED` with `REALTIME_PROVIDERS_JSON` bridges authenticated OpenAI-compatible Realtime sockets directly in the Worker, under a per-session cost cap (migration 0029).
- [Stream cost breaker](docs/stream-cost-breaker.md): `STREAM_COST_BREAKER_ENABLED` lets a key carry `max_stream_cost_usd`, and stops a stream with the protocol's error when its observed or conservatively estimated cost would pass the cap (migration 0030).
- [Learned provider cooldowns](docs/learned-cooldown.md): `LEARNED_COOLDOWN_MODE=shadow` or `apply` with `LEARNED_COOLDOWN_POLICY_JSON` narrows a bounded cooldown interval from natural throttle and recovery observations, without probes (migration 0031).
- [Context-leak canary tokens](docs/context-canary.md): `CONTEXT_CANARY_MODE=log` or `block` with `CONTEXT_CANARY_POLICY_JSON` detects opted-in disclosure of a request-specific marker. It does not claim complete leak detection.
- [Signed usage receipts](docs/usage-receipts.md): `USAGE_RECEIPTS_ENABLED` with a reviewed Ed25519 key writes a hash-chained receipt per settled usage event that its owner can fetch and verify, keeping unknown cost explicit (migration 0032).
- [Enterprise integration contracts](docs/enterprise-contracts.md): typed identity, tenancy, quota and credit boundaries for later organisation, SSO and payment modules. `ENTERPRISE_PREVIEW_ENABLED` shows a read-only administrator preview; nothing else activates.
- [Organisations and teams](docs/tenant-hierarchy.md): `ORGANISATIONS_ENABLED` adds optional organisation and team membership with a workspace identity the server verifies, and keeps each workspace's caches, idempotency, hosted state and usage separate (migration 0033).
- [Tenant governance](docs/tenant-governance.md): `TENANT_GOVERNANCE_ENABLED` intersects organisation, team and key grants, reserves organisation and team budgets together with the key's, and attributes usage to the verified workspace (migration 0034).
- [SAML federation](docs/saml-federation.md): `SAML_ENABLED` with `SAML_BROKER_URL`, `SAML_TRUST_CONFIG_JSON` and `SAML_CALLBACK_URL` signs gateway accounts in to the dashboard through an explicitly trusted SAML broker, only for subjects an administrator has linked to an account (migration 0035).
- [SCIM provisioning](docs/scim-provisioning.md): `SCIM_ENABLED` with `SCIM_TRUST_CONFIG_JSON` provisions scoped users and organisation teams through SCIM 2.0; it needs D1 storage and security revision sync (migration 0036).
- [Gateway credits](docs/credits-ledger.md): `CREDITS_ENABLED` keeps an append-only ledger of credits, holds and settlements, separate from usage estimates. `CREDITS_ENFORCEMENT=funded` or `all` makes requests reserve and settle credits (migration 0037).
- [Top-ups](docs/payment-billing.md): `PAYMENTS_ENABLED` with `PAYMENT_PROCESSOR_CONFIG_JSON` and `PAYMENT_WEBHOOK_KEY_REF` creates Stripe Checkout top-ups and credits explicit USD payments only from verified, idempotent webhooks (migration 0038).

## Setup

Expand Down
22 changes: 20 additions & 2 deletions app.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,26 +19,31 @@
login_required,
)
from routes.config_snapshots import register_config_snapshot_routes
from routes.enterprise_preview import register_enterprise_preview_routes
from routes.tenants import register_tenant_routes
from routes.core import register_core_routes
from routes.dashboard_security import register_dashboard_security_routes
from routes.intelligence_media import register_intelligence_media_routes
from routes.knowledge import register_knowledge_routes
from routes.documentation import register_documentation_routes
from routes.free_routes import register_free_routes
from routes.gateway_mcp import register_gateway_mcp_routes
from routes.gateway_batches import register_gateway_batch_routes
from routes.context_pages import register_context_page_routes
from routes.optimized import register_optimized_routes
from routes.proxy import register_proxy_routes
from routes.status_page import register_status_routes
from routes.unified import register_unified_routes
from routes.media import register_media_routes
from routes.usage import register_usage_routes
from routes.usage_receipts import register_usage_receipt_routes
from routes.workbench import register_workbench_routes
from security_config import load_max_content_length, validate_runtime_secrets
from services.auth_service import AuthService
from services.cache_service import CacheService
from services.image_relay_catalog import ImageRelayCatalogRefresh
from services.provider_catalog_refresh import ProviderCatalogAutoRefresh
from services.gateway_extensions import gateway_callbacks, register_gateway_extensions
from services.gateway_extensions import gateway_callbacks, register_gateway_extensions, register_enterprise_features
from services.metrics_service import MetricsService
from services.proxy_service import ProxyService
from services import usage_ledger
Expand Down Expand Up @@ -142,12 +147,25 @@ def create_app() -> Flask:
register_dashboard_security_routes(app)
register_workbench_routes(app)
register_config_snapshot_routes(app)
register_enterprise_preview_routes(app)
register_tenant_routes(app, csrf=csrf)
register_enterprise_features(app, csrf)
register_shadow_eval_routes(app, csrf, AuthService, MetricsService, ProxyService)
register_knowledge_routes(app, csrf)
register_gateway_mcp_routes(app, csrf)
register_documentation_routes(app, AuthService, ProxyService)
register_usage_routes(app, csrf)
register_gateway_extensions(app, callbacks=gateway_callbacks())
register_usage_receipt_routes(app, csrf)
register_gateway_extensions(app, callbacks=gateway_callbacks(csrf=csrf))
register_gateway_batch_routes(app)
from functools import partial
from route_helpers import api_authenticate_only
from services.context_pages import ContextPageService, PrivatePageStore, managed_authority
from services.intelligence_d1_store import request_private_intelligence
app.extensions["context_page_service"] = ContextPageService(PrivatePageStore(
partial(request_private_intelligence, endpoint="context-pages")))
register_context_page_routes(app, service=app.extensions["context_page_service"],
authorize=managed_authority, authenticate=api_authenticate_only)
# Restore the dashboard's recent requests from the durable ledger in the background.
usage_ledger.start(MetricsService.get_instance())

Expand Down
18 changes: 14 additions & 4 deletions cloudflare-worker.mjs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { nativeGenerationFetch, withForwardedCorrelation, withNativeMetrics, nativeCacheHeader, nativeGenerationSetup, generationErrorResponse, forwardedGenerationHeaders, runScheduledMaintenance, scheduledMaintenanceEnabled } from "./worker/gateway-extensions.mjs";
import { nativeGenerationFetch, withForwardedCorrelation, withNativeMetrics, nativeCacheHeader, nativeGenerationSetup, generationErrorResponse, forwardedGenerationHeaders, runScheduledMaintenance, scheduledMaintenanceEnabled, handleRoleplayContextPageRequest, handleRealtimeRequest } from "./worker/gateway-extensions.mjs";
import { batchesEnabled, runScheduledBatches } from "./worker/batch-jobs.mjs";
import { tickNativeRevisionSync } from "./worker/native-config-sync.mjs";
import { firewallFetch } from "./worker/secret-firewall.mjs";
import { Container, ContainerProxy, getContainer } from "@cloudflare/containers";
Expand Down Expand Up @@ -1499,7 +1500,7 @@ async function handleDirectOpencodeRequest(request, env, requestUrl, ctx) {
...(bodyAllowed && request.body ? { duplex: "half" } : {}),
});
const upstreamResponse = await nativeGenerationFetch(upstreamRequest, env, ctx, {
deadlineHook: generationSetup.hook, route: requestUrl.pathname, cacheRequest: request, retentionHeader: request.headers.get("X-MultiLLM-Retention"), principal: { id: env.ADMIN_USERNAME || "admin" },
authenticated: true, keyId: env.ADMIN_USERNAME || "admin", deadlineHook: generationSetup.hook, route: requestUrl.pathname, cacheRequest: request, retentionHeader: request.headers.get("X-MultiLLM-Retention"), principal: { id: env.ADMIN_USERNAME || "admin" },
provider: requestUrl.pathname.split("/")[1],
}, async (upstreamRequest, env, authority) => await firewallFetch(upstreamRequest, env, authority));

Expand Down Expand Up @@ -1635,7 +1636,7 @@ async function handleDirectLinkApiRequest(request, env, requestUrl, ctx) {
...(bodyAllowed && normalizedRequest.body ? { duplex: "half" } : {}),
});
const upstreamResponse = await nativeGenerationFetch(upstreamRequest, env, ctx, {
deadlineHook: generationSetup.hook, route: requestUrl.pathname, cacheRequest: request, retentionHeader: request.headers.get("X-MultiLLM-Retention"), principal: { id: env.ADMIN_USERNAME || "admin" },
authenticated: true, keyId: env.ADMIN_USERNAME || "admin", deadlineHook: generationSetup.hook, route: requestUrl.pathname, cacheRequest: request, retentionHeader: request.headers.get("X-MultiLLM-Retention"), principal: { id: env.ADMIN_USERNAME || "admin" },
provider: requestUrl.pathname.split("/")[1],
}, async (upstreamRequest, env, authority) => await firewallFetch(upstreamRequest, env, authority));

Expand Down Expand Up @@ -1694,7 +1695,7 @@ async function handleDirectCodexEasyRequest(request, env, requestUrl, ctx) {
...(bodyAllowed && normalizedRequest.body ? { duplex: "half" } : {}),
});
const upstreamResponse = await nativeGenerationFetch(upstreamRequest, env, ctx, {
deadlineHook: generationSetup.hook, route: requestUrl.pathname, cacheRequest: request, retentionHeader: request.headers.get("X-MultiLLM-Retention"), principal: { id: env.ADMIN_USERNAME || "admin" },
authenticated: true, keyId: env.ADMIN_USERNAME || "admin", deadlineHook: generationSetup.hook, route: requestUrl.pathname, cacheRequest: request, retentionHeader: request.headers.get("X-MultiLLM-Retention"), principal: { id: env.ADMIN_USERNAME || "admin" },
provider: requestUrl.pathname.split("/")[1],
}, async (upstreamRequest, env, authority) => await firewallFetch(upstreamRequest, env, authority));

Expand Down Expand Up @@ -1837,6 +1838,9 @@ export default {
tickNativeRevisionSync(env, ctx);
if (scheduledMaintenanceEnabled(env)) ctx.waitUntil(runScheduledMaintenance(env));
const container = getContainer(env.MULTILLM_PROXY_CONTAINER, "primary");
if (batchesEnabled(env)) ctx.waitUntil(runScheduledBatches(env, container).catch(() => {
console.warn(JSON.stringify({ event: "gateway_batch_schedule_failed" }));
}));
ctx.waitUntil(runScheduledShadowEval(env, container));
ctx.waitUntil(runScheduledHealth(controller, env, container).catch((error) => {
logStructuredError("scheduled_health_failed", error);
Expand Down Expand Up @@ -1887,6 +1891,9 @@ export default {
return applyCorsHeaders(request, buildFallbackHealthResponse(), env);
}

const realtimeResponse = await handleRealtimeRequest(request, env, ctx);
if (realtimeResponse) return realtimeResponse.status === 101 ? realtimeResponse : applyCorsHeaders(request, realtimeResponse, env);

if (requestUrl.pathname === "/status.prometheus" || requestUrl.pathname === "/v1/metrics/prometheus") {
if (!prometheusEnabled(env.PROMETHEUS_ENABLED)) {
return new Response("Not found", { status: 404, headers: { "Cache-Control": "no-store" } });
Expand Down Expand Up @@ -1968,6 +1975,9 @@ export default {
}
}

const roleplayPage = await handleRoleplayContextPageRequest(request, env);
if (roleplayPage) return applyCorsHeaders(request, roleplayPage, env);

if (kimiCodePath) {
try {
const response = await handleDirectKimiCodeRequest(request, env, requestUrl);
Expand Down
2 changes: 2 additions & 0 deletions config.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from providers.cline_pass import CLINE_API_BASE_URL
from providers.codex_everywhere import codex_everywhere_base_urls
from providers.image_relays import image_relay_base_urls
from providers.local_inference import local_inference_base_urls
from providers.nanogpt import nanogpt_text_base_url


Expand Down Expand Up @@ -212,6 +213,7 @@ class Config:
'gemma': 'https://generativelanguage.googleapis.com/v1beta',
**codex_everywhere_base_urls(),
**image_relay_base_urls(),
**local_inference_base_urls(),
}

# Provider-specific timeouts (connect_timeout, read_timeout)
Expand Down
Loading
Loading