Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
206 commits
Select commit Hold shift + click to select a range
88d595f
docs(plans): add LLM gateway gap analysis
ALikesToCode Oct 8, 2026
87c8151
feat(resilience): separate health outcomes from replay permission
ALikesToCode Oct 8, 2026
a4b75e1
feat(auto-routes): validate the first stream event before commitment
ALikesToCode Oct 8, 2026
b167e96
feat(diagnostics): add provider authenticity and conformance checks
ALikesToCode Oct 8, 2026
bb69d84
feat(roleplay): fit fallback context to each candidate window
ALikesToCode Oct 8, 2026
854a731
Merge remote-tracking branch 'origin/docs/llm-gateway-gap-analysis' i…
ALikesToCode Oct 8, 2026
c58c8ab
Merge remote-tracking branch 'origin/feat/upstream-outcomes' into int…
ALikesToCode Oct 8, 2026
2f40591
Merge remote-tracking branch 'origin/feat/stream-preflight' into inte…
ALikesToCode Oct 8, 2026
5dc619e
Merge remote-tracking branch 'origin/feat/provider-authenticity' into…
ALikesToCode Oct 8, 2026
7ee7cde
Merge remote-tracking branch 'origin/feat/roleplay-candidate-context'…
ALikesToCode Oct 8, 2026
d750bf4
feat(auto-routes): forward the stream preflight setting to the Container
ALikesToCode Oct 8, 2026
8224c19
docs(resilience): list upstream outcomes in the README
ALikesToCode Oct 8, 2026
5a2de17
docs(roleplay): list candidate context in the README
ALikesToCode Oct 8, 2026
40c00fd
docs(diagnostics): list provider authenticity in the README
ALikesToCode Oct 8, 2026
53f475c
feat(messages): count tokens with configured native provider endpoints
ALikesToCode Oct 8, 2026
c5f4e8e
Merge remote-tracking branch 'origin/feat/upstream-outcomes' into int…
ALikesToCode Oct 8, 2026
f709fde
Merge remote-tracking branch 'origin/feat/stream-preflight' into inte…
ALikesToCode Oct 8, 2026
9164e43
Merge remote-tracking branch 'origin/feat/provider-authenticity' into…
ALikesToCode Oct 8, 2026
13a369c
Merge remote-tracking branch 'origin/feat/roleplay-candidate-context'…
ALikesToCode Oct 8, 2026
71f9da6
Merge remote-tracking branch 'origin/feat/native-token-count' into in…
ALikesToCode Oct 8, 2026
b1652f1
feat(metrics): add opt-in Prometheus exposition
ALikesToCode Oct 8, 2026
e0c6b66
Merge remote-tracking branch 'origin/feat/prometheus-export' into int…
ALikesToCode Oct 8, 2026
e62decf
feat(catalog): add opt-in model precision preferences
ALikesToCode Oct 8, 2026
490bbe4
feat(cache): isolate response cache entries by resolved policy
ALikesToCode Oct 8, 2026
73d9353
feat(diagnostics): add an offline health probe cost forecast
ALikesToCode Oct 8, 2026
f4aef3c
feat(docs): publish an OpenAPI 3.1 contract
ALikesToCode Oct 8, 2026
9391ee5
feat(diagnostics): add configuration-bound capability probes
ALikesToCode Oct 8, 2026
d1f3ed1
feat(protocols): add opt-in conversion fidelity diagnostics
ALikesToCode Oct 8, 2026
88212a8
feat(shadow-eval): add opt-in three-arm noise floor and cost replay
ALikesToCode Oct 8, 2026
5e3fd86
fix(accounting): keep missing usage and unpriced cost distinct from zero
ALikesToCode Oct 8, 2026
5362f68
Merge remote-tracking branch 'origin/feat/model-precision' into integ…
ALikesToCode Oct 8, 2026
b05fff4
Merge remote-tracking branch 'origin/feat/cache-policy' into integrat…
ALikesToCode Oct 8, 2026
665d13a
Merge remote-tracking branch 'origin/feat/probe-cost-forecast' into i…
ALikesToCode Oct 8, 2026
448c5b6
Merge remote-tracking branch 'origin/feat/openapi-contract' into inte…
ALikesToCode Oct 8, 2026
8d1db45
Merge remote-tracking branch 'origin/feat/capability-probes' into int…
ALikesToCode Oct 8, 2026
72be02f
Merge remote-tracking branch 'origin/feat/conversion-fidelity' into i…
ALikesToCode Oct 8, 2026
457a6b9
Merge remote-tracking branch 'origin/feat/evaluation-noise-floor' int…
ALikesToCode Oct 8, 2026
24ad261
Merge remote-tracking branch 'origin/feat/usage-uncertainty' into int…
ALikesToCode Oct 8, 2026
ccea9d6
docs(precision): describe an invalid preference as ignored
ALikesToCode Oct 8, 2026
6a82834
docs(cache): say how the Container receives the policy revision
ALikesToCode Oct 8, 2026
2ed0193
docs(probes): link the capability probe budget
ALikesToCode Oct 8, 2026
b599fb8
Merge remote-tracking branch 'origin/feat/model-precision' into integ…
ALikesToCode Oct 8, 2026
36f86e1
Merge remote-tracking branch 'origin/feat/cache-policy' into integrat…
ALikesToCode Oct 8, 2026
7900c90
Merge remote-tracking branch 'origin/feat/probe-cost-forecast' into i…
ALikesToCode Oct 8, 2026
1d0be75
feat(workbench): add opt-in versioned prompt templates
ALikesToCode Oct 8, 2026
bd70b57
Merge remote-tracking branch 'origin/feat/prompt-templates' into inte…
ALikesToCode Oct 8, 2026
287f929
feat(routing): add opt-in reviewed configuration snapshots
ALikesToCode Oct 8, 2026
3038de0
Merge remote-tracking branch 'origin/feat/config-snapshots' into inte…
ALikesToCode Oct 8, 2026
58d4f6f
docs(readme): list the opt-in gateway features
ALikesToCode Oct 8, 2026
08a4094
feat(credentials): add opt-in model-scoped credential cooldown
ALikesToCode Oct 8, 2026
aa62d68
feat(transport): add opt-in bounded upstream retry advice
ALikesToCode Oct 8, 2026
afcf1e8
feat(rate-limit): add opt-in gateway rate-limit headers
ALikesToCode Oct 8, 2026
4497f7a
feat(admission): add opt-in shared concurrency admission leases
ALikesToCode Oct 8, 2026
220fbde
feat(metrics): add opt-in native edge request metrics
ALikesToCode Oct 8, 2026
6d34713
fix(transport): close upstream work once on client disconnect
ALikesToCode Oct 8, 2026
30f8c89
Merge remote-tracking branch 'origin/feat/retry-advice' into integrat…
ALikesToCode Oct 8, 2026
daa32fa
Merge remote-tracking branch 'origin/feat/model-cooldown' into integr…
ALikesToCode Oct 8, 2026
cf9df29
Merge remote-tracking branch 'origin/feat/rate-limit-headers' into in…
ALikesToCode Oct 8, 2026
10c1c9b
Merge remote-tracking branch 'origin/feat/admission-leases' into inte…
ALikesToCode Oct 8, 2026
d45aba5
Merge remote-tracking branch 'origin/feat/edge-request-metrics' into …
ALikesToCode Oct 8, 2026
9e456fc
Merge remote-tracking branch 'origin/feat/request-cancellation' into …
ALikesToCode Oct 8, 2026
3153640
feat(mcp): add opt-in tool contract digests and pin checks
ALikesToCode Oct 8, 2026
655eaba
feat(retention): add opt-in zero-content retention policy
ALikesToCode Oct 8, 2026
9d43b50
Merge remote-tracking branch 'origin/feat/mcp-contract-drift' into in…
ALikesToCode Oct 8, 2026
37a7eef
Merge remote-tracking branch 'origin/feat/zero-content-retention' int…
ALikesToCode Oct 8, 2026
2bef0d4
feat(usage): add opt-in prompt cache usage and cost buckets
ALikesToCode Oct 8, 2026
75a8a53
Merge remote-tracking branch 'origin/feat/prompt-cache-cost' into int…
ALikesToCode Oct 8, 2026
0d886bb
refactor(rate-limit): drop the unused Worker advice module
ALikesToCode Oct 8, 2026
1c441c3
Merge remote-tracking branch 'origin/feat/rate-limit-headers' into in…
ALikesToCode Oct 8, 2026
c4b22a9
feat(config): add opt-in revision-aware configuration sync
ALikesToCode Oct 8, 2026
9007d5b
Merge remote-tracking branch 'origin/feat/config-revision-sync' into …
ALikesToCode Oct 8, 2026
26d55ea
feat(gateway): add opt-in output schema validation
ALikesToCode Oct 8, 2026
e185390
feat(gateway): add opt-in source-scoped protocol extras
ALikesToCode Oct 8, 2026
056d44d
feat(mcp): add opt-in permission-aware deferred tool discovery
ALikesToCode Oct 8, 2026
8657ce1
feat(gateway): wire the Wave 3 gateway features together
ALikesToCode Oct 8, 2026
6cabc31
Merge remote-tracking branch 'origin/integrate/gateway-waves-1-3' int…
ALikesToCode Oct 9, 2026
54e810d
feat(cache): add opt-in shared exact generation cache
ALikesToCode Oct 9, 2026
c7fa539
Merge branch 'feat/deferred-tools' into feat/shared-generation-cache
ALikesToCode Oct 9, 2026
0539efb
feat(budget): add opt-in durable ambiguous usage holds
ALikesToCode Oct 9, 2026
4194733
Merge branch 'feat/shared-generation-cache' into feat/usage-reservations
ALikesToCode Oct 9, 2026
9da1b66
feat(routing): add opt-in role-scoped sticky session tiers
ALikesToCode Oct 9, 2026
a128589
Merge branch 'feat/usage-reservations' into feat/session-tiers
ALikesToCode Oct 9, 2026
22d205c
feat(routing): add opt-in outcome-bound prompt cache affinity
ALikesToCode Oct 9, 2026
631c5cb
feat(gateway): add opt-in end-to-end generation deadlines
ALikesToCode Oct 9, 2026
f1e4188
feat(alerts): add opt-in spend, pool and provider health alerts
ALikesToCode Oct 9, 2026
00481a6
fix(deps): clear the CI dependency audit advisories
ALikesToCode Oct 9, 2026
aa85d93
Merge branch 'feat/session-tiers' into feat/gateway-alerts
ALikesToCode Oct 9, 2026
f3499b2
feat(idempotency): add opt-in managed request idempotency
ALikesToCode Oct 9, 2026
9aea628
Merge branch 'feat/gateway-alerts' into feat/managed-idempotency
ALikesToCode Oct 9, 2026
c8787fc
Merge branch 'feat/managed-idempotency' into integrate/gateway-waves-1-4
ALikesToCode Oct 9, 2026
4c24921
Merge branch 'feat/output-schema-validation' into integrate/gateway-w…
ALikesToCode Oct 9, 2026
a4fe9ea
Merge branch 'feat/protocol-extras' into integrate/gateway-waves-1-4
ALikesToCode Oct 9, 2026
07e45d9
Merge branch 'feat/prompt-cache-affinity' into integrate/gateway-wave…
ALikesToCode Oct 9, 2026
7f21af1
Merge branch 'feat/generation-deadlines' into integrate/gateway-waves…
ALikesToCode Oct 9, 2026
2c5213c
test(session-tiers): keep the Worker module inventory sorted
ALikesToCode Oct 9, 2026
6487bac
Merge branch 'feat/session-tiers' into feat/gateway-alerts
ALikesToCode Oct 9, 2026
24d66c0
Merge branch 'feat/gateway-alerts' into feat/managed-idempotency
ALikesToCode Oct 9, 2026
0b36352
Merge branch 'feat/managed-idempotency' into integrate/gateway-waves-1-4
ALikesToCode Oct 9, 2026
bafa1ea
docs(readme): list the Wave 4 gateway features
ALikesToCode Oct 9, 2026
64037f4
feat(routing): add opt-in bandit route-order recommendations
ALikesToCode Oct 9, 2026
a15daf3
feat(context): add opt-in reversible context paging
ALikesToCode Oct 9, 2026
b636129
feat(providers): add opt-in local inference provider adapters
ALikesToCode Oct 9, 2026
ab5be97
feat(observability): add opt-in Langfuse and Helicone exports
ALikesToCode Oct 9, 2026
a7f8d76
feat(security): add opt-in prompt injection heuristics
ALikesToCode Oct 9, 2026
cd4ac5f
feat(gateway): wire the Wave 4 gateway features together
ALikesToCode Oct 9, 2026
b0bf0bf
feat(routing): add opt-in canary and A/B auto-route cohorts
ALikesToCode Oct 9, 2026
a8bc284
Merge branch 'feat/context-pages' into feat/canary-traffic
ALikesToCode Oct 9, 2026
2e27ecc
feat(pools): add opt-in reset-aware account pool scheduling
ALikesToCode Oct 9, 2026
c91fae2
feat(cache): add opt-in scoped semantic generation cache
ALikesToCode Oct 9, 2026
5940bdc
Merge branch 'integrate/gateway-waves-1-4' into feat/context-pages
ALikesToCode Oct 9, 2026
6150133
Merge branch 'feat/context-pages' into feat/canary-traffic
ALikesToCode Oct 9, 2026
ddffeba
Merge branch 'feat/canary-traffic' into feat/semantic-cache
ALikesToCode Oct 9, 2026
585f5b7
feat(batches): add an opt-in gateway Batch API with explicit spillover
ALikesToCode Oct 9, 2026
ab9445f
Merge branch 'feat/semantic-cache' into feat/gateway-batches
ALikesToCode Oct 9, 2026
84d76ff
feat(pii): add opt-in PII redaction with streaming rehydration
ALikesToCode Oct 9, 2026
ae345fc
feat(responses): add opt-in hosted Responses continuation state
ALikesToCode Oct 9, 2026
a4630c5
Merge branch 'feat/gateway-batches' into feat/responses-state
ALikesToCode Oct 9, 2026
555c4dd
Merge branch 'feat/responses-state' into integrate/gateway-waves-1-5
ALikesToCode Oct 9, 2026
7d80d5d
Merge branch 'feat/observability-exports' into integrate/gateway-wave…
ALikesToCode Oct 9, 2026
3e28b16
Merge branch 'feat/local-inference' into integrate/gateway-waves-1-5
ALikesToCode Oct 9, 2026
2037c03
Merge branch 'feat/bandit-recommendations' into integrate/gateway-wav…
ALikesToCode Oct 9, 2026
2d0f2a6
Merge branch 'feat/prompt-injection' into integrate/gateway-waves-1-5
ALikesToCode Oct 9, 2026
c2d897a
Merge branch 'feat/pool-reset-scheduling' into integrate/gateway-wave…
ALikesToCode Oct 9, 2026
7198a88
Merge branch 'feat/pii-rehydration' into integrate/gateway-waves-1-5
ALikesToCode Oct 9, 2026
6c11a27
feat(gateway): wire the Wave 5 gateway features together
ALikesToCode Oct 9, 2026
6c67249
feat(routing): add opt-in latency SLO admission
ALikesToCode Oct 9, 2026
a4b2231
feat(pools): add opt-in learned provider cooldowns
ALikesToCode Oct 9, 2026
c3712d1
feat(realtime): add opt-in Worker Realtime WebSocket transport
ALikesToCode Oct 9, 2026
4af6c28
feat(security): add opt-in context-leak canary tokens
ALikesToCode Oct 9, 2026
5581f44
feat(usage): add opt-in signed usage receipts
ALikesToCode Oct 9, 2026
13231aa
feat(enterprise): add optional enterprise integration contracts
ALikesToCode Oct 9, 2026
1ef4abf
feat(routing): add opt-in bounded hedged auto requests
ALikesToCode Oct 9, 2026
ffb93c6
feat(streaming): add opt-in per-key stream cost breaker
ALikesToCode Oct 9, 2026
f0ceaf3
Merge branch 'feat/realtime' into feat/stream-cost-breaker
ALikesToCode Oct 9, 2026
46a1c40
Merge branch 'feat/stream-cost-breaker' into feat/learned-cooldown
ALikesToCode Oct 9, 2026
5402b2d
Merge branch 'feat/learned-cooldown' into feat/usage-receipts
ALikesToCode Oct 9, 2026
c1c3646
Merge branch 'feat/usage-receipts' into integrate/gateway-waves-1-6
ALikesToCode Oct 9, 2026
3c9ba9f
Merge branch 'feat/hedged-requests' into integrate/gateway-waves-1-6
ALikesToCode Oct 9, 2026
6514d0c
Merge branch 'feat/latency-slo' into integrate/gateway-waves-1-6
ALikesToCode Oct 9, 2026
12f83c9
Merge branch 'feat/context-canary' into integrate/gateway-waves-1-6
ALikesToCode Oct 9, 2026
b2f2fd7
Merge branch 'feat/enterprise-contracts' into integrate/gateway-waves…
ALikesToCode Oct 9, 2026
82270c6
test(d1): skip the stream cost column with its accounts table
ALikesToCode Oct 9, 2026
7f65c83
Merge branch 'feat/stream-cost-breaker' into feat/learned-cooldown
ALikesToCode Oct 9, 2026
fa431c3
Merge branch 'feat/learned-cooldown' into feat/usage-receipts
ALikesToCode Oct 9, 2026
4e5e3d0
Merge branch 'feat/usage-receipts' into integrate/gateway-waves-1-6
ALikesToCode Oct 9, 2026
470a092
test(d1): skip the stream cost column in the unmigrated integration s…
ALikesToCode Oct 9, 2026
ad04d06
Merge branch 'feat/stream-cost-breaker' into feat/learned-cooldown
ALikesToCode Oct 9, 2026
ec0f357
Merge branch 'feat/learned-cooldown' into feat/usage-receipts
ALikesToCode Oct 9, 2026
314c2fe
Merge branch 'feat/usage-receipts' into integrate/gateway-waves-1-6
ALikesToCode Oct 9, 2026
db0968d
feat(gateway): wire the Wave 6 gateway features together
ALikesToCode Oct 9, 2026
3d427ee
feat(auth): add opt-in SAML federation through a trusted broker
ALikesToCode Oct 9, 2026
5953994
feat(billing): add opt-in Stripe top-ups credited only from verified …
ALikesToCode Oct 9, 2026
248addd
feat(provisioning): add opt-in SCIM 2.0 user and group provisioning
ALikesToCode Oct 9, 2026
4d362a7
feat(credits): add an opt-in gateway credits ledger
ALikesToCode Oct 9, 2026
649f8f9
feat(governance): add opt-in hierarchical grants, budgets and attribu…
ALikesToCode Oct 9, 2026
048e2ff
feat(tenants): add opt-in organisations, teams and verified workspaces
ALikesToCode Oct 9, 2026
389f1c6
Merge branch 'feat/tenant-hierarchy' into feat/tenant-governance
ALikesToCode Oct 9, 2026
5083e2f
Merge branch 'feat/tenant-governance' into feat/saml-federation
ALikesToCode Oct 9, 2026
20f68a1
Merge branch 'feat/saml-federation' into feat/scim-provisioning
ALikesToCode Oct 9, 2026
7538aec
Merge branch 'feat/scim-provisioning' into feat/credits-ledger
ALikesToCode Oct 9, 2026
81629b4
Merge branch 'feat/credits-ledger' into feat/payment-billing
ALikesToCode Oct 9, 2026
0cfdf92
test(d1): start the integration Worker with the production nodejs_com…
ALikesToCode Oct 9, 2026
764de2a
Merge branch 'feat/payment-billing' into integrate/gateway-waves-1-7
ALikesToCode Oct 9, 2026
7559c87
Merge branch 'feat/tenant-hierarchy' into integrate/gateway-waves-1-7
ALikesToCode Oct 9, 2026
de9a471
Merge branch 'feat/tenant-hierarchy' into feat/tenant-governance
ALikesToCode Oct 9, 2026
0c9d36f
Merge branch 'feat/tenant-governance' into feat/saml-federation
ALikesToCode Oct 9, 2026
606bba8
Merge branch 'feat/saml-federation' into feat/scim-provisioning
ALikesToCode Oct 9, 2026
3a2a109
Merge branch 'feat/scim-provisioning' into feat/credits-ledger
ALikesToCode Oct 9, 2026
cc933b4
Merge branch 'feat/credits-ledger' into feat/payment-billing
ALikesToCode Oct 9, 2026
cee6ade
Merge branch 'feat/payment-billing' into integrate/gateway-waves-1-7
ALikesToCode Oct 9, 2026
83e9193
feat(gateway): wire the Wave 7 enterprise features together
ALikesToCode Oct 9, 2026
f96656f
Merge remote-tracking branch 'origin/integrate/gateway-waves-1-4' int…
ALikesToCode Oct 9, 2026
a532fae
Merge remote-tracking branch 'origin/integrate/gateway-waves-1-4' int…
ALikesToCode Oct 9, 2026
62a9950
Merge remote-tracking branch 'origin/fix/dependency-audit' into integ…
ALikesToCode Oct 9, 2026
d570b2e
Merge branch 'integrate/gateway-waves-1-4' into integrate/gateway-wav…
ALikesToCode Oct 9, 2026
01d6729
Merge branch 'integrate/gateway-waves-1-3' into integrate/gateway-wav…
ALikesToCode Oct 9, 2026
2c9829f
Merge branch 'integrate/gateway-waves-1-2' into integrate/gateway-wav…
ALikesToCode Oct 9, 2026
eb20ed7
Merge branch 'integrate/gateway-wave2' into integrate/gateway-waves-1-2
ALikesToCode Oct 9, 2026
fb6ed66
Merge remote-tracking branch 'origin/feat/observability-exports' into…
ALikesToCode Oct 9, 2026
df64359
Merge branch 'integrate/gateway-waves-1-6' into integrate/gateway-wav…
ALikesToCode Oct 9, 2026
b4ec747
Merge branch 'integrate/gateway-waves-1-5' into integrate/gateway-wav…
ALikesToCode Oct 9, 2026
522f315
Merge remote-tracking branch 'origin/feat/prompt-injection' into inte…
ALikesToCode Oct 9, 2026
4437c0b
Merge branch 'integrate/gateway-waves-1-3' into integrate/gateway-wav…
ALikesToCode Oct 9, 2026
c8e3ae3
Merge branch 'integrate/gateway-waves-1-2' into integrate/gateway-wav…
ALikesToCode Oct 9, 2026
9615c33
fix(ci): run the Python parity checks with the CI interpreter
ALikesToCode Oct 9, 2026
7f13874
fix(ci): run the reservation parity check with the CI interpreter
ALikesToCode Oct 9, 2026
e83bec4
Merge branch 'integrate/gateway-waves-1-6' into integrate/gateway-wav…
ALikesToCode Oct 9, 2026
e7e2166
Merge branch 'integrate/gateway-waves-1-5' into integrate/gateway-wav…
ALikesToCode Oct 9, 2026
5fc1aa3
Merge branch 'integrate/gateway-waves-1-4' into integrate/gateway-wav…
ALikesToCode Oct 9, 2026
4510992
fix(ci): run the credits parity check with the CI interpreter
ALikesToCode Oct 9, 2026
1a35feb
test(worker): give the post-handoff deadline test room for setup
ALikesToCode Oct 9, 2026
b4587aa
test(admission): widen the deadline bound for slow runners
ALikesToCode Oct 9, 2026
857fe4e
test(intelligence): allow shared CI runners more time in the deadline…
ALikesToCode Oct 9, 2026
c84f937
test(output-schema): show the response when a registered route test f…
ALikesToCode Oct 9, 2026
b06d06e
Merge branch 'integrate/gateway-waves-1-2' into integrate/gateway-wav…
ALikesToCode Oct 9, 2026
cf60acc
Merge branch 'integrate/gateway-wave2' into integrate/gateway-waves-1-2
ALikesToCode Oct 9, 2026
27c5f1d
Merge branch 'integrate/gateway-waves-1-6' into integrate/gateway-wav…
ALikesToCode Oct 9, 2026
44519f4
Merge branch 'integrate/gateway-waves-1-5' into integrate/gateway-wav…
ALikesToCode Oct 9, 2026
dd44237
Merge branch 'integrate/gateway-waves-1-4' into integrate/gateway-wav…
ALikesToCode Oct 9, 2026
ae88472
Merge branch 'integrate/gateway-waves-1-3' into integrate/gateway-wav…
ALikesToCode Oct 9, 2026
546cd41
Merge pull request #76 from ALikesToCode/integrate/gateway-waves-1-7
ALikesToCode Oct 9, 2026
3fee165
Merge pull request #69 from ALikesToCode/integrate/gateway-waves-1-6
ALikesToCode Oct 9, 2026
efb3012
Merge pull request #60 from ALikesToCode/integrate/gateway-waves-1-5
ALikesToCode Oct 9, 2026
bf939e0
Merge pull request #53 from ALikesToCode/integrate/gateway-waves-1-4
ALikesToCode Oct 9, 2026
6ed5e64
Merge pull request #39 from ALikesToCode/integrate/gateway-waves-1-3
ALikesToCode Oct 9, 2026
f33ffa7
Merge pull request #25 from ALikesToCode/integrate/gateway-waves-1-2
ALikesToCode Oct 9, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
7 changes: 7 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,11 @@ jobs:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.11"
- name: Install Python dependencies
run: python -m pip install --disable-pip-version-check --require-hashes -r requirements-ci.lock
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: "22"
Expand All @@ -61,6 +66,8 @@ jobs:
- name: Frontend build contracts
run: node --test tests/test_frontend_build.mjs
- name: Worker tests
env:
PYTHON: python
run: npm run test:worker
- name: D1 migration checks
run: npm run cf:migrations:check
Expand Down
69 changes: 69 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,19 @@ A powerful proxy server that provides a unified interface for multiple LLM provi
- Nineteen AI
- Chutes AI
- 🧭 Operator control plane with provider health, circuit state, route traces, and request exploration
- 🔍 [Provider authenticity diagnostics](docs/provider-authenticity.md) give bounded, content-free conformance signals for each provider
- 🧩 Dashboard-managed `auto:<model>` priorities with safe rate-limit failover across providers
- 🔄 Four-state provider recovery with bounded parallel half-open probes
- 🩺 [Upstream outcomes](docs/upstream-outcomes.md) separate valid responses, credential rejection, throttling and ambiguous transport failure in circuit recovery
- 📊 Request, latency, response-class, and configured cost telemetry
- 📈 Opt-in Prometheus scrapes for public route health and admin-only request window metrics
- 🚀 Streaming support for compatible providers
- 🛂 Opt-in [first-event stream validation](docs/stream-preflight.md) before an `auto:` route commits to a provider
- 🎭 Cloudflare-native roleplay sessions with adaptive Kimi/GLM routing and durable continuity memory
- 🪟 Opt-in [candidate-specific roleplay context](docs/roleplay-candidate-context.md) fits protected fallback history to each model window
- ⚡ Configurable timeouts and retry mechanisms per provider
- 🔄 Automatic parameter handling and compatibility checks
- 🧮 [`/v1/messages/count_tokens`](docs/token-counting.md) can use explicitly configured native provider counts, with labeled local estimates otherwise

The rationale for selectively adopting OmniRoute-style operational features
without replacing MultiLLM's provider adapters is documented in
Expand All @@ -51,6 +57,69 @@ branches, explicit interrupted-response recovery, and Worker/Container release
checks. [External PostgreSQL and encrypted backups](docs/control-plane-storage.md)
keep control-plane settings independent of ephemeral Container storage.

### Opt-in gateway features

Most of these stay off until a setting or request header enables them, or an
operator runs a script. Partial usage accounting and the authenticated OpenAPI
document are always on. Each document lists its settings, and features that
store data name the D1 migration to apply before enabling them.

- [Cache policy isolation](docs/cache-policy.md): `RESPONSE_CACHE_POLICY_REVISION` keys cached chat responses by the current route and guardrail policy.
- [Partial usage and unknown cost](docs/usage-uncertainty.md): the usage ledger records a missing token count or price as unknown instead of zero.
- [Conversion fidelity diagnostics](docs/conversion-fidelity.md): sending `X-MultiLLM-Conversion-Report: 1` returns headers that rate a chat, responses or messages translation and list the affected fields.
- [OpenAPI contract](docs/openapi.md): `GET /openapi.json` returns the client API description to authenticated callers.
- [Versioned prompt templates](docs/prompt-templates.md): `PROMPT_TEMPLATES_ENABLED` adds admin-published templates and a render endpoint.
- [Capability probes](docs/capability-probes.md): an operator script checks tool calls, JSON schema output and vision on selected models.
- [Configuration snapshots](docs/config-snapshots.md): `CONFIG_SNAPSHOTS_ENABLED` adds reviewed, revisioned auto-route changes with a diff and audit records.
- [Health probe cost forecast](docs/probe-cost-forecast.md): an offline script bounds the API calls and spend of a probe plan.
- [Three-arm evaluation](docs/evaluation-noise-floor.md): `SHADOW_EVAL_NOISE_FLOOR_ENABLED` compares a candidate against two replays of the production model, and an offline script replays routing cost.
- [Quantization and precision preferences](docs/model-precision.md): `MODEL_PRECISION_PREFERENCE` orders catalog and intelligence candidates by model precision.
- [Upstream Retry-After advice](docs/retry-advice.md): `UPSTREAM_RETRY_AFTER_ADVICE_ENABLED` waits for a provider's bounded reset hint before an already-permitted retry.
- [Native edge request metrics](docs/edge-request-metrics.md): `NATIVE_EDGE_METRICS_ENABLED` records native Worker generations in the usage ledger and Prometheus without double-counting forwarded requests.
- [Prompt cache usage and cost](docs/prompt-cache-cost.md): `PROMPT_CACHE_USAGE_BUCKETS_ENABLED` records ordinary input, cache-read, cache-write and output tokens and costs separately (migration 0016).
- [Model-scoped credential cooldown](docs/model-cooldown.md): `MODEL_COOLDOWN_ENABLED` rests a pooled key for one model or quota bucket instead of the whole provider.
- [Zero-content retention](docs/zero-content-retention.md): `CONTENT_RETENTION_ENABLED` lets a policy or `X-MultiLLM-Retention: zero` keep request content out of caches, samples, roleplay memory and Knowledge memos.
- [Revision-aware configuration sync](docs/config-revision-sync.md): `CONFIG_REVISION_SYNC_ENABLED` reloads configuration from committed revisions and fails closed when key controls can't be verified (migration 0017).
- [Shared concurrency admission](docs/admission-leases.md): `ADMISSION_ENABLED` limits concurrent generations per account and model group across the Worker and Containers.
- [Disconnect-aware cancellation](docs/request-cancellation.md): a client disconnect closes the upstream request once and records the usage as unknown, not zero.
- [Rate-limit headers](docs/rate-limit-headers.md): `RATE_LIMIT_HEADERS_ENABLED` adds `X-MultiLLM-RateLimit-*` headers for the gateway's own request counters.
- [MCP contract digests](docs/mcp-contract-drift.md): `MCP_CONTRACT_DIGESTS_ENABLED` adds schema digests to `/mcp` discovery and rejects a call whose pinned digest no longer matches.
- [Prompt cache affinity](docs/prompt-cache-affinity.md): `PROMPT_CACHE_AFFINITY_ENABLED` sends a managed automatic route back to the credential that last completed the same reusable prompt prefix, so provider prompt caches are reused, without overriding eligibility.
- [Generation deadlines](docs/generation-deadline.md): `X-MultiLLM-Deadline-Ms` gives one generation a single time budget across setup, admission, retries, preflight and streaming, capped by `GENERATION_DEADLINE_MAX_MS`.
- [Shared generation cache](docs/shared-generation-cache.md): `GENERATION_CACHE_SHARED_ENABLED` with `GENERATION_CACHE_BACKEND=d1-r2` shares exact, deterministic, complete responses between Containers and the Worker, isolated by account and policy (migration 0019).
- [Gateway alerts](docs/gateway-alerts.md): `GATEWAY_ALERTS_ENABLED` sends content-free webhook alerts for spend, unpriced usage, open circuits, exhausted pools and provider failures (migration 0022).
- [Managed request idempotency](docs/managed-idempotency.md): `MANAGED_IDEMPOTENCY_ENABLED` accepts `Idempotency-Key` on managed non-streaming chat and replays a completed response instead of generating again (migration 0023).
- [Usage reservations](docs/usage-reservations.md): `USAGE_RESERVATIONS_ENABLED` keeps a budget hold when a provider call's outcome or usage is uncertain, until it is settled or reconciled (migration 0020).
- [Output schema validation](docs/output-schema-validation.md): `OUTPUT_SCHEMA_VALIDATION_ENABLED` lets a managed request ask the gateway to check the response against a JSON schema and return 502 when it does not match.
- [Deferred tool discovery](docs/deferred-tools.md): `DEFERRED_TOOLS_ENABLED` adds `multillm.tools.discover` to the MCP endpoints, with per-principal tool grants (migration 0018).
- [Protocol extras](docs/protocol-extras.md): `PROTOCOL_EXTRAS_ENABLED` keeps reviewed protocol-specific fields, such as Messages `top_k`, through translation; they reach a provider only if it admits the field.
- [Sticky session tiers](docs/session-tiers.md): `SESSION_TIER_MODE=sticky` keeps a session's approved model tier stable through tool calls in the main, delegation and aux lanes (migration 0021).
- [Hosted Responses state](docs/responses-state.md): `HOSTED_RESPONSES_ENABLED` lets a managed `/v1/responses` request with `gateway_state: true` keep its completed response under a gateway ID, continue from it with `previous_response_id`, and read or delete it later (migration 0028).
- [Gateway batches](docs/gateway-batches.md): `GATEWAY_BATCHES_ENABLED` adds OpenAI-compatible `/v1/files` and `/v1/batches` with a required spend budget, and `BATCH_SPILLOVER_ENABLED` turns a chat request sent with `X-MultiLLM-Priority: batch` and `Prefer: respond-async` into a one-item batch (migration 0027).
- [Observability exports](docs/observability-exports.md): `OBSERVABILITY_EXPORTERS_JSON` sends content-free request observations to the Langfuse or Helicone collectors you name.
- [Context pages](docs/context-pages.md): `CONTEXT_PAGING_ENABLED` lets a client that sends the `multillm_context_retrieve` capability replace older conversation turns with exact, owner-scoped page handles it can retrieve later (migration 0024).
- [Semantic generation cache](docs/semantic-generation-cache.md): `SEMANTIC_CACHE_ENABLED` with `SEMANTIC_CACHE_POLICY_JSON` reuses a stored answer for a near-identical final user message on opted-in routes or accounts, under strict matching and a per-lookup embedding spend cap (migration 0026).
- [Canary traffic](docs/canary-traffic.md): `CANARY_TRAFFIC_ENABLED` splits an auto route's sessions into baseline and candidate cohorts, in shadow or live mode, from a reviewed policy (migration 0025).
- [Bandit recommendations](docs/bandit-recommendations.md): `BANDIT_MODE` proposes a reviewed one-step route order change from stored evaluation results, without calling providers.
- [Local inference](docs/local-inference.md): `OLLAMA_BASE_URL`, `VLLM_BASE_URL`, `LM_STUDIO_BASE_URL`, `LLAMA_CPP_BASE_URL` and `SGLANG_BASE_URL` add explicitly configured OpenAI-compatible local servers as providers.
- [PII redaction](docs/pii-rehydration.md): `PII_REDACTION_ENABLED` with `PII_REDACTION_POLICY_JSON` replaces detected emails, phone numbers and card numbers with request-local placeholders before dispatch and restores them in JSON and streamed responses.
- [Prompt injection heuristics](docs/prompt-injection.md): `PROMPT_INJECTION_MODE=log` or `block` scores managed requests for injection and jailbreak patterns before dispatch.
- [Reset-aware pool scheduling](docs/pool-reset-scheduling.md): `POOL_RESET_SCHEDULING_ENABLED` skips pooled accounts whose quota is freshly exhausted and prefers the earliest reset, using only data the gateway already receives.
- [Latency SLO admission](docs/latency-slo.md): `LATENCY_SLO_MODE=reject` or `reroute` with `LATENCY_SLO_POLICY_JSON` predicts deadline risk from recent model measurements and can reject requests or reroute eligible automatic selections before dispatch.
- [Hedged auto requests](docs/hedged-requests.md): `HEDGED_REQUESTS_ENABLED` with `HEDGED_REQUESTS_POLICY_JSON` races at most two candidates on auto routes explicitly marked idempotent-safe, and accounts for every attempt.
- [Realtime WebSocket transport](docs/realtime.md): `REALTIME_ENABLED` with `REALTIME_PROVIDERS_JSON` bridges authenticated OpenAI-compatible Realtime sockets directly in the Worker, under a per-session cost cap (migration 0029).
- [Stream cost breaker](docs/stream-cost-breaker.md): `STREAM_COST_BREAKER_ENABLED` lets a key carry `max_stream_cost_usd`, and stops a stream with the protocol's error when its observed or conservatively estimated cost would pass the cap (migration 0030).
- [Learned provider cooldowns](docs/learned-cooldown.md): `LEARNED_COOLDOWN_MODE=shadow` or `apply` with `LEARNED_COOLDOWN_POLICY_JSON` narrows a bounded cooldown interval from natural throttle and recovery observations, without probes (migration 0031).
- [Context-leak canary tokens](docs/context-canary.md): `CONTEXT_CANARY_MODE=log` or `block` with `CONTEXT_CANARY_POLICY_JSON` detects opted-in disclosure of a request-specific marker. It does not claim complete leak detection.
- [Signed usage receipts](docs/usage-receipts.md): `USAGE_RECEIPTS_ENABLED` with a reviewed Ed25519 key writes a hash-chained receipt per settled usage event that its owner can fetch and verify, keeping unknown cost explicit (migration 0032).
- [Enterprise integration contracts](docs/enterprise-contracts.md): typed identity, tenancy, quota and credit boundaries for later organisation, SSO and payment modules. `ENTERPRISE_PREVIEW_ENABLED` shows a read-only administrator preview; nothing else activates.
- [Organisations and teams](docs/tenant-hierarchy.md): `ORGANISATIONS_ENABLED` adds optional organisation and team membership with a workspace identity the server verifies, and keeps each workspace's caches, idempotency, hosted state and usage separate (migration 0033).
- [Tenant governance](docs/tenant-governance.md): `TENANT_GOVERNANCE_ENABLED` intersects organisation, team and key grants, reserves organisation and team budgets together with the key's, and attributes usage to the verified workspace (migration 0034).
- [SAML federation](docs/saml-federation.md): `SAML_ENABLED` with `SAML_BROKER_URL`, `SAML_TRUST_CONFIG_JSON` and `SAML_CALLBACK_URL` signs gateway accounts in to the dashboard through an explicitly trusted SAML broker, only for subjects an administrator has linked to an account (migration 0035).
- [SCIM provisioning](docs/scim-provisioning.md): `SCIM_ENABLED` with `SCIM_TRUST_CONFIG_JSON` provisions scoped users and organisation teams through SCIM 2.0; it needs D1 storage and security revision sync (migration 0036).
- [Gateway credits](docs/credits-ledger.md): `CREDITS_ENABLED` keeps an append-only ledger of credits, holds and settlements, separate from usage estimates. `CREDITS_ENFORCEMENT=funded` or `all` makes requests reserve and settle credits (migration 0037).
- [Top-ups](docs/payment-billing.md): `PAYMENTS_ENABLED` with `PAYMENT_PROCESSOR_CONFIG_JSON` and `PAYMENT_WEBHOOK_KEY_REF` creates Stripe Checkout top-ups and credits explicit USD payments only from verified, idempotent webhooks (migration 0038).

## Setup

1. Clone the repository:
Expand Down
22 changes: 22 additions & 0 deletions app.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,25 +18,32 @@
is_api_request_path,
login_required,
)
from routes.config_snapshots import register_config_snapshot_routes
from routes.enterprise_preview import register_enterprise_preview_routes
from routes.tenants import register_tenant_routes
from routes.core import register_core_routes
from routes.dashboard_security import register_dashboard_security_routes
from routes.intelligence_media import register_intelligence_media_routes
from routes.knowledge import register_knowledge_routes
from routes.documentation import register_documentation_routes
from routes.free_routes import register_free_routes
from routes.gateway_mcp import register_gateway_mcp_routes
from routes.gateway_batches import register_gateway_batch_routes
from routes.context_pages import register_context_page_routes
from routes.optimized import register_optimized_routes
from routes.proxy import register_proxy_routes
from routes.status_page import register_status_routes
from routes.unified import register_unified_routes
from routes.media import register_media_routes
from routes.usage import register_usage_routes
from routes.usage_receipts import register_usage_receipt_routes
from routes.workbench import register_workbench_routes
from security_config import load_max_content_length, validate_runtime_secrets
from services.auth_service import AuthService
from services.cache_service import CacheService
from services.image_relay_catalog import ImageRelayCatalogRefresh
from services.provider_catalog_refresh import ProviderCatalogAutoRefresh
from services.gateway_extensions import gateway_callbacks, register_gateway_extensions, register_enterprise_features
from services.metrics_service import MetricsService
from services.proxy_service import ProxyService
from services import usage_ledger
Expand Down Expand Up @@ -139,11 +146,26 @@ def create_app() -> Flask:
register_core_routes(app)
register_dashboard_security_routes(app)
register_workbench_routes(app)
register_config_snapshot_routes(app)
register_enterprise_preview_routes(app)
register_tenant_routes(app, csrf=csrf)
register_enterprise_features(app, csrf)
register_shadow_eval_routes(app, csrf, AuthService, MetricsService, ProxyService)
register_knowledge_routes(app, csrf)
register_gateway_mcp_routes(app, csrf)
register_documentation_routes(app, AuthService, ProxyService)
register_usage_routes(app, csrf)
register_usage_receipt_routes(app, csrf)
register_gateway_extensions(app, callbacks=gateway_callbacks(csrf=csrf))
register_gateway_batch_routes(app)
from functools import partial
from route_helpers import api_authenticate_only
from services.context_pages import ContextPageService, PrivatePageStore, managed_authority
from services.intelligence_d1_store import request_private_intelligence
app.extensions["context_page_service"] = ContextPageService(PrivatePageStore(
partial(request_private_intelligence, endpoint="context-pages")))
register_context_page_routes(app, service=app.extensions["context_page_service"],
authorize=managed_authority, authenticate=api_authenticate_only)
# Restore the dashboard's recent requests from the durable ledger in the background.
usage_ledger.start(MetricsService.get_instance())

Expand Down
Loading
Loading