From 220fbde9f2a51fb1a5b5b12feca6928030412cb2 Mon Sep 17 00:00:00 2001 From: Abhyudaya Baiju Tharakan Date: Fri, 9 Oct 2026 03:25:06 +0530 Subject: [PATCH] feat(metrics): add opt-in native edge request metrics Record authenticated native Worker generations in the D1 usage ledger and expose isolate-local Prometheus counters behind NATIVE_EDGE_METRICS_ENABLED, default off. Adds a static lifecycle hook chain for native dispatch. --- cloudflare-worker.mjs | 31 ++- docs/edge-request-metrics.md | 100 +++++++ package.json | 2 +- tests/fixtures/secret_firewall_coverage.json | 3 + tests/helpers/load_cloudflare_worker.mjs | 1 + tests/test_edge_request_metrics_worker.mjs | 274 +++++++++++++++++++ worker/gateway-extensions.mjs | 72 +++++ worker/gateway-lifecycle.mjs | 23 ++ worker/request-telemetry.mjs | 198 ++++++++++++++ worker/usage-ledger-d1.mjs | 18 ++ 10 files changed, 708 insertions(+), 14 deletions(-) create mode 100644 docs/edge-request-metrics.md create mode 100644 tests/test_edge_request_metrics_worker.mjs create mode 100644 worker/gateway-extensions.mjs create mode 100644 worker/gateway-lifecycle.mjs create mode 100644 worker/request-telemetry.mjs diff --git a/cloudflare-worker.mjs b/cloudflare-worker.mjs index 4578979d..1fb7e6a9 100644 --- a/cloudflare-worker.mjs +++ b/cloudflare-worker.mjs @@ -1,3 +1,4 @@ +import { nativeGenerationFetch, withForwardedCorrelation, withNativeMetrics } from "./worker/gateway-extensions.mjs"; import { firewallFetch } from "./worker/secret-firewall.mjs"; import { Container, ContainerProxy, getContainer } from "@cloudflare/containers"; import { collectContainerEnv } from "./worker/container-env.mjs"; @@ -1446,7 +1447,7 @@ function createOpencodeStreamResponse(upstreamResponse) { }); } -async function handleDirectOpencodeRequest(request, env, requestUrl) { +async function handleDirectOpencodeRequest(request, env, requestUrl, ctx) { const providedToken = extractOpencodeCallerToken(request); if (!(await timingSafeTokenMatch(providedToken, env.ADMIN_API_KEY))) { return applyCorsHeaders(request, buildUnauthorizedResponse(), env); @@ -1492,10 +1493,10 @@ async function handleDirectOpencodeRequest(request, env, requestUrl) { signal: request.signal, ...(bodyAllowed && request.body ? { duplex: "half" } : {}), }); - const upstreamResponse = await firewallFetch(upstreamRequest, env, { + const upstreamResponse = await nativeGenerationFetch(upstreamRequest, env, ctx, { route: requestUrl.pathname, principal: { id: env.ADMIN_USERNAME || "admin" }, provider: requestUrl.pathname.split("/")[1], - }); + }, async (upstreamRequest, env, authority) => await firewallFetch(upstreamRequest, env, authority)); if (isOpencodeNativeRequest(requestUrl.pathname, request.method)) { const downstreamResponse = new Response(upstreamResponse.body, { @@ -1580,7 +1581,7 @@ async function authorizeContainerOpencodeRequest(request, env, requestUrl) { return null; } -async function handleDirectLinkApiRequest(request, env, requestUrl) { +async function handleDirectLinkApiRequest(request, env, requestUrl, ctx) { const providedToken = extractLinkApiCallerToken(request, requestUrl); if (!(await timingSafeTokenMatch(providedToken, env.ADMIN_API_KEY))) { return applyCorsHeaders(request, buildLinkApiUnauthorizedResponse(), env); @@ -1627,10 +1628,10 @@ async function handleDirectLinkApiRequest(request, env, requestUrl) { signal: normalizedRequest.signal, ...(bodyAllowed && normalizedRequest.body ? { duplex: "half" } : {}), }); - const upstreamResponse = await firewallFetch(upstreamRequest, env, { + const upstreamResponse = await nativeGenerationFetch(upstreamRequest, env, ctx, { route: requestUrl.pathname, principal: { id: env.ADMIN_USERNAME || "admin" }, provider: requestUrl.pathname.split("/")[1], - }); + }, async (upstreamRequest, env, authority) => await firewallFetch(upstreamRequest, env, authority)); return applyCorsHeaders( request, @@ -1643,7 +1644,7 @@ async function handleDirectLinkApiRequest(request, env, requestUrl) { ); } -async function handleDirectCodexEasyRequest(request, env, requestUrl) { +async function handleDirectCodexEasyRequest(request, env, requestUrl, ctx) { const providedToken = extractBearerToken(request); if (!(await timingSafeTokenMatch(providedToken, env.ADMIN_API_KEY))) { return applyCorsHeaders(request, buildUnauthorizedResponse(), env); @@ -1685,10 +1686,10 @@ async function handleDirectCodexEasyRequest(request, env, requestUrl) { signal: normalizedRequest.signal, ...(bodyAllowed && normalizedRequest.body ? { duplex: "half" } : {}), }); - const upstreamResponse = await firewallFetch(upstreamRequest, env, { + const upstreamResponse = await nativeGenerationFetch(upstreamRequest, env, ctx, { route: requestUrl.pathname, principal: { id: env.ADMIN_USERNAME || "admin" }, provider: requestUrl.pathname.split("/")[1], - }); + }, async (upstreamRequest, env, authority) => await firewallFetch(upstreamRequest, env, authority)); return applyCorsHeaders( request, @@ -1982,7 +1983,7 @@ export default { if (codexEasyPath) { try { - return await handleDirectCodexEasyRequest(request, env, requestUrl); + return await handleDirectCodexEasyRequest(request, env, requestUrl, ctx); } catch (error) { logStructuredError("direct_codex_easy_fetch_failed", error); return applyCorsHeaders( @@ -2001,7 +2002,7 @@ export default { if (linkapiPath) { try { - return await handleDirectLinkApiRequest(request, env, requestUrl); + return await handleDirectLinkApiRequest(request, env, requestUrl, ctx); } catch (error) { logStructuredError("direct_linkapi_fetch_failed", error); return applyCorsHeaders( @@ -2021,7 +2022,7 @@ export default { if (opencodePath) { try { if (env.OPENCODE_EDGE_FETCH === "true") { - return await handleDirectOpencodeRequest(request, env, requestUrl); + return await handleDirectOpencodeRequest(request, env, requestUrl, ctx); } const rejected = await authorizeContainerOpencodeRequest( request, @@ -2059,6 +2060,7 @@ export default { await withAccessIdentity(request, env, headers); headers.delete("content-length"); headers.delete("host"); + withForwardedCorrelation(headers, env); headers.set("x-forwarded-proto", requestUrl.protocol.slice(0, -1)); headers.set("x-forwarded-host", requestUrl.host); headers.set("x-multillm-external-origin", requestUrl.origin); @@ -2074,7 +2076,10 @@ export default { signal: request.signal, ...(bodyAllowed && request.body ? { duplex: "half" } : {}), }); - const response = await container.fetch(forwardedRequest); + let response = await container.fetch(forwardedRequest); + if (requestUrl.pathname === "/v1/metrics/prometheus") { + response = await withNativeMetrics(response, env); + } if (await isContainerPackageStartupFailure(response)) { logStructuredError("container_start_failed_response", new Error("Container unavailable")); if (rootPath) { diff --git a/docs/edge-request-metrics.md b/docs/edge-request-metrics.md new file mode 100644 index 00000000..ae3697e5 --- /dev/null +++ b/docs/edge-request-metrics.md @@ -0,0 +1,100 @@ +# Native edge request metrics + +`NATIVE_EDGE_METRICS_ENABLED` defaults to false. Unset, empty and `false` values preserve +existing native responses, headers, streams, ledger writes and logs. No lifecycle hook +runs. Set the Worker variable to `true` to enable observation; values are trimmed and +case-insensitive. A malformed setting disables the feature and emits one content-free +warning per isolate. For example: + +```json +{"NATIVE_EDGE_METRICS_ENABLED": "true", "PROMETHEUS_ENABLED": "true"} +``` + +To disable it: + +```json +{"NATIVE_EDGE_METRICS_ENABLED": "false"} +``` + +## Scope and identity + +After existing route authentication, native Codex Easy, LinkAPI and edge OpenCode +POST generations are observed. Kimi generations and non-edge OpenCode currently go +through the Container and retain Flask ledger ownership. Models, health, token-count, +Knowledge and roleplay paths retain their existing accounting. No retry is introduced. + +Each native attempt gets a generated request ID and a SHA-256 digest of the verified +native admin identity with an `edge:` namespace. Client identity and request-ID headers +cannot supply these values. This identity is separate from the Flask username principal; +reports and budgets must not assume the two namespaces are combined. Selected models +come from the authenticated request, never an upstream response's model field. The +native admin route currently has no narrower per-model grant. A request model that is +missing, malformed or outside the parse bound remains unknown. + +Enabled Container forwarding strips client copies of edge principal, request ID and +ledger-owner headers and generates `x-request-id`. Flask still authenticates the caller +and owns its usage row. Edge forwarding never writes a second row. + +## Usage, failures and cost + +The observer accepts OpenAI Chat/Responses and Anthropic usage envelopes, retaining +input and output counts independently. Zero is measured zero; missing, malformed or +negative components remain null. Split SSE events merge the latest valid component. +Complete measured usage on a completed successful response has basis `measured`; +partial, usage-less, canceled, failed and incomplete attempts have basis `unknown`. +No token estimates are invented. The observer accepts a code-supplied `usageEstimate` +collaborator value, fills only missing components and marks those observations +`estimated`. Measured zero is preserved. Usage-less and canceled attempts stay unknown +even with estimates. The registered native routes supply no estimates by default. + +Costs use `MODEL_PRICING_USD_PER_MILLION` with explicit input/output prices, optional +per-request prices, provider wildcards and the global wildcard. Costs remain null without +complete successful usage and valid prices. They are configured estimates, not invoices. +Known configured costs map to the existing D1 `usage` basis; unknown costs map to null. +Prompt-cache usage buckets are not priced separately. + +Native downstream HTTP status, headers and bytes are unchanged. Ledger-only status 499 +means cancellation, 502 means a transport or in-stream error, and 520 means incomplete +or unknown usage on an otherwise successful HTTP response. Upstream HTTP errors keep +their status. These classifications prevent unknown attempts from becoming successes +in daily rollups. Partial counts remain useful without treating missing cost as free. + +## Operational bounds and retention + +Response observation uses one pull-driven reader with no response clone, tee or +full-body queue. SSE line/event carry and non-stream JSON observation are bounded at +64 KiB. Oversized events are discarded for accounting and bytes are still forwarded. +Parsing recovers at subsequent event boundaries. JSON responses above the bound have +unknown usage. Authenticated request-model extraction also reads at most 64 KiB of a +request clone; its reader is canceled afterwards. First-token latency is recorded only +for recognizable streaming content, reasoning or tool deltas; it stays unknown for +non-stream responses. Duration uses a monotonic clock, includes dispatch, and is clamped +to the ledger's 24-hour bound. A terminal SSE marker and complete usage are required +for successful stream accounting. + +Cancellation and body/fetch failures finalize once. Finalization uses `ctx.waitUntil` +when available; missing/broken D1 emits fixed content-free errors and never replaces +the native result. This is best-effort telemetry, not a billing guarantee. No prompt, +tool arguments, output, raw key, exception detail or key prefix is persisted or logged +by this feature. The existing `usage_events`, `usage_daily` and `usage_batches` schema +and pruning policy apply; there is no new retention store or migration. Request IDs +and model identity are durable; TTFT and richer outcomes are not additional D1 columns. + +With both flags enabled, the existing authenticated `/v1/metrics/prometheus` endpoint +appends native counters using the existing exporter's content type and label escaping. Authentication +errors and non-Prometheus responses pass through unchanged. The public status endpoint +is unchanged. Labels contain only four fixed provider names and five fixed outcomes: +at most 120 series, with no principal, request ID, model or content labels. Counters +track requests, duration, observed TTFT and its coverage, measured usage coverage and +priced coverage. Counters are isolate-local and reset when the isolate restarts; a +scrape does not aggregate other isolates. D1 remains the durable count source. + +## Static lifecycle integration + +`createGatewayLifecycle` exposes `authorize`, `admit`, `before_dispatch`, `observe` and +`finalize` in registration order with one finalizer per request. Hook context contains +only content-free metadata. Native dispatch accepts code-supplied named collaborators; +there is no dynamic import, executable configuration or discovery. Finalization tries +every collaborator even if one fails, then reports the failure without changing response +bytes. Other gateway features attach to native dispatch as code-registered +collaborators on these hooks. diff --git a/package.json b/package.json index 42ee5f27..6fea8972 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "lint": "ruff check . --select E9,F63,F7,F82", "typecheck": "mypy --ignore-missing-imports --follow-imports=skip --disable-error-code import-untyped --disable-error-code var-annotated --disable-error-code arg-type --disable-error-code assignment --disable-error-code operator app.py routes services providers streaming", "test": "python -m pytest -q", - "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs", + "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_edge_request_metrics_worker.mjs", "db:migrate": "python scripts/validate_sqlite_schema.py --apply", "db:validate": "python scripts/validate_sqlite_schema.py", "security:secrets": "python scripts/check_static_secrets.py", diff --git a/tests/fixtures/secret_firewall_coverage.json b/tests/fixtures/secret_firewall_coverage.json index ff0fe87b..70679946 100644 --- a/tests/fixtures/secret_firewall_coverage.json +++ b/tests/fixtures/secret_firewall_coverage.json @@ -229,6 +229,8 @@ "worker/cors-policy.mjs", "worker/d1-schema.mjs", "worker/fallback-page.mjs", + "worker/gateway-extensions.mjs", + "worker/gateway-lifecycle.mjs", "worker/gpt-image-moderation.mjs", "worker/health-schedule.mjs", "worker/intelligence-auth-d1.mjs", @@ -293,6 +295,7 @@ "worker/opencode/reasoning-response.mjs", "worker/prometheus.mjs", "worker/prompt-templates-d1.mjs", + "worker/request-telemetry.mjs", "worker/roleplay/candidate-context.mjs", "worker/roleplay/capacity.mjs", "worker/roleplay/checkpoint.mjs", diff --git a/tests/helpers/load_cloudflare_worker.mjs b/tests/helpers/load_cloudflare_worker.mjs index 21205186..a2b690df 100644 --- a/tests/helpers/load_cloudflare_worker.mjs +++ b/tests/helpers/load_cloudflare_worker.mjs @@ -739,6 +739,7 @@ export async function loadWorkerModule() { .replace('from "./worker/health-schedule.mjs";', `from "${new URL("../../worker/health-schedule.mjs", import.meta.url)}";`) .replace('from "./worker/status-page.mjs";', `from "${new URL("../../worker/status-page.mjs", import.meta.url)}";`) .replace('from "./worker/intelligence-outbound.mjs";', `from "${new URL("../../worker/intelligence-outbound.mjs", import.meta.url)}";`) + .replace('from "./worker/gateway-extensions.mjs";', `from "${new URL("../../worker/gateway-extensions.mjs", import.meta.url)}";`) .replace('from "./worker/client-headers.mjs";', `from "${new URL("../../worker/client-headers.mjs", import.meta.url)}";`) .replace('from "./worker/opencode-session.mjs";', `from "${new URL("../../worker/opencode-session.mjs", import.meta.url)}";`) .replace('from "./worker/opencode/reasoning-request.mjs";', `from "${new URL("../../worker/opencode/reasoning-request.mjs", import.meta.url)}";`) diff --git a/tests/test_edge_request_metrics_worker.mjs b/tests/test_edge_request_metrics_worker.mjs new file mode 100644 index 00000000..04e713fb --- /dev/null +++ b/tests/test_edge_request_metrics_worker.mjs @@ -0,0 +1,274 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { readFile } from "node:fs/promises"; +import { spawn } from "node:child_process"; +import { recordNativeUsage } from "../worker/usage-ledger-d1.mjs"; +import { createGatewayLifecycle } from "../worker/gateway-lifecycle.mjs"; +import { createUsageObserver, observeNativeResponse, renderNativeMetrics } from "../worker/request-telemetry.mjs"; +import { nativeGenerationFetch, withForwardedCorrelation, withNativeMetrics } from "../worker/gateway-extensions.mjs"; + +// Keep the shared loader untouched; register the owned import in this test's copy. +const helperUrl = new URL("./helpers/load_cloudflare_worker.mjs", import.meta.url); +const helper = (await readFile(helperUrl, "utf8")) + .replaceAll("import.meta.url", JSON.stringify(helperUrl.href)) + .replace("const patchedSource = source", `const patchedSource = source.replace('from "./worker/gateway-extensions.mjs";', 'from "${new URL("../worker/gateway-extensions.mjs", import.meta.url)}";')`); +const registeredHelperUrl = `data:text/javascript;base64,${Buffer.from(helper).toString("base64")}`; +const worker = (await (await import(registeredHelperUrl)).loadWorkerModule()).default; +const encoder = new TextEncoder(); +const chunks = values => new ReadableStream({ start(c) { for (const value of values) c.enqueue(encoder.encode(value)); c.close(); } }); +function fixture() { + const rows = [], pending = []; + const env = { NATIVE_EDGE_METRICS_ENABLED: "true", ADMIN_API_KEY: "synthetic-admin", ADMIN_USERNAME: "test-owner", + CODEX_EASY_API_KEY: "synthetic-upstream", INTELLIGENCE_DB: { + prepare(sql) { return { bind(...values) { return { sql, values }; } }; }, + async batch(statements) { rows.push(...JSON.parse(statements[1].values[0])); return [{ meta: { changes: 1 } }]; }, + } }; + return { env, rows, ctx: { waitUntil(p) { pending.push(p); } }, async flush() { await Promise.all(pending); } }; +} +const req = (path = "/codex-easy/v1/chat/completions", token = "synthetic-admin", signal) => new Request(`https://gateway.example${path}`, { + method: "POST", headers: { authorization: `Bearer ${token}`, "content-type": "application/json", "x-multillm-principal": "spoofed", "x-request-id": "spoofed" }, + body: JSON.stringify({ model: "test-model", messages: [{ role: "user", content: "private-prompt" }] }), signal, +}); +async function upstream(response, run) { + const original = globalThis.fetch; + globalThis.fetch = async () => response; + try { return await run(); } finally { globalThis.fetch = original; } +} + +test("default and empty settings preserve native bytes, headers and storage", async () => { + for (const setting of [undefined, "", "false"]) { + const f = fixture(); f.env.NATIVE_EDGE_METRICS_ENABLED = setting; + const raw = '{"usage":{"prompt_tokens":2,"completion_tokens":0},"choices":[]}'; + const response = await upstream(new Response(raw, { headers: { "content-type": "application/json" } }), () => worker.fetch(req(), f.env, f.ctx)); + assert.equal(await response.text(), raw); await f.flush(); assert.equal(f.rows.length, 0); + const headers = new Headers({ "x-request-id": "client" }); + withForwardedCorrelation(headers, f.env); assert.equal(headers.get("x-request-id"), "client"); + } +}); + +test("registered native success stores opaque authorized identity and exact bytes", async () => { + const f = fixture(); f.env.MODEL_PRICING_USD_PER_MILLION = '{"codex-easy:test-model":{"input":1,"output":2}}'; + const raw = '{"model":"untrusted-model","usage":{"prompt_tokens":4,"completion_tokens":0},"choices":[]}'; + const response = await upstream(new Response(raw, { headers: { "content-type": "application/json" } }), () => worker.fetch(req(), f.env, f.ctx)); + assert.equal(await response.text(), raw); await f.flush(); + assert.equal(f.rows.length, 1); const row = f.rows[0]; + assert.match(row.principal, /^edge:[0-9a-f]{64}$/); assert.notEqual(row.request_id, "spoofed"); + assert.equal(row.selected_model, "codex-easy:test-model"); assert.equal(row.key_prefix, null); + assert.deepEqual([row.input_tokens, row.output_tokens, row.cost_usd, row.cost_basis], [4, 0, 0.000004, "usage"]); + assert.ok(!JSON.stringify(row).includes("private-prompt")); +}); + +test("unauthenticated spoof creates no native row", async () => { + const f = fixture(); const response = await worker.fetch(req(undefined, "wrong"), f.env, f.ctx); + assert.equal(response.status, 401); await f.flush(); assert.equal(f.rows.length, 0); +}); + +test("split SSE usage preserves bytes and finalizes exactly once", async () => { + const values = ['data: {"choices":[{"delta":{"content":"private-output"}}]}\r\n\r\n', 'data: {"usage":{"prompt_to', 'kens":7}}\n\n', 'data: {"usage":{"completion_tokens":3}}\n\ndata: [DONE]\n\n']; + const events = [], f = fixture(); + const response = await observeNativeResponse(new Response(chunks(values), { headers: { "content-type": "text/event-stream" } }), { + provider: "codex-easy", model: "test-model", principal: "edge:trusted", requestId: "req-test", endpoint: "/codex-easy/v1/chat/completions", startedAt: 0, + }, { clock: () => 12, finalize: event => events.push(event) }); + assert.equal(await response.text(), values.join("")); assert.equal(events.length, 1); + assert.deepEqual([events[0].input_tokens, events[0].output_tokens, events[0].usage_basis, events[0].outcome, events[0].ttft_ms], [7, 3, "measured", "success", 12]); +}); + +test("parser carry stays at most 64 KiB and recovers after oversized lines", () => { + const parser = createUsageObserver(true); + parser.feed(encoder.encode('data: ' + 'x'.repeat(200000))); assert.ok(parser.carryBytes <= 65536); + parser.feed(encoder.encode('\n\ndata: {"usage":{"input_tokens":0,"output_tokens":2}}\n\ndata: [DONE]\n\n')); + assert.deepEqual(parser.finish().usage, { input_tokens: 0, output_tokens: 2 }); +}); + +test("partial usage, missing usage, stream error and cancellation remain unknown", async () => { + for (const raw of ['data: {"usage":{"input_tokens":2}}\n\ndata: [DONE]\n\n', 'data: [DONE]\n\n', 'data: {"error":{"message":"private-error"}}\n\n']) { + const events = []; + const response = await observeNativeResponse(new Response(raw, { headers: { "content-type": "text/event-stream" } }), { provider: "linkapi", principal: "edge:trusted", startedAt: 0 }, { finalize: e => events.push(e), clock: () => 5 }); + await response.text(); assert.equal(events.length, 1); assert.notEqual(events[0].outcome, "success"); assert.equal(events[0].cost_usd, null); + } + const events = []; let canceled = 0; + const response = await observeNativeResponse(new Response(new ReadableStream({ pull(c) { c.enqueue(encoder.encode('data: {}\n\n')); }, cancel() { canceled++; } }), { headers: { "content-type": "text/event-stream" } }), { provider: "linkapi", principal: "edge:trusted", startedAt: 0 }, { finalize: e => events.push(e) }); + const reader = response.body.getReader(); await reader.read(); await reader.cancel("private-reason"); + assert.equal(canceled, 1); assert.equal(events.length, 1); assert.equal(events[0].outcome, "canceled"); assert.equal(events[0].status, 499); +}); + +test("lifecycle is ordered, static and finalize is idempotent under concurrency", async () => { + const seen = []; const lifecycle = createGatewayLifecycle([{ authorize: () => seen.push("authorize"), admit: () => seen.push("admit"), before_dispatch: () => seen.push("before_dispatch"), observe: () => seen.push("observe"), finalize: () => seen.push("finalize") }]); + await lifecycle.authorize({}); await lifecycle.admit({}); await lifecycle.before_dispatch({}); await lifecycle.observe({}); + await Promise.all([lifecycle.finalize({}), lifecycle.finalize({})]); + assert.deepEqual(seen, ["authorize", "admit", "before_dispatch", "observe", "finalize"]); +}); + +test("forwarded Container traffic gets trusted correlation and only its own ledger row", async () => { + const f = fixture(); let calls = 0; + f.env.MULTILLM_PROXY_CONTAINER = { getByName() { return { async fetch(request) { + calls++; assert.notEqual(request.headers.get("x-request-id"), "spoofed"); assert.equal(request.headers.get("x-multillm-principal"), null); + f.rows.push({ owner: "flask" }); return new Response("unchanged"); + } }; } }; + const response = await worker.fetch(req("/kimi-code/v1/chat/completions"), { ...f.env, KIMI_CODE_API_KEY: "synthetic-kimi" }, f.ctx); + assert.equal(await response.text(), "unchanged"); await f.flush(); assert.equal(calls, 1); assert.deepEqual(f.rows, [{ owner: "flask" }]); +}); + +test("W5 registered private Prometheus endpoint appends bounded content-free metrics", async () => { + const f = fixture(); + f.env.PROMETHEUS_ENABLED = "true"; + f.env.MULTILLM_PROXY_CONTAINER = { getByName() { return { fetch: async () => new Response("# upstream metrics\n", { headers: { "content-type": "text/plain; version=0.0.4; charset=utf-8" } }) }; } }; + const response = await worker.fetch(new Request("https://gateway.example/v1/metrics/prometheus"), f.env, f.ctx); + const body = await response.text(); assert.match(body, /multillm_native_requests_total/); + for (const privateValue of ["private-prompt", "private-output", "test-owner", "test-model", "synthetic-admin"]) assert.ok(!body.includes(privateValue)); + const denied = await withNativeMetrics(new Response("denied", { status: 401 }), f.env); assert.equal(await denied.text(), "denied"); + assert.ok(Buffer.byteLength(renderNativeMetrics()) < 16384); +}); + + +test("semantic SSE errors are ledger errors even with upstream HTTP 200", async () => { + const events = []; + const response = await observeNativeResponse(new Response('data: {"error":{"message":"private"}}\n\n', { headers: { "content-type": "text/event-stream" } }), { startedAt: 0, provider: "opencode" }, { finalize: event => events.push(event) }); + assert.equal(response.status, 200); await response.text(); + assert.equal(events[0].outcome, "upstream_error"); assert.equal(events[0].status, 502); +}); + +test("registered Link and OpenCode native paths observe errors without changing them", async () => { + for (const path of ["/linkapi/v1/chat/completions", "/opencode/v1/chat/completions"]) { + const f = fixture(); f.env.LINKAPI_KEY = "synthetic-link"; f.env.OPENCODE_GO_API_KEY = "synthetic-open"; f.env.OPENCODE_EDGE_FETCH = "true"; + const raw = '{"error":{"message":"provider error"}}'; + const response = await upstream(new Response(raw, { status: 429, headers: { "content-type": "application/json", "retry-after": "3" } }), () => worker.fetch(req(path), f.env, f.ctx)); + assert.equal(response.status, 429); assert.equal(await response.text(), raw); await f.flush(); + assert.equal(f.rows.length, 1); assert.equal(f.rows[0].status, 429); assert.equal(f.rows[0].cost_usd, null); + } +}); + +test("injected named hooks see only metadata and stay dormant while disabled", async () => { + for (const enabled of ["false", "true"]) { + const f = fixture(); f.env.NATIVE_EDGE_METRICS_ENABLED = enabled; const seen = []; + const hooks = Object.fromEntries(["authorize", "admit", "before_dispatch", "observe", "finalize"].map(name => [name, context => { + assert.ok(!JSON.stringify(context).includes("private-prompt")); seen.push(name); + }])); + const response = await nativeGenerationFetch(req(), f.env, f.ctx, { route: "/codex-easy/v1/chat/completions", principal: { id: "trusted" }, provider: "codex-easy" }, async () => new Response('{"usage":{"input_tokens":0,"output_tokens":0}}'), [hooks]); + await response.text(); await f.flush(); + assert.deepEqual(seen, enabled === "false" ? [] : ["authorize", "admit", "before_dispatch", "observe", "finalize"]); + } +}); + +test("upstream exceptions, body failures and abort finalize once without hiding errors", async () => { + const f = fixture(); + const authority = { route: "/codex-easy/v1/chat/completions", provider: "codex-easy", principal: { id: "trusted" } }; + await assert.rejects(nativeGenerationFetch(req(), f.env, f.ctx, authority, async () => { throw new Error("transport failed"); }), /transport failed/); + await f.flush(); assert.equal(f.rows.length, 1); assert.equal(f.rows[0].status, 502); + const events = []; + const failed = await observeNativeResponse(new Response(new ReadableStream({ pull(controller) { controller.error(new Error("body failed")); } })), { provider: "codex-easy" }, { finalize: event => events.push(event) }); + await assert.rejects(failed.text(), /body failed/); assert.equal(events.length, 1); assert.equal(events[0].outcome, "transport_error"); + const abortEvents = [], abort = new AbortController(); + const hanging = await observeNativeResponse(new Response(new ReadableStream({ pull() {} })), { provider: "linkapi" }, { signal: abort.signal, finalize: event => abortEvents.push(event) }); + const reading = hanging.text(); abort.abort(); await assert.rejects(reading, { name: "AbortError" }); + await new Promise(resolve => setImmediate(resolve)); assert.equal(abortEvents.length, 1); assert.equal(abortEvents[0].usage_basis, "unknown"); +}); + +test("simultaneous native requests each write exactly one independent row", async () => { + const f = fixture(); const authority = { route: "/codex-easy/v1/chat/completions", provider: "codex-easy", principal: { id: "trusted" } }; + await Promise.all(Array.from({ length: 8 }, async () => { + const response = await nativeGenerationFetch(req(), f.env, f.ctx, authority, async () => new Response('{"usage":{"input_tokens":1,"output_tokens":1}}')); + await response.text(); + })); + await f.flush(); assert.equal(f.rows.length, 8); assert.equal(new Set(f.rows.map(row => row.request_id)).size, 8); +}); + +test("malformed flag warns once without its value, and storage failures preserve response", async () => { + const original = console.error, logs = []; console.error = value => logs.push(value); + try { + const f = fixture(); f.env.NATIVE_EDGE_METRICS_ENABLED = "private-invalid"; + for (let i = 0; i < 2; i++) { + const response = await nativeGenerationFetch(req(), f.env, f.ctx, { provider: "linkapi" }, async () => new Response("unchanged")); + assert.equal(await response.text(), "unchanged"); + } + assert.equal(logs.length, 1); assert.ok(!logs.join("").includes("private-invalid")); + f.env.NATIVE_EDGE_METRICS_ENABLED = "true"; + f.env.INTELLIGENCE_DB.batch = async () => { throw new Error("private database detail"); }; + const raw = '{"usage":{"input_tokens":1,"output_tokens":1}}'; + const response = await upstream(new Response(raw), () => worker.fetch(req(), f.env, f.ctx)); + assert.equal(await response.text(), raw); await f.flush(); + assert.ok(logs.some(value => value.includes("native_usage_storage_failed"))); assert.ok(!logs.join("").includes("private database detail")); + } finally { console.error = original; } +}); + + +test("explicit estimate collaborator retains measured zero and marks estimated coverage", async () => { + const events = []; + const response = await observeNativeResponse(new Response('{"usage":{"input_tokens":0}}'), { provider: "linkapi", model: "test-model" }, { + usageEstimate: { input_tokens: 99, output_tokens: 2 }, env: { MODEL_PRICING_USD_PER_MILLION: '{"linkapi:test-model":{"input":1,"output":1}}' }, finalize: event => events.push(event), + }); + await response.text(); assert.equal(events[0].input_tokens, 0); assert.equal(events[0].output_tokens, 2); + assert.equal(events[0].usage_basis, "estimated"); assert.equal(events[0].cost_basis, "estimate"); + const unknown = []; + const missing = await observeNativeResponse(new Response('{}'), { provider: "linkapi" }, { usageEstimate: { input_tokens: 2, output_tokens: 2 }, finalize: event => unknown.push(event) }); + await missing.text(); assert.equal(unknown[0].outcome, "unknown"); assert.equal(unknown[0].usage_basis, "unknown"); +}); + +test("multiline UTF-8 SSE and large JSON keep bounds and original bytes", async () => { + const parser = createUsageObserver(true), raw = 'data: {"usage":\r\ndata: {"input_tokens":0,"output_tokens":0}}\r\n\r\ndata: [DONE]\r\n\r\n'; + const bytes = encoder.encode(raw); for (const byte of bytes) { parser.feed(new Uint8Array([byte])); assert.ok(parser.carryBytes <= 65536); } + assert.deepEqual(parser.finish().usage, { input_tokens: 0, output_tokens: 0 }); + const events = [], big = JSON.stringify({ output: "漢".repeat(40000), usage: { input_tokens: 2, output_tokens: 3 } }); + const response = await observeNativeResponse(new Response(big), { provider: "linkapi" }, { finalize: event => events.push(event) }); + assert.equal(await response.text(), big); assert.equal(events[0].usage_basis, "unknown"); assert.equal(events[0].cost_usd, null); +}); + +test("all finalizers run once even if a collaborator fails", async () => { + let calls = 0; + const lifecycle = createGatewayLifecycle([{ finalize() { throw new Error("hook failed"); } }, { finalize() { calls++; } }]); + await assert.rejects(lifecycle.finalize({}), /hook failed/); await assert.rejects(lifecycle.finalize({}), /hook failed/); + assert.equal(calls, 1); +}); + +test("native ledger statements preserve nullable usage and batch idempotency in SQLite", async () => { + const statements = []; + const env = { INTELLIGENCE_DB: { + prepare(sql) { return { bind(...values) { return { sql, values }; } }; }, + async batch(batch) { statements.push(...batch); return [{ meta: { changes: 1 } }]; }, + } }; + const event = { provider: "codex-easy", model: "test-model", principal: "edge:trusted", endpoint: "/codex-easy/v1/chat/completions", + status: 520, duration_ms: 9, input_tokens: 0, output_tokens: null, cost_usd: null, cost_basis: null, requestId: crypto.randomUUID() }; + await recordNativeUsage(env, event); await recordNativeUsage(env, event); + const schema = (await Promise.all(["0003_control_users.sql", "0007_usage_ledger.sql"].map(name => + readFile(new URL(`../intelligence-migrations/${name}`, import.meta.url), "utf8")))).join("\n"); + const code = `import json,sqlite3,sys +payload=json.loads(sys.argv[1]) +db=sqlite3.connect(":memory:") +db.executescript(payload["schema"]) +for statement in payload["statements"]: + db.execute(statement["sql"],statement["values"]) +print(json.dumps({"events":db.execute("SELECT COUNT(*) FROM usage_events").fetchone()[0],"usage":db.execute("SELECT input_tokens,output_tokens,cost_usd FROM usage_events").fetchone(),"daily":db.execute("SELECT requests,errors,priced_requests FROM usage_daily").fetchone()}))`; + const result = await new Promise((resolve, reject) => { + const child = spawn("python3", + ["-I", "-c", code, JSON.stringify({ schema, statements })], { + cwd: new URL("../", import.meta.url), stdio: ["ignore", "pipe", "pipe"], timeout: 10000, + }); + let stdout = "", stderr = ""; + child.stdout.on("data", value => { stdout += value; }); + child.stderr.on("data", value => { stderr += value; }); + child.on("error", reject); + child.on("close", status => resolve({ status, stdout, stderr })); + }); + assert.equal(result.status, 0, result.stderr || result.error?.message); + assert.deepEqual(JSON.parse(result.stdout), { events: 1, usage: [0, null, null], daily: [1, 1, 0] }); +}); + +test("malformed optional provider fields never change native bytes or error semantics", async () => { + for (const choices of [{}, [null], "invalid"]) { + const raw = JSON.stringify({ usage: { input_tokens: 1, output_tokens: 0 }, choices }); + const response = await observeNativeResponse(new Response(raw), { provider: "linkapi" }); + assert.equal(await response.text(), raw); + } +}); + +// Run unchanged existing tests with the same in-memory import registration when requested. +if (process.env.W18_EXISTING_REGRESSIONS === "true") { + for (const name of ["test_cloudflare_worker.mjs", "test_prometheus_worker.mjs", "test_opencode_reasoning_worker.mjs", + "test_opencode_thinking_worker.mjs", "test_sse_heartbeat_worker.mjs"]) { + const url = new URL(`./${name}`, import.meta.url); + const source = (await readFile(url, "utf8")).replaceAll("import.meta.url", JSON.stringify(url.href)) + .replace(/from\s+"(\.\.?\/[^"\n]+)"/g, (_, path) => `from "${path === "./helpers/load_cloudflare_worker.mjs" + ? registeredHelperUrl : new URL(path, url).href}"`); + await import(`data:text/javascript;base64,${Buffer.from(source).toString("base64")}`); + } +} diff --git a/worker/gateway-extensions.mjs b/worker/gateway-extensions.mjs new file mode 100644 index 00000000..0aa1500b --- /dev/null +++ b/worker/gateway-extensions.mjs @@ -0,0 +1,72 @@ +/** Static registration boundary for native dispatch; later integrations supply named hooks. */ +import { createGatewayLifecycle } from "./gateway-lifecycle.mjs"; +import { observeNativeResponse, appendNativeMetrics, PARSER_LIMIT } from "./request-telemetry.mjs"; +import { recordNativeUsage } from "./usage-ledger-d1.mjs"; + +let warned = false; +export function nativeMetricsEnabled(env) { + const value = String(env.NATIVE_EDGE_METRICS_ENABLED ?? "").trim().toLowerCase(); + if (!value || value === "false") return false; + if (value === "true") return true; + if (!warned) { warned = true; console.error(JSON.stringify({ event: "native_metrics_invalid_setting" })); } + return false; +} + +async function requestedModel(request) { + if (!request.body) return null; + const reader = request.clone().body.getReader(); let size = 0, parts = []; + try { + while (true) { + const result = await reader.read(); if (result.done) break; + size += result.value.byteLength; if (size > PARSER_LIMIT) return null; + parts.push(result.value); + } + const bytes = new Uint8Array(size); let offset = 0; + for (const part of parts) { bytes.set(part, offset); offset += part.length; } + const model = JSON.parse(new TextDecoder().decode(bytes)).model; + return typeof model === "string" && /^[A-Za-z0-9][A-Za-z0-9._:/+@-]{0,223}$/.test(model) ? model : null; + } catch { return null; } + finally { void reader.cancel().catch(() => {}); } +} + +export async function nativeGenerationFetch(request, env, ctx, authority, fetcher, collaborators = []) { + const path = new URL(request.url).pathname; + if (!nativeMetricsEnabled(env) || request.method !== "POST" + || !/\/(?:chat\/completions|responses|messages|embeddings|images\/generations)$/.test(path)) { + return fetcher(request, env, authority); + } + // This call site is after native route authentication. Ignore all client identity labels. + const startedAt = performance.now(); + const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(`native-admin:${authority.principal.id}`)); + const principal = `edge:${[...new Uint8Array(digest)].map(n => n.toString(16).padStart(2, "0")).join("")}`; + const context = Object.freeze({ provider: authority.provider, principal, model: await requestedModel(request), + requestId: crypto.randomUUID(), endpoint: authority.route, startedAt }); + let resolveDone; + const done = new Promise(resolve => { resolveDone = resolve; }); + ctx?.waitUntil?.(done); + const lifecycle = createGatewayLifecycle([...collaborators, { async finalize(event) { + try { await recordNativeUsage(env, event); } + finally { resolveDone(); } + } }]); + try { + await lifecycle.authorize(context); await lifecycle.admit(context); await lifecycle.before_dispatch(context); + const response = await fetcher(request, env, authority); + await lifecycle.observe(context); + return await observeNativeResponse(response, context, { env, signal: request.signal, finalize: event => lifecycle.finalize(event) }); + } catch (error) { + await observeNativeResponse(new Response(null, { status: request.signal.aborted ? 499 : 502 }), context, + { env, outcome: request.signal.aborted ? "canceled" : "transport_error", finalize: event => lifecycle.finalize(event) }); + throw error; + } +} + +export function withForwardedCorrelation(headers, env) { + if (!nativeMetricsEnabled(env)) return headers; + for (const key of ["x-multillm-principal", "x-multillm-request-id", "x-multillm-ledger-owner"]) headers.delete(key); + headers.set("x-request-id", crypto.randomUUID()); + return headers; +} + +export function withNativeMetrics(response, env) { + return nativeMetricsEnabled(env) ? appendNativeMetrics(response) : response; +} diff --git a/worker/gateway-lifecycle.mjs b/worker/gateway-lifecycle.mjs new file mode 100644 index 00000000..ff6d0b3a --- /dev/null +++ b/worker/gateway-lifecycle.mjs @@ -0,0 +1,23 @@ +/** Fixed request-scoped hooks. Registrations are code, never environment-provided plugins. */ +export const GATEWAY_HOOKS = Object.freeze(["authorize", "admit", "before_dispatch", "observe", "finalize"]); + +export function createGatewayLifecycle(collaborators = []) { + const registrations = Object.freeze([...collaborators]); + let finalization; + const invoke = async (name, context) => { + for (const collaborator of registrations) { + if (typeof collaborator[name] === "function") await collaborator[name](context); + } + }; + return Object.freeze(Object.fromEntries(GATEWAY_HOOKS.map(name => [name, context => { + if (name !== "finalize") return invoke(name, context); + // Assign before invoking any asynchronous collaborator, including concurrent callers. + return finalization ??= Promise.resolve().then(async () => { + let failure; + for (const collaborator of registrations) { + try { await collaborator.finalize?.(context); } catch (error) { failure ??= error; } + } + if (failure) throw failure; + }); + }]))); +} diff --git a/worker/request-telemetry.mjs b/worker/request-telemetry.mjs new file mode 100644 index 00000000..c1e8c9c5 --- /dev/null +++ b/worker/request-telemetry.mjs @@ -0,0 +1,198 @@ +/** Bounded, byte-preserving native response observation. No payload leaves this module. */ +import { escapeLabel, PROMETHEUS_CONTENT_TYPE } from "./prometheus.mjs"; + +export const PARSER_LIMIT = 64 * 1024; +const decoder = new TextDecoder(); +const providers = new Set(["codex-easy", "linkapi", "opencode", "kimi-code"]); +const outcomes = new Set(["success", "unknown", "upstream_error", "transport_error", "canceled"]); +const totals = new Map(); +const component = value => Number.isSafeInteger(value) && value >= 0 ? value : null; +const millis = value => Math.min(86_400_000, Math.max(0, Math.round(value))); + +export function createUsageObserver(sse = false, onToken = () => {}) { + let carry = new Uint8Array(), data = [], dataBytes = 0, oversized = false, eventOversized = false; + let usage = { input_tokens: null, output_tokens: null }, completed = false, failed = false; + const consume = text => { + if (text.trim() === "[DONE]") { completed = true; return; } + let value; + try { value = JSON.parse(text); } catch { return; } + if (!value || typeof value !== "object") return; + const observed = value.usage ?? value.response?.usage ?? value.message?.usage; + if (observed && typeof observed === "object") { + const input = component(observed.prompt_tokens ?? observed.input_tokens); + const output = component(observed.completion_tokens ?? observed.output_tokens); + if (input !== null) usage.input_tokens = input; + if (output !== null) usage.output_tokens = output; + } + const choices = Array.isArray(value.choices) ? value.choices : []; + if (value.error || value.type === "error" || value.type === "response.failed") failed = true; + if (value.type === "message_stop" || value.type === "response.completed" + || choices.some(choice => choice?.finish_reason != null)) completed = true; + if (choices.some(choice => choice?.delta?.content || choice?.delta?.reasoning_content || choice?.delta?.tool_calls?.length) + || value.delta?.text || value.delta?.thinking || value.type === "response.output_text.delta" + || value.type === "response.function_call_arguments.delta") onToken(); + }; + const line = () => { + const text = oversized ? "" : decoder.decode(carry).replace(/\r$/, ""); + carry = new Uint8Array(); + if (oversized) eventOversized = true; + oversized = false; + if (text === "") { + if (!eventOversized && data.length) consume(data.join("\n")); + data = []; dataBytes = 0; eventOversized = false; + } else if (text.startsWith("data:")) { + const part = text.slice(5).replace(/^ /, ""); + const size = new TextEncoder().encode(part).length + 1; + if (dataBytes + size > PARSER_LIMIT) { data = []; dataBytes = 0; eventOversized = true; } + else if (!eventOversized) { data.push(part); dataBytes += size; } + } + }; + const append = bytes => { + if (oversized) return; + if (carry.length + dataBytes + bytes.length > PARSER_LIMIT) { + carry = new Uint8Array(); oversized = true; return; + } + const next = new Uint8Array(carry.length + bytes.length); + next.set(carry); next.set(bytes, carry.length); carry = next; + }; + return { + get carryBytes() { return carry.length + dataBytes; }, + feed(bytes) { + if (!sse) { append(bytes); return; } + let start = 0, end; + while ((end = bytes.indexOf(10, start)) !== -1) { + append(bytes.subarray(start, end)); line(); start = end + 1; + } + append(bytes.subarray(start)); + }, + finish() { + if (sse) { if (carry.length || oversized) line(); if (!eventOversized && data.length) consume(data.join("\n")); } + else if (!oversized) { consume(decoder.decode(carry)); completed = true; } + carry = new Uint8Array(); data = []; dataBytes = 0; + return { usage: { ...usage }, completed, failed }; + }, + }; +} + +function priceUsage(env, provider, model, usage, known) { + if (!known || !model) return null; + let table; + const raw = env.MODEL_PRICING_USD_PER_MILLION; + if (typeof raw !== "string" || raw.length > PARSER_LIMIT) return null; + try { table = JSON.parse(raw); } catch { return null; } + const name = (model.includes(":") ? model : `${provider}:${model}`).toLowerCase(); + const price = table?.[name] ?? table?.[`${provider}:*`] ?? table?.["*"]; + if (!price || typeof price !== "object") return null; + const flatOnly = Object.hasOwn(price, "request") && !["input", "output", "input_cost_per_million", "output_cost_per_million"].some(k => Object.hasOwn(price, k)); + const rate = value => typeof value === "number" || typeof value === "string" && value.trim() + ? Number(value) : NaN; + const input = flatOnly ? 0 : rate(price.input ?? price.input_cost_per_million); + const output = flatOnly ? 0 : rate(price.output ?? price.output_cost_per_million); + const request = Object.hasOwn(price, "request") ? rate(price.request) : 0; + if (![input, output, request].every(n => Number.isFinite(n) && n >= 0)) return null; + const cost = (usage.input_tokens * input + usage.output_tokens * output) / 1_000_000 + request; + return Number.isFinite(cost) && cost <= 1_000_000 ? cost : null; +} + +function countEvent(event) { + if (!providers.has(event.provider) || !outcomes.has(event.outcome)) return; + const key = `${event.provider}:${event.outcome}`; + const item = totals.get(key) ?? { provider: event.provider, outcome: event.outcome, requests: 0, + duration: 0, ttft: 0, firstTokens: 0, measured: 0, priced: 0 }; + item.requests++; item.duration += event.duration_ms / 1000; + if (event.ttft_ms !== null) { item.ttft += event.ttft_ms / 1000; item.firstTokens++; } + if (event.usage_basis === "measured") item.measured++; + if (event.cost_usd !== null) item.priced++; + totals.set(key, item); +} + +export function renderNativeMetrics() { + const families = [ + ["requests_total", "Finalized native attempts in this isolate.", "requests"], + ["duration_seconds_total", "Total native attempt duration in this isolate.", "duration"], + ["ttft_seconds_total", "Total observed first-token time in this isolate.", "ttft"], + ["first_token_observations_total", "Attempts with a first-token observation.", "firstTokens"], + ["measured_usage_total", "Attempts with complete measured token counts.", "measured"], + ["priced_requests_total", "Attempts with known configured cost.", "priced"], + ]; + return families.map(([suffix, help, field]) => { + const name = `multillm_native_${suffix}`; + return `# HELP ${name} ${help}\n# TYPE ${name} counter\n` + [...totals.values()] + .map(item => `${name}{provider="${escapeLabel(item.provider)}",outcome="${item.outcome}"} ${item[field]}\n`).join(""); + }).join(""); +} + +export async function appendNativeMetrics(response) { + if (response.status !== 200 || !response.headers.get("content-type")?.includes("text/plain")) return response; + const suffix = new TextEncoder().encode(`\n${renderNativeMetrics()}`); + const reader = response.body?.getReader(); let appended = false; + const body = new ReadableStream({ + async pull(controller) { + if (reader) { + const result = await reader.read(); + if (!result.done) { controller.enqueue(result.value); return; } + } + if (!appended) { appended = true; controller.enqueue(suffix); } + controller.close(); + }, cancel(reason) { return reader?.cancel(reason); }, + }, { highWaterMark: 0 }); + const headers = new Headers(response.headers); headers.delete("content-length"); + headers.set("content-type", PROMETHEUS_CONTENT_TYPE); + return new Response(body, { status: response.status, statusText: response.statusText, headers }); +} + +function finalEvent(response, context, options, parsed, reason, duration, ttft) { + const usage = { ...parsed.usage }; + const hasProviderUsage = usage.input_tokens !== null || usage.output_tokens !== null; + let estimated = false; + for (const key of ["input_tokens", "output_tokens"]) { + const estimate = component(options.usageEstimate?.[key]); + if (usage[key] === null && estimate !== null) { usage[key] = estimate; estimated = true; } + } + const known = usage.input_tokens !== null && usage.output_tokens !== null; + const outcome = reason ?? options.outcome ?? (response.status >= 400 || parsed.failed ? "upstream_error" + : parsed.completed && known && hasProviderUsage ? "success" : "unknown"); + const basis = outcome === "success" && known ? estimated ? "estimated" : "measured" : "unknown"; + const cost = priceUsage(options.env ?? {}, context.provider, context.model, usage, known && outcome === "success"); + const status = outcome === "canceled" ? 499 : outcome === "transport_error" ? 502 + : outcome === "unknown" && response.status < 400 ? 520 + : outcome === "upstream_error" && response.status < 400 ? 502 : response.status; + return { ...context, status, outcome, duration_ms: duration, ttft_ms: ttft, ...usage, + usage_basis: basis, cost_usd: cost, cost_basis: cost === null ? null : estimated ? "estimate" : "usage" }; +} + +export async function observeNativeResponse(response, context, options = {}) { + const clock = options.clock ?? (() => performance.now()); + let ttft = null, finalization; + const started = context.startedAt ?? clock(); + const sse = response.headers.get("content-type")?.includes("text/event-stream") ?? false; + const observer = createUsageObserver(sse, () => { ttft ??= millis(clock() - started); }); + const finish = reason => finalization ??= Promise.resolve().then(async () => { + options.signal?.removeEventListener("abort", abort); + const parsed = observer.finish(); + const event = finalEvent(response, context, options, parsed, reason, millis(clock() - started), ttft); + countEvent(event); + try { await options.finalize?.(event); } catch { console.error(JSON.stringify({ event: "native_metrics_finalize_failed" })); } + }); + const reader = response.body?.getReader(); let controllerRef; + const abort = () => { + void finish("canceled"); + void reader?.cancel().catch(() => {}); + controllerRef?.error(new DOMException("The operation was aborted", "AbortError")); + }; + if (!reader) { await finish(); return response; } + const body = new ReadableStream({ + start(controller) { controllerRef = controller; }, + async pull(controller) { + try { + const result = await reader.read(); + if (result.done) { await finish(); controller.close(); return; } + observer.feed(result.value); controller.enqueue(result.value); + } catch (error) { await finish(options.signal?.aborted ? "canceled" : "transport_error"); controller.error(error); } + }, + async cancel(reason) { await finish("canceled"); await reader.cancel(reason); }, + }, { highWaterMark: 0 }); + options.signal?.addEventListener("abort", abort, { once: true }); + if (options.signal?.aborted) abort(); + return new Response(body, { status: response.status, statusText: response.statusText, headers: response.headers }); +} diff --git a/worker/usage-ledger-d1.mjs b/worker/usage-ledger-d1.mjs index edfae505..88ea78d1 100644 --- a/worker/usage-ledger-d1.mjs +++ b/worker/usage-ledger-d1.mjs @@ -193,3 +193,21 @@ export async function handleUsageLedgerRequest(request, env) { return reply({ error: "storage_unavailable" }, 503); } } + +/** Serialize native telemetry into the existing strict ledger; richer fields need a migration. */ +export async function recordNativeUsage(env, event) { + const model = event.model ? `${event.provider}:${event.model}` : null; + const kind = event.endpoint.endsWith("/responses") ? "responses" : event.endpoint.endsWith("/embeddings") ? "embeddings" + : event.endpoint.endsWith("/images/generations") ? "images" : "chat"; + const row = { at: new Date().toISOString(), principal: event.principal, key_prefix: null, + kind, endpoint: event.endpoint, requested_model: model, selected_model: model, + status: event.status, latency_ms: event.duration_ms, input_tokens: event.input_tokens, + output_tokens: event.output_tokens, cost_usd: event.cost_usd, cost_basis: event.cost_basis, + request_id: event.requestId }; + if (!validRow(row)) throw new Error("Invalid native usage metadata"); + if (!env.INTELLIGENCE_DB) { console.error(JSON.stringify({ event: "native_usage_storage_unavailable" })); return; } + try { + return await record(env.INTELLIGENCE_DB, env, { version: 1, operation: "record", + batch: event.requestId.replaceAll("-", ""), rows: [row] }); + } catch { console.error(JSON.stringify({ event: "native_usage_storage_failed" })); } +}