Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 18 additions & 13 deletions cloudflare-worker.mjs
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { nativeGenerationFetch, withForwardedCorrelation, withNativeMetrics } from "./worker/gateway-extensions.mjs";
import { firewallFetch } from "./worker/secret-firewall.mjs";
import { Container, ContainerProxy, getContainer } from "@cloudflare/containers";
import { collectContainerEnv } from "./worker/container-env.mjs";
Expand Down Expand Up @@ -1446,7 +1447,7 @@ function createOpencodeStreamResponse(upstreamResponse) {
});
}

async function handleDirectOpencodeRequest(request, env, requestUrl) {
async function handleDirectOpencodeRequest(request, env, requestUrl, ctx) {
const providedToken = extractOpencodeCallerToken(request);
if (!(await timingSafeTokenMatch(providedToken, env.ADMIN_API_KEY))) {
return applyCorsHeaders(request, buildUnauthorizedResponse(), env);
Expand Down Expand Up @@ -1492,10 +1493,10 @@ async function handleDirectOpencodeRequest(request, env, requestUrl) {
signal: request.signal,
...(bodyAllowed && request.body ? { duplex: "half" } : {}),
});
const upstreamResponse = await firewallFetch(upstreamRequest, env, {
const upstreamResponse = await nativeGenerationFetch(upstreamRequest, env, ctx, {
route: requestUrl.pathname, principal: { id: env.ADMIN_USERNAME || "admin" },
provider: requestUrl.pathname.split("/")[1],
});
}, async (upstreamRequest, env, authority) => await firewallFetch(upstreamRequest, env, authority));

if (isOpencodeNativeRequest(requestUrl.pathname, request.method)) {
const downstreamResponse = new Response(upstreamResponse.body, {
Expand Down Expand Up @@ -1580,7 +1581,7 @@ async function authorizeContainerOpencodeRequest(request, env, requestUrl) {
return null;
}

async function handleDirectLinkApiRequest(request, env, requestUrl) {
async function handleDirectLinkApiRequest(request, env, requestUrl, ctx) {
const providedToken = extractLinkApiCallerToken(request, requestUrl);
if (!(await timingSafeTokenMatch(providedToken, env.ADMIN_API_KEY))) {
return applyCorsHeaders(request, buildLinkApiUnauthorizedResponse(), env);
Expand Down Expand Up @@ -1627,10 +1628,10 @@ async function handleDirectLinkApiRequest(request, env, requestUrl) {
signal: normalizedRequest.signal,
...(bodyAllowed && normalizedRequest.body ? { duplex: "half" } : {}),
});
const upstreamResponse = await firewallFetch(upstreamRequest, env, {
const upstreamResponse = await nativeGenerationFetch(upstreamRequest, env, ctx, {
route: requestUrl.pathname, principal: { id: env.ADMIN_USERNAME || "admin" },
provider: requestUrl.pathname.split("/")[1],
});
}, async (upstreamRequest, env, authority) => await firewallFetch(upstreamRequest, env, authority));

return applyCorsHeaders(
request,
Expand All @@ -1643,7 +1644,7 @@ async function handleDirectLinkApiRequest(request, env, requestUrl) {
);
}

async function handleDirectCodexEasyRequest(request, env, requestUrl) {
async function handleDirectCodexEasyRequest(request, env, requestUrl, ctx) {
const providedToken = extractBearerToken(request);
if (!(await timingSafeTokenMatch(providedToken, env.ADMIN_API_KEY))) {
return applyCorsHeaders(request, buildUnauthorizedResponse(), env);
Expand Down Expand Up @@ -1685,10 +1686,10 @@ async function handleDirectCodexEasyRequest(request, env, requestUrl) {
signal: normalizedRequest.signal,
...(bodyAllowed && normalizedRequest.body ? { duplex: "half" } : {}),
});
const upstreamResponse = await firewallFetch(upstreamRequest, env, {
const upstreamResponse = await nativeGenerationFetch(upstreamRequest, env, ctx, {
route: requestUrl.pathname, principal: { id: env.ADMIN_USERNAME || "admin" },
provider: requestUrl.pathname.split("/")[1],
});
}, async (upstreamRequest, env, authority) => await firewallFetch(upstreamRequest, env, authority));

return applyCorsHeaders(
request,
Expand Down Expand Up @@ -1982,7 +1983,7 @@ export default {

if (codexEasyPath) {
try {
return await handleDirectCodexEasyRequest(request, env, requestUrl);
return await handleDirectCodexEasyRequest(request, env, requestUrl, ctx);
} catch (error) {
logStructuredError("direct_codex_easy_fetch_failed", error);
return applyCorsHeaders(
Expand All @@ -2001,7 +2002,7 @@ export default {

if (linkapiPath) {
try {
return await handleDirectLinkApiRequest(request, env, requestUrl);
return await handleDirectLinkApiRequest(request, env, requestUrl, ctx);
} catch (error) {
logStructuredError("direct_linkapi_fetch_failed", error);
return applyCorsHeaders(
Expand All @@ -2021,7 +2022,7 @@ export default {
if (opencodePath) {
try {
if (env.OPENCODE_EDGE_FETCH === "true") {
return await handleDirectOpencodeRequest(request, env, requestUrl);
return await handleDirectOpencodeRequest(request, env, requestUrl, ctx);
}
const rejected = await authorizeContainerOpencodeRequest(
request,
Expand Down Expand Up @@ -2059,6 +2060,7 @@ export default {
await withAccessIdentity(request, env, headers);
headers.delete("content-length");
headers.delete("host");
withForwardedCorrelation(headers, env);
headers.set("x-forwarded-proto", requestUrl.protocol.slice(0, -1));
headers.set("x-forwarded-host", requestUrl.host);
headers.set("x-multillm-external-origin", requestUrl.origin);
Expand All @@ -2074,7 +2076,10 @@ export default {
signal: request.signal,
...(bodyAllowed && request.body ? { duplex: "half" } : {}),
});
const response = await container.fetch(forwardedRequest);
let response = await container.fetch(forwardedRequest);
if (requestUrl.pathname === "/v1/metrics/prometheus") {
response = await withNativeMetrics(response, env);
}
if (await isContainerPackageStartupFailure(response)) {
logStructuredError("container_start_failed_response", new Error("Container unavailable"));
if (rootPath) {
Expand Down
100 changes: 100 additions & 0 deletions docs/edge-request-metrics.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
# Native edge request metrics

`NATIVE_EDGE_METRICS_ENABLED` defaults to false. Unset, empty and `false` values preserve
existing native responses, headers, streams, ledger writes and logs. No lifecycle hook
runs. Set the Worker variable to `true` to enable observation; values are trimmed and
case-insensitive. A malformed setting disables the feature and emits one content-free
warning per isolate. For example:

```json
{"NATIVE_EDGE_METRICS_ENABLED": "true", "PROMETHEUS_ENABLED": "true"}
```

To disable it:

```json
{"NATIVE_EDGE_METRICS_ENABLED": "false"}
```

## Scope and identity

After existing route authentication, native Codex Easy, LinkAPI and edge OpenCode
POST generations are observed. Kimi generations and non-edge OpenCode currently go
through the Container and retain Flask ledger ownership. Models, health, token-count,
Knowledge and roleplay paths retain their existing accounting. No retry is introduced.

Each native attempt gets a generated request ID and a SHA-256 digest of the verified
native admin identity with an `edge:` namespace. Client identity and request-ID headers
cannot supply these values. This identity is separate from the Flask username principal;
reports and budgets must not assume the two namespaces are combined. Selected models
come from the authenticated request, never an upstream response's model field. The
native admin route currently has no narrower per-model grant. A request model that is
missing, malformed or outside the parse bound remains unknown.

Enabled Container forwarding strips client copies of edge principal, request ID and
ledger-owner headers and generates `x-request-id`. Flask still authenticates the caller
and owns its usage row. Edge forwarding never writes a second row.

## Usage, failures and cost

The observer accepts OpenAI Chat/Responses and Anthropic usage envelopes, retaining
input and output counts independently. Zero is measured zero; missing, malformed or
negative components remain null. Split SSE events merge the latest valid component.
Complete measured usage on a completed successful response has basis `measured`;
partial, usage-less, canceled, failed and incomplete attempts have basis `unknown`.
No token estimates are invented. The observer accepts a code-supplied `usageEstimate`
collaborator value, fills only missing components and marks those observations
`estimated`. Measured zero is preserved. Usage-less and canceled attempts stay unknown
even with estimates. The registered native routes supply no estimates by default.

Costs use `MODEL_PRICING_USD_PER_MILLION` with explicit input/output prices, optional
per-request prices, provider wildcards and the global wildcard. Costs remain null without
complete successful usage and valid prices. They are configured estimates, not invoices.
Known configured costs map to the existing D1 `usage` basis; unknown costs map to null.
Prompt-cache usage buckets are not priced separately.

Native downstream HTTP status, headers and bytes are unchanged. Ledger-only status 499
means cancellation, 502 means a transport or in-stream error, and 520 means incomplete
or unknown usage on an otherwise successful HTTP response. Upstream HTTP errors keep
their status. These classifications prevent unknown attempts from becoming successes
in daily rollups. Partial counts remain useful without treating missing cost as free.

## Operational bounds and retention

Response observation uses one pull-driven reader with no response clone, tee or
full-body queue. SSE line/event carry and non-stream JSON observation are bounded at
64 KiB. Oversized events are discarded for accounting and bytes are still forwarded.
Parsing recovers at subsequent event boundaries. JSON responses above the bound have
unknown usage. Authenticated request-model extraction also reads at most 64 KiB of a
request clone; its reader is canceled afterwards. First-token latency is recorded only
for recognizable streaming content, reasoning or tool deltas; it stays unknown for
non-stream responses. Duration uses a monotonic clock, includes dispatch, and is clamped
to the ledger's 24-hour bound. A terminal SSE marker and complete usage are required
for successful stream accounting.

Cancellation and body/fetch failures finalize once. Finalization uses `ctx.waitUntil`
when available; missing/broken D1 emits fixed content-free errors and never replaces
the native result. This is best-effort telemetry, not a billing guarantee. No prompt,
tool arguments, output, raw key, exception detail or key prefix is persisted or logged
by this feature. The existing `usage_events`, `usage_daily` and `usage_batches` schema
and pruning policy apply; there is no new retention store or migration. Request IDs
and model identity are durable; TTFT and richer outcomes are not additional D1 columns.

With both flags enabled, the existing authenticated `/v1/metrics/prometheus` endpoint
appends native counters using the existing exporter's content type and label escaping. Authentication
errors and non-Prometheus responses pass through unchanged. The public status endpoint
is unchanged. Labels contain only four fixed provider names and five fixed outcomes:
at most 120 series, with no principal, request ID, model or content labels. Counters
track requests, duration, observed TTFT and its coverage, measured usage coverage and
priced coverage. Counters are isolate-local and reset when the isolate restarts; a
scrape does not aggregate other isolates. D1 remains the durable count source.

## Static lifecycle integration

`createGatewayLifecycle` exposes `authorize`, `admit`, `before_dispatch`, `observe` and
`finalize` in registration order with one finalizer per request. Hook context contains
only content-free metadata. Native dispatch accepts code-supplied named collaborators;
there is no dynamic import, executable configuration or discovery. Finalization tries
every collaborator even if one fails, then reports the failure without changing response
bytes. Other gateway features attach to native dispatch as code-registered
collaborators on these hooks.
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
"lint": "ruff check . --select E9,F63,F7,F82",
"typecheck": "mypy --ignore-missing-imports --follow-imports=skip --disable-error-code import-untyped --disable-error-code var-annotated --disable-error-code arg-type --disable-error-code assignment --disable-error-code operator app.py routes services providers streaming",
"test": "python -m pytest -q",
"test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs",
"test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_edge_request_metrics_worker.mjs",
"db:migrate": "python scripts/validate_sqlite_schema.py --apply",
"db:validate": "python scripts/validate_sqlite_schema.py",
"security:secrets": "python scripts/check_static_secrets.py",
Expand Down
3 changes: 3 additions & 0 deletions tests/fixtures/secret_firewall_coverage.json
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,8 @@
"worker/cors-policy.mjs",
"worker/d1-schema.mjs",
"worker/fallback-page.mjs",
"worker/gateway-extensions.mjs",
"worker/gateway-lifecycle.mjs",
"worker/gpt-image-moderation.mjs",
"worker/health-schedule.mjs",
"worker/intelligence-auth-d1.mjs",
Expand Down Expand Up @@ -293,6 +295,7 @@
"worker/opencode/reasoning-response.mjs",
"worker/prometheus.mjs",
"worker/prompt-templates-d1.mjs",
"worker/request-telemetry.mjs",
"worker/roleplay/candidate-context.mjs",
"worker/roleplay/capacity.mjs",
"worker/roleplay/checkpoint.mjs",
Expand Down
1 change: 1 addition & 0 deletions tests/helpers/load_cloudflare_worker.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -739,6 +739,7 @@ export async function loadWorkerModule() {
.replace('from "./worker/health-schedule.mjs";', `from "${new URL("../../worker/health-schedule.mjs", import.meta.url)}";`)
.replace('from "./worker/status-page.mjs";', `from "${new URL("../../worker/status-page.mjs", import.meta.url)}";`)
.replace('from "./worker/intelligence-outbound.mjs";', `from "${new URL("../../worker/intelligence-outbound.mjs", import.meta.url)}";`)
.replace('from "./worker/gateway-extensions.mjs";', `from "${new URL("../../worker/gateway-extensions.mjs", import.meta.url)}";`)
.replace('from "./worker/client-headers.mjs";', `from "${new URL("../../worker/client-headers.mjs", import.meta.url)}";`)
.replace('from "./worker/opencode-session.mjs";', `from "${new URL("../../worker/opencode-session.mjs", import.meta.url)}";`)
.replace('from "./worker/opencode/reasoning-request.mjs";', `from "${new URL("../../worker/opencode/reasoning-request.mjs", import.meta.url)}";`)
Expand Down
Loading
Loading