From 585f5b70638a995c8bf700a8fe293e3201e24766 Mon Sep 17 00:00:00 2001 From: Abhyudaya Baiju Tharakan Date: Fri, 9 Oct 2026 08:09:29 +0530 Subject: [PATCH] feat(batches): add an opt-in gateway Batch API with explicit spillover OpenAI-compatible /v1/files and /v1/batches backed by D1 and R2, executed item by item through the managed chat route by the scheduled handler. Explicit spillover turns a chat request with X-MultiLLM-Priority: batch and Prefer: respond-async into a one-item batch. Both flags default to off. --- cloudflare-worker.mjs | 4 + docs/gateway-batches.md | 52 +++ .../0027_gateway_batches.sql | 29 ++ package.json | 2 +- route_helpers.py | 4 +- routes/gateway_batches.py | 268 +++++++++++++ services/gateway_batches.py | 202 ++++++++++ tests/fixtures/secret_firewall_coverage.json | 11 + tests/helpers/load_cloudflare_worker.mjs | 1 + tests/test_gateway_batches.py | 353 ++++++++++++++++++ tests/test_gateway_batches_worker.mjs | 309 +++++++++++++++ worker/batch-files.mjs | 124 ++++++ worker/batch-jobs.mjs | 315 ++++++++++++++++ worker/container-env.mjs | 2 + worker/cors-policy.mjs | 4 +- worker/d1-schema.mjs | 2 +- worker/intelligence-outbound.mjs | 2 + 17 files changed, 1678 insertions(+), 6 deletions(-) create mode 100644 docs/gateway-batches.md create mode 100644 intelligence-migrations/0027_gateway_batches.sql create mode 100644 routes/gateway_batches.py create mode 100644 services/gateway_batches.py create mode 100644 tests/test_gateway_batches.py create mode 100644 tests/test_gateway_batches_worker.mjs create mode 100644 worker/batch-files.mjs create mode 100644 worker/batch-jobs.mjs diff --git a/cloudflare-worker.mjs b/cloudflare-worker.mjs index 7f9d70a8..0576d094 100644 --- a/cloudflare-worker.mjs +++ b/cloudflare-worker.mjs @@ -1,4 +1,5 @@ import { nativeGenerationFetch, withForwardedCorrelation, withNativeMetrics, nativeCacheHeader, nativeGenerationSetup, generationErrorResponse, forwardedGenerationHeaders, runScheduledMaintenance, scheduledMaintenanceEnabled } from "./worker/gateway-extensions.mjs"; +import { batchesEnabled, runScheduledBatches } from "./worker/batch-jobs.mjs"; import { tickNativeRevisionSync } from "./worker/native-config-sync.mjs"; import { firewallFetch } from "./worker/secret-firewall.mjs"; import { Container, ContainerProxy, getContainer } from "@cloudflare/containers"; @@ -1837,6 +1838,9 @@ export default { tickNativeRevisionSync(env, ctx); if (scheduledMaintenanceEnabled(env)) ctx.waitUntil(runScheduledMaintenance(env)); const container = getContainer(env.MULTILLM_PROXY_CONTAINER, "primary"); + if (batchesEnabled(env)) ctx.waitUntil(runScheduledBatches(env, container).catch(() => { + console.warn(JSON.stringify({ event: "gateway_batch_schedule_failed" })); + })); ctx.waitUntil(runScheduledShadowEval(env, container)); ctx.waitUntil(runScheduledHealth(controller, env, container).catch((error) => { logStructuredError("scheduled_health_failed", error); diff --git a/docs/gateway-batches.md b/docs/gateway-batches.md new file mode 100644 index 00000000..b9668a7f --- /dev/null +++ b/docs/gateway-batches.md @@ -0,0 +1,52 @@ +# Gateway batches + +The Batch API stores owned JSONL files and executes each item through the gateway's managed Chat Completions or Responses route. Both `GATEWAY_BATCHES_ENABLED` and `BATCH_SPILLOVER_ENABLED` default to `false`. Empty settings are off; malformed settings are off with one warning that omits the value. Disabled batch routes return 404 and normal generations continue unchanged. + +Apply the gateway batch D1 migration before enabling `GATEWAY_BATCHES_ENABLED` in the Worker and Container. Storage uses `INTELLIGENCE_DB` and the existing `multillm_media` R2 binding. No extra binding or cron schedule is required. An enabled endpoint with missing tables or storage returns JSON 503 `gateway_batches_unavailable` before provider submission. + +## Upload and create + +Each UTF-8 JSONL line contains a unique `custom_id`, `method: "POST"`, a supported `url`, and a nonstreaming request `body` with a model. A file can contain at most 1,000 items and 10 MiB. Invalid input returns 400 `invalid_file` with the first bad line number. Other file purposes return 400. Requests must use a concrete priced model or a route whose every candidate has a known price; custom routing definitions are rejected. + +```json +{"custom_id":"question-1","method":"POST","url":"/v1/chat/completions","body":{"model":"mimo:mimo-v2.5","messages":[{"role":"user","content":"Explain tides briefly."}],"max_tokens":128}} +``` + +Upload with an existing gateway credential in your client environment: + +```sh +curl "$GATEWAY_URL/v1/files" -H "Authorization: Bearer $GATEWAY_KEY" \ + -F purpose=batch -F file=@input.jsonl +``` + +Create a batch using the returned file ID: + +```json +{"input_file_id":"file_RETURNED_ID","endpoint":"/v1/chat/completions","completion_window":"24h","metadata":{"multillm_budget_usd":"0.50"}} +``` + +Send this JSON to `POST /v1/batches`. All items must match `endpoint`, which accepts `/v1/chat/completions` or `/v1/responses`. The budget is required: a positive decimal string of at most 100000 USD with at most ten fractional digits. Metadata accepts at most 16 string values, with keys up to 64 and values up to 512 characters. Unknown prices fail creation with 400 `unknown_price`. Model permissions apply at creation and execution. + +Explicit output limits must be integers from 1 to 262144 tokens; completion count `n`, when present, is from 1 to 16. Admission uses input JSON byte count and the largest output bound, multiplied by `n`, rounded upwards in units of 0.0000000001 USD. Without an output limit, admission conservatively reserves 262144 output tokens. Set an explicit output limit to keep the reservation useful. This is a gateway spend bound, not a provider batch discount or a guarantee of provider billing accuracy. + +## Read, cancel and delete + +`GET /v1/files` and `GET /v1/batches` return list objects with `data`, `first_id`, `last_id`, and `has_more`. They accept `limit` from 1 to 100 and an `after` ID. File metadata is available at `GET /v1/files/`; bytes at `GET /v1/files//content`; deletion at `DELETE /v1/files/`. Every operation is restricted to the authenticated owner, including administrators. Another owner's ID returns 404. An input file used by an active batch cannot be deleted (409 `file_in_use`). Existing media routes remain unchanged. + +`GET /v1/batches/` returns the batch object, timestamps, request counts, and output/error file IDs. `POST /v1/batches//cancel` stops new work; already dispatched items finish and remain recorded. Cancellation moves through `cancelling` to `cancelled`. Normal execution moves through `validating`, `in_progress`, `finalizing`, and `completed`. Expiry stops unfinished work 24 hours after creation and ends as `expired`. Individual failures appear in the error file; a completed batch can contain failed items. + +Output and error files contain JSONL records keyed by `custom_id`, with `response` and `error` fields. They use the same owned file APIs. Individual results are limited to 1 MiB; aggregate output and error content are bounded at 16 MiB each. Oversized results become errors and are never retried. API files remain stored until their owner deletes them; the 24-hour completion window is not file retention expiry. Intermediate result objects and assembly checkpoints persist independently under `batches/`; there is no automatic TTL or batch deletion endpoint. + +## Execution and recovery + +The existing Worker scheduler claims at most two items per invocation using an atomic D1 lease and spend hold. Each item has a 30-second deadline; the invocation has a 65-second work budget. Remaining items continue on the next invocation. Completion time depends on the configured cron frequency and the two-items-per-run limit. A single shard processing 1,000 items needs at least 500 invocations. + +The scheduler sends the Container an expiring, signed owner capability and an item lease. It does not store or replay the owner's API credential. The Container consumes the durable lease once, loads canonical content from batch storage, reloads the owner's current account, and checks key rotation, expiry, original client-IP restrictions, scopes, model permissions, rate limits, budget, and retention. It invokes the real managed route with its dispatch, conversion, validation, cache, accounting and cancellation machinery. A missing or revoked account cannot execute. The private item route rejects unsigned calls. + +Measured spend replaces each known hold. Missing usage, an uncertain provider result, lost transport, or an expired dispatched lease produces `outcome_unknown` and preserves the hold. That item is never dispatched again. Result-storage failures also retain the hold conservatively. Once measured plus held spend reaches the batch budget, further items fail `budget_exhausted`; an item that would exceed the remaining budget is also refused before dispatch. A pricing increase beyond its reserved estimate fails before dispatch. Finalization uses durable checkpoints so output assembly can continue after a scheduler interruption. + +Zero-content retention rejects upload, batch creation and spillover with 400 `retention_conflict`. Current retention is checked again before executing stored work. Content lives only under the R2 `batches/` prefix; D1 stores ownership, opaque signing capability, control identifiers, status, spend, leases and content pointers. Scheduler warnings omit prompts, completions, credentials and transport details. + +## Explicit asynchronous submission + +With both flags enabled, a managed `POST /v1/chat/completions` containing both `X-MultiLLM-Priority: batch` and `Prefer: respond-async`, plus `metadata.multillm_budget_usd`, creates a real one-item batch and returns HTTP 202. Its `Location` header points to `/v1/batches/`. Without both headers, or with spillover disabled, the request follows the existing synchronous path. Submission is explicit; overload does not queue a request automatically. diff --git a/intelligence-migrations/0027_gateway_batches.sql b/intelligence-migrations/0027_gateway_batches.sql new file mode 100644 index 00000000..181c1402 --- /dev/null +++ b/intelligence-migrations/0027_gateway_batches.sql @@ -0,0 +1,29 @@ +CREATE TABLE IF NOT EXISTS gateway_batch_files ( + id TEXT PRIMARY KEY, owner TEXT NOT NULL, filename TEXT NOT NULL, + purpose TEXT NOT NULL CHECK (purpose IN ('batch', 'batch_output')), + bytes INTEGER NOT NULL, r2_key TEXT NOT NULL, created_at INTEGER NOT NULL +); +CREATE INDEX IF NOT EXISTS gateway_batch_files_owner ON gateway_batch_files(owner, created_at, id); +CREATE TABLE IF NOT EXISTS gateway_batches ( + id TEXT PRIMARY KEY, owner TEXT NOT NULL, input_file_id TEXT NOT NULL, + endpoint TEXT NOT NULL, status TEXT NOT NULL, metadata TEXT NOT NULL, + principal TEXT NOT NULL, client_ip TEXT NOT NULL, key_hash TEXT NOT NULL, key_prefix TEXT NOT NULL, + budget_units INTEGER NOT NULL CHECK (budget_units > 0), created_at INTEGER NOT NULL, expires_at INTEGER NOT NULL, + in_progress_at INTEGER, finalizing_at INTEGER, completed_at INTEGER, + cancelling_at INTEGER, cancelled_at INTEGER, expired_at INTEGER, failed_at INTEGER, + output_file_id TEXT, error_file_id TEXT, terminal_status TEXT, + finalize_cursor INTEGER NOT NULL DEFAULT 0, checkpoint TEXT, + lease_token TEXT, lease_until INTEGER +); +CREATE INDEX IF NOT EXISTS gateway_batches_owner ON gateway_batches(owner, created_at, id); +CREATE INDEX IF NOT EXISTS gateway_batches_active ON gateway_batches(status, expires_at); +CREATE TABLE IF NOT EXISTS gateway_batch_items ( + batch_id TEXT NOT NULL REFERENCES gateway_batches(id), idx INTEGER NOT NULL, + owner TEXT NOT NULL, custom_id TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'queued', + estimate_units INTEGER NOT NULL CHECK (estimate_units >= 0), + held_units INTEGER NOT NULL DEFAULT 0 CHECK (held_units >= 0), + cost_units INTEGER NOT NULL DEFAULT 0 CHECK (cost_units >= 0), + result_key TEXT, error_code TEXT, lease_token TEXT, lease_until INTEGER, + PRIMARY KEY (batch_id, idx), UNIQUE (batch_id, custom_id) +); +CREATE INDEX IF NOT EXISTS gateway_batch_items_claim ON gateway_batch_items(status, batch_id, idx); diff --git a/package.json b/package.json index 4dc35c96..042471f0 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "lint": "ruff check . --select E9,F63,F7,F82", "typecheck": "mypy --ignore-missing-imports --follow-imports=skip --disable-error-code import-untyped --disable-error-code var-annotated --disable-error-code arg-type --disable-error-code assignment --disable-error-code operator app.py routes services providers streaming", "test": "python -m pytest -q", - "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_managed_idempotency_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_gateway_alerts_worker.mjs tests/test_session_tiers_worker.mjs tests/test_usage_reservations_worker.mjs tests/test_shared_generation_cache_worker.mjs tests/test_deferred_tools_worker.mjs tests/test_generation_deadline_worker.mjs tests/test_wave4_worker.mjs", + "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_managed_idempotency_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_gateway_alerts_worker.mjs tests/test_session_tiers_worker.mjs tests/test_usage_reservations_worker.mjs tests/test_shared_generation_cache_worker.mjs tests/test_deferred_tools_worker.mjs tests/test_generation_deadline_worker.mjs tests/test_wave4_worker.mjs tests/test_gateway_batches_worker.mjs", "db:migrate": "python scripts/validate_sqlite_schema.py --apply", "db:validate": "python scripts/validate_sqlite_schema.py", "security:secrets": "python scripts/check_static_secrets.py", diff --git a/route_helpers.py b/route_helpers.py index b5f9fb02..ff0f3ce0 100644 --- a/route_helpers.py +++ b/route_helpers.py @@ -29,7 +29,7 @@ "Authorization, Content-Type, Accept, Origin, X-Requested-With, X-MultiLLM-Tool-Repair, X-MultiLLM-Conversion-Report, X-MultiLLM-Retention, X-MultiLLM-Deadline-Ms, " "X-Api-Key, X-Goog-Api-Key, X-MultiLLM-Api-Key, Anthropic-Version, " "Anthropic-Beta, Anthropic-Dangerous-Direct-Browser-Access, " - "Idempotency-Key, X-MultiLLM-Image-QA, X-MultiLLM-Cascade, OpenAI-Beta, OpenAI-Organization, " + "Idempotency-Key, X-MultiLLM-Priority, Prefer, X-MultiLLM-Image-QA, X-MultiLLM-Cascade, OpenAI-Beta, OpenAI-Organization, " "OpenAI-Project, Moderation, Moderation-Model, Redaction, X-App-Name, " "X-Billing-Mode, X-BYOK-Provider, X-Client-Request-ID, X-Encryption-Key, " "X-Encryption-Passphrase, X-Fal-Object-Lifecycle-Preference, X-PAYMENT, " @@ -37,7 +37,7 @@ + ", ".join([*CLIENT_HEADER_NAMES.values(), *OPENCODE_CLIENT_HEADER_NAMES.values()]) ) CORS_EXPOSE_HEADERS = ( - "Retry-After, X-Request-ID, X-MultiLLM-RateLimit-Limit, X-MultiLLM-RateLimit-Remaining, X-MultiLLM-RateLimit-Reset, X-MultiLLM-Idempotency, X-MultiLLM-Optimization, X-MultiLLM-Tool-Repair, X-MultiLLM-Image-QA, X-MultiLLM-Cascade, X-MultiLLM-Conversion-Fidelity, X-MultiLLM-Conversion-Fields, " + "Retry-After, Location, X-Request-ID, X-MultiLLM-RateLimit-Limit, X-MultiLLM-RateLimit-Remaining, X-MultiLLM-RateLimit-Reset, X-MultiLLM-Idempotency, X-MultiLLM-Optimization, X-MultiLLM-Tool-Repair, X-MultiLLM-Image-QA, X-MultiLLM-Cascade, X-MultiLLM-Conversion-Fidelity, X-MultiLLM-Conversion-Fields, " "X-MultiLLM-Optimization-Mode, X-MultiLLM-Estimated-Input-Before, " "X-MultiLLM-Estimated-Input-After, X-MultiLLM-Image-Prompts-Compacted, " "X-MultiLLM-Messages-Summarized, X-MultiLLM-Optimization-Target-Met, " diff --git a/routes/gateway_batches.py b/routes/gateway_batches.py new file mode 100644 index 00000000..4177bec9 --- /dev/null +++ b/routes/gateway_batches.py @@ -0,0 +1,268 @@ +"""Owned files, batch lifecycle and explicit asynchronous managed submission.""" +from __future__ import annotations + +import base64 +import json +import re +import time +from functools import wraps + +from flask import Response, g, jsonify, request + +from services import gateway_batches as batches +from services.retention_policy import request_policy, resolve_policy +from services.cost_service import CostService + + +def batch_error(error): + return jsonify({"error": {"code": error.code, "message": error.message, + **({"line": error.line} if error.line else {})}}), error.status + + +def require_content(): + if not request_policy().allows_content: + raise batches.BatchError(400, "retention_conflict", "Zero-content retention cannot persist batch files or requests.") + + +def spillover_hook(): + if (not batches.enabled() or not batches.flag("BATCH_SPILLOVER_ENABLED") or request.method != "POST" + or request.path != "/v1/chat/completions" or getattr(g, "gateway_batch_execution", False) + or request.headers.get("X-MultiLLM-Priority", "").strip().lower() != "batch" + or "respond-async" not in {v.strip().lower() for v in request.headers.get("Prefer", "").split(",")}): + return None + try: + from route_helpers import _authorize_api_scope + refused = _authorize_api_scope("chat") + if refused is not None: + return refused + require_content() + body = request.get_json(silent=True) + if not isinstance(body, dict): + raise batches.BatchError(400, "invalid_batch", "Send a JSON request with metadata.multillm_budget_usd.") + metadata = body.get("metadata") + batches.budget_units(metadata.get("multillm_budget_usd") if isinstance(metadata, dict) else None) + item = {"custom_id": "request-1", "method": "POST", "url": request.path, "body": body} + content = (json.dumps(item, ensure_ascii=False) + "\n").encode() + batches.validate_jsonl(content) + batches.estimate_item(item) + stored = batches.call("file_create", owner=g.authenticated_user["username"], filename="spillover.jsonl", + content=base64.b64encode(content).decode()) + file_id = stored.get("file", stored).get("id") + batch = batches.create_batch({"input_file_id": file_id, "endpoint": request.path, "completion_window": "24h", + "metadata": metadata}, g.authenticated_user, **batches.request_identity()) + response = jsonify(batch) + response.status_code = 202 + response.headers["Location"] = f"/v1/batches/{batch['id']}" + return response + except batches.BatchError as error: + return batch_error(error) + + +def register_gateway_batch_routes(app) -> None: + from route_helpers import api_authenticate_only + if app.extensions.get("gateway_batches_registered"): + return + app.extensions["gateway_batches_registered"] = True + def gate_disabled_routes(): + batch_path = request.path == "/internal/gateway/batch-item" or re.fullmatch( + r"/v1/(?:files(?:/[^/]+(?:/content)?)?|batches(?:/[^/]+(?:/cancel)?)?)", request.path) + if batch_path and not batches.enabled(): + return jsonify({"error": {"code": "not_found", "message": "Not found."}}), 404 + return None + # Gate before the application's authentication redirect and CSRF middleware. + # This touches only the newly registered endpoints. + app.before_request_funcs.setdefault(None, []).insert(0, gate_disabled_routes) + app.extensions.setdefault("gateway_after_authentication", []).insert(1, spillover_hook) + + def guarded(view): + authenticated = api_authenticate_only(view) + @wraps(view) + def wrapped(*args, **kwargs): + if not batches.enabled(): + return jsonify({"error": {"code": "not_found", "message": "Not found."}}), 404 + try: + return authenticated(*args, **kwargs) + except batches.BatchError as error: + return batch_error(error) + csrf = app.extensions.get("csrf") + return csrf.exempt(wrapped) if csrf is not None else wrapped + + @app.route("/v1/files", methods=["GET", "POST", "OPTIONS"]) + @guarded + def gateway_files(): + owner = g.authenticated_user["username"] + if request.method == "GET": + return jsonify(public_reply(batches.call("file_list", owner=owner, **pagination()))) + require_content() + request.max_content_length = batches.MAX_FILE_BYTES + 65536 + if request.mimetype != "multipart/form-data" or request.form.get("purpose") != "batch": + raise batches.BatchError(400, "invalid_purpose", "Upload multipart JSONL with purpose=batch.") + files = request.files.getlist("file") + if len(files) != 1 or set(request.files) != {"file"}: + raise batches.BatchError(400, "invalid_file", "Upload exactly one file.") + data = files[0].read(batches.MAX_FILE_BYTES + 1) + items = batches.validate_jsonl(data) + batches.require_item_retention(items, g.authenticated_user, batches.request_identity()["key_hash"]) + return jsonify(batches.call("file_create", owner=owner, filename=(files[0].filename or "input.jsonl")[:255], + content=base64.b64encode(data).decode()).get("file")) + + @app.route("/v1/files/", methods=["GET", "DELETE", "OPTIONS"]) + @guarded + def gateway_file(file_id): + result = batches.call("file_delete" if request.method == "DELETE" else "file_get", + owner=g.authenticated_user["username"], id=file_id) + return jsonify(result.get("file", public_reply(result))) + + @app.route("/v1/files//content", methods=["GET", "OPTIONS"]) + @guarded + def gateway_file_content(file_id): + content = batches.call("file_content", owner=g.authenticated_user["username"], id=file_id)["content"] + return Response(base64.b64decode(content, validate=True), content_type="application/jsonl", + headers={"Cache-Control": "private, no-store", "X-Content-Type-Options": "nosniff"}) + + @app.route("/v1/batches", methods=["GET", "POST", "OPTIONS"]) + @guarded + def gateway_batches(): + if request.method == "GET": + return jsonify(public_reply(batches.call("batch_list", owner=g.authenticated_user["username"], **pagination()))) + require_content() + return jsonify(batches.create_batch(request.get_json(silent=True), g.authenticated_user, **batches.request_identity())) + + @app.route("/v1/batches/", methods=["GET", "OPTIONS"]) + @guarded + def gateway_batch(batch_id): + return jsonify(batches.call("batch_get", owner=g.authenticated_user["username"], id=batch_id)["batch"]) + + @app.route("/v1/batches//cancel", methods=["POST", "OPTIONS"]) + @guarded + def gateway_batch_cancel(batch_id): + return jsonify(batches.call("batch_cancel", owner=g.authenticated_user["username"], id=batch_id)["batch"]) + + @app.route("/internal/gateway/batch-item", methods=["POST"]) + def gateway_batch_item(): + if not batches.enabled(): + return jsonify({"error": {"code": "not_found"}}), 404 + try: + return jsonify(execute_batch_item(app)) + except batches.BatchError as error: + return batch_error(error) + csrf = app.extensions.get("csrf") + if csrf is not None: + csrf.exempt(gateway_batch_item) + + +def public_reply(body): + return {k: v for k, v in body.items() if k != "version"} + + +def pagination(): + try: + limit = int(request.args.get("limit", "20")) + except ValueError: + raise batches.BatchError(400, "invalid_limit", "limit must be between 1 and 100.") from None + if not 1 <= limit <= 100: + raise batches.BatchError(400, "invalid_limit", "limit must be between 1 and 100.") + return {"limit": limit, "after": request.args.get("after", "")[:128]} + + +def execute_batch_item(app): + from services.media_signing import read_principal + from routes.media_batches import principal_user + from services.auth_service import AuthService + header = request.headers.get("Authorization", "") + if not header.startswith("BatchPrincipal "): + raise batches.BatchError(403, "principal_rejected", "Invalid batch principal.") + claims = read_principal(header[len("BatchPrincipal "):], "gateway_batch") + body = request.get_json(silent=True) + if not isinstance(body, dict) or body.get("batch_id") != claims["s"]: + raise batches.BatchError(403, "principal_rejected", "Batch principal does not match.") + # A second Container call cannot reuse a claimed capability. Canonical content + # and controls come from durable storage, never the request's item body. + started = batches.call("item_start", owner=claims["o"], id=claims["s"], idx=body.get("idx"), lease_token=body.get("lease_token")) + user = principal_user(AuthService, claims["o"]) + if user is None or (started["batch"]["key_prefix"] and user.get("api_key_prefix") != started["batch"]["key_prefix"]): + return {"status_code": 403, "body": {"error": {"code": "principal_rejected"}}, "cost_units": 0, "ambiguous": False} + try: + return run_managed_item(app, user, started["item"], started["batch"]) + except Exception: + # The managed accounting finalizer has already kept an uncertain hold. + return {"status_code": 502, "body": {}, "cost_units": None, "ambiguous": True} + + +def run_managed_item(app, user, item, batch): + from route_helpers import _authorize_api_scope, provider_from_request_path + from services import request_accounting + from services.gateway_extensions import after_authentication + from services.generation_deadline import Deadline + from services.request_cancellation import RequestCancellation + from services.rate_limit_service import RateLimitService + with app.app_context(), app.test_request_context(item["url"], method="POST", json=item["body"], + environ_base={"REMOTE_ADDR": batch["client_ip"]}, headers={"CF-Connecting-IP": batch["client_ip"], "Authorization": "BatchPrincipal managed-item"}): + g.authenticated_user = user + g.gateway_batch_execution = True + g.multillm_content_retention = resolve_policy(key_id=str(user.get("id") or user["username"]), + key_hash=batch["key_hash"], route=item["url"]) + deadline = Deadline(time.monotonic() + 30) + g.generation_deadline = deadline + g.generation_deadline_initialized = True + g.cascade_deadline = g.gateway_generation_deadline = deadline.expires_at + g.gateway_cancellation = RequestCancellation() + deadline.arm(g.gateway_cancellation.cancel) + try: + refused = app.preprocess_request() + if refused is None: + refused = request_accounting.check_key_controls(user) + if refused is None: + refused = _authorize_api_scope("chat") + if refused is None and not request_policy().allows_content: + refused = batch_error(batches.BatchError(400, "retention_conflict", "Zero-content retention cannot execute stored batches.")) + if refused is None: + refused = after_authentication() + if refused is None and batches.estimate_item(item) > item["estimate_units"]: + refused = batch_error(batches.BatchError(400, "price_changed", "Current estimated price exceeds the reserved batch hold.")) + if refused is None: + decision = RateLimitService.enforce_request(provider=provider_from_request_path(item["url"], item["body"]), + user=user, payload_bytes=request.get_data(), payload_json=item["body"], remote_addr=batch["client_ip"]) + g.rate_limit = decision.metadata + if not decision.allowed: + refused = (jsonify({"error": {"code": decision.error}}), decision.status_code) + if refused is None: + response, context = dispatch_managed_view(app) + else: + response, context = app.make_response(refused), None + return item_outcome(response, context, item) + finally: + deadline.stop() + + +def dispatch_managed_view(app): + from route_helpers import _call_accounted + view = app.view_functions[request.endpoint].__wrapped__ + captured = [] + @wraps(view) + def dispatch(): + captured.append(getattr(g, "usage_context", None)) + return view() + # Skip only API-key authentication. All managed dispatch, cache, schema and + # accounting wrappers remain; capture the context before finish clears g. + response = app.make_response(_call_accounted(dispatch, (), {})) + return response, captured[0] if captured else None + + +def item_outcome(response, context, item): + from services.managed_turn import completed_envelope + from services.usage_types import UsageObservation + data = response.get_json(silent=True) + if response.is_streamed or not isinstance(data, dict): + response.close() + return {"status_code": 502, "body": {}, "cost_units": None, "ambiguous": True} + usage = UsageObservation.from_body(data) + model = (context.selected if context else None) or item["body"]["model"] + cost = CostService.estimate(model, usage.input_tokens, usage.output_tokens) if usage else None + if context is None or context.cached: + cost = 0 + ambiguous = bool(context and context.ambiguous) or (context is not None and cost is None) + if response.status_code == 200 and not completed_envelope(data): + ambiguous = True + return {"status_code": response.status_code, "body": data, "cost_units": batches.cost_units(cost), + "ambiguous": ambiguous, "request_id": response.headers.get("X-Request-ID")} diff --git a/services/gateway_batches.py b/services/gateway_batches.py new file mode 100644 index 00000000..47ff90ad --- /dev/null +++ b/services/gateway_batches.py @@ -0,0 +1,202 @@ +"""Batch admission, validation and single-submission private storage transport.""" +from __future__ import annotations + +import base64 +import hashlib +import json +import logging +import os +import re +import secrets +from decimal import Decimal, ROUND_CEILING +from functools import lru_cache + +import requests +from requests.adapters import HTTPAdapter + +from services.cost_service import CostService + +ENDPOINT = "http://intelligence.internal/v1/gateway-batches" +ENDPOINTS = frozenset({"/v1/chat/completions", "/v1/responses"}) +MAX_FILE_BYTES = 10 * 1024 * 1024 +MAX_RESPONSE_BYTES = 24 * 1024 * 1024 +MONEY_SCALE = 10_000_000_000 +logger = logging.getLogger(__name__) + + +class BatchError(Exception): + def __init__(self, status: int, code: str, message: str, line: int | None = None): + super().__init__(code) + self.status, self.code, self.message, self.line = status, code, message, line + + +@lru_cache(maxsize=2) +def _warn_once(name: str): + logger.warning("Invalid %s; gateway batch setting disabled", name) + + +def flag(name: str) -> bool: + value = os.environ.get(name, "").strip().lower() + if value in {"", "0", "false", "off", "no"}: + return False + if value in {"1", "true", "on", "yes"}: + return True + _warn_once(name) + return False + + +def enabled() -> bool: + return flag("GATEWAY_BATCHES_ENABLED") + + +def validate_jsonl(data: bytes) -> list[dict]: + def bad(number, message): + raise BatchError(400, "invalid_file", message, number) + if len(data) > MAX_FILE_BYTES: + bad(1, "Batch input exceeds 10 MiB.") + try: + text = data.decode("utf-8") + except UnicodeDecodeError as error: + bad(data[:error.start].count(b"\n") + 1, "Batch input must be UTF-8 JSONL.") + lines = text.split("\n") + if lines[-1] == "": + lines.pop() + if not lines: + bad(1, "Batch input is empty.") + items, seen = [], set() + for number, line in enumerate(lines, 1): + if number > 1000: + bad(number, "Batch input exceeds 1,000 lines.") + try: + item = json.loads(line, parse_constant=lambda _: bad(number, "Non-finite JSON number.")) + except (ValueError, RecursionError): + bad(number, "Invalid JSON object.") + if not isinstance(item, dict): + bad(number, "Each line must contain a JSON object.") + custom_id, body = item.get("custom_id"), item.get("body") + if (not isinstance(custom_id, str) or not re.fullmatch(r"[^\x00-\x1f\x7f]{1,64}", custom_id) + or custom_id in seen or item.get("method") != "POST" or item.get("url") not in ENDPOINTS + or not isinstance(body, dict) or not isinstance(body.get("model"), str) + or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._:/+@-]{0,255}", body["model"]) + or ("stream" in body and body["stream"] is not False)): + bad(number, "Invalid batch item or duplicate custom_id.") + for name in ("max_tokens", "max_completion_tokens", "max_output_tokens"): + if name in body and (type(body[name]) is not int or not 1 <= body[name] <= 262144): + bad(number, "Output token limit must be between 1 and 262144.") + if "n" in body and (type(body["n"]) is not int or not 1 <= body["n"] <= 16): + bad(number, "Completion count must be between 1 and 16.") + seen.add(custom_id) + items.append(item) + return items + + +def budget_units(value) -> int: + if (not isinstance(value, str) or not re.fullmatch(r"[0-9]{1,6}(?:\.[0-9]{1,10})?", value) + or not Decimal(0) < Decimal(value) <= Decimal(100_000)): + raise BatchError(400, "invalid_budget", "metadata.multillm_budget_usd must be a positive decimal string, at most 100000 USD and 10 fractional digits.") + return int(Decimal(value) * MONEY_SCALE) + + +def cost_units(value) -> int | None: + if value is None: + return None + amount = Decimal(str(value)) + if not amount.is_finite() or amount < 0 or amount > 100_000: + return None + return int((amount * MONEY_SCALE).to_integral_value(rounding=ROUND_CEILING)) + + +def estimate_item(item: dict) -> int: + from services.request_accounting import _candidates + body = item["body"] + # Byte count is a conservative input estimate. Output exposure uses the caller's + # bound when supplied and a conservative 262144-token bound otherwise. + input_tokens = len(json.dumps(body, ensure_ascii=False).encode()) + output_tokens = max((body[name] for name in ("max_tokens", "max_completion_tokens", "max_output_tokens") if name in body), default=262144) * body.get("n", 1) + candidates = _candidates(body["model"]) + prices = [CostService.estimate(model, input_tokens, output_tokens) for model in candidates] + if "routing" in body or not prices or any(price is None for price in prices): + raise BatchError(400, "unknown_price", "Every batch item and route candidate must have a known model price.") + estimates = [cost_units(price) for price in prices] + if any(value is None for value in estimates): + raise BatchError(400, "invalid_price", "The estimated item cost exceeds the batch monetary bound.") + return max(value for value in estimates if value is not None) + + +def call(operation: str, **fields) -> dict: + """No HTTP retries or redirects: an uncertain write is never resubmitted.""" + data = json.dumps({"version": 1, "operation": operation, **fields}, separators=(",", ":")).encode() + if len(data) > 16 * 1024 * 1024: + raise BatchError(400, "request_too_large", "Batch operation exceeds its size bound.") + try: + with requests.Session() as session: + session.trust_env = False + session.mount("http://", HTTPAdapter(max_retries=0)) + with session.post(ENDPOINT, data=data, timeout=(3, 30), allow_redirects=False, stream=True, + headers={"Content-Type": "application/json"}) as response: + content = bytearray() + for chunk in response.iter_content(65536): + content.extend(chunk) + if len(content) > MAX_RESPONSE_BYTES: + raise BatchError(503, "gateway_batches_unavailable", "Batch storage returned an oversized reply.") + status = response.status_code + except requests.exceptions.RequestException: + raise BatchError(503, "gateway_batches_unavailable", "Batch storage could not be reached.") from None + try: + payload = json.loads(content) + if not isinstance(payload, dict) or payload.get("version") != 1: + raise ValueError() + except (ValueError, RecursionError): + raise BatchError(503, "gateway_batches_unavailable", "Batch storage returned an invalid reply.") from None + if status != 200: + error = payload.get("error") or {} + raise BatchError(status, error.get("code", "gateway_batches_unavailable"), + error.get("message", "Batch operation failed."), error.get("line")) + return payload + + +def create_batch(body: dict, user: dict, *, client_ip: str, key_hash: str, key_prefix: str) -> dict: + from services import key_controls + from services.media_signing import issue_principal + if not isinstance(body, dict) or body.get("endpoint") not in ENDPOINTS or body.get("completion_window") != "24h": + raise BatchError(400, "invalid_batch", "Send an input_file_id, supported endpoint and completion_window of 24h.") + if not isinstance(body.get("input_file_id"), str) or not re.fullmatch(r"file_[A-Za-z0-9_]{1,128}", body["input_file_id"]): + raise BatchError(400, "invalid_batch", "Send a valid input_file_id.") + metadata = body.get("metadata") + if (not isinstance(metadata, dict) or len(metadata) > 16 + or any(not isinstance(k, str) or len(k) > 64 or not isinstance(v, str) or len(v) > 512 for k, v in metadata.items())): + raise BatchError(400, "invalid_batch", "metadata accepts at most 16 bounded string values, including multillm_budget_usd.") + budget = budget_units(metadata.get("multillm_budget_usd")) + owner = user["username"] + content = call("file_content", id=body.get("input_file_id"), owner=owner)["content"] + items = validate_jsonl(base64.b64decode(content, validate=True)) + require_item_retention(items, user, key_hash) + estimates = [] + for item in items: + if item["url"] != body["endpoint"]: + raise BatchError(400, "endpoint_mismatch", "Every item must match the batch endpoint.") + if not key_controls.model_allowed(user, item["body"]["model"]): + raise BatchError(403, "model_not_allowed", "A batch item is outside this key's model permissions.") + estimates.append({"custom_id": item["custom_id"], "estimate_units": estimate_item(item)}) + batch_id = "batch_" + secrets.token_hex(16) + return call("batch_create", id=batch_id, owner=owner, input_file_id=body["input_file_id"], endpoint=body["endpoint"], + completion_window="24h", metadata=metadata, budget_units=budget, estimates=estimates, + principal=issue_principal("gateway_batch", batch_id, owner, 86500), client_ip=client_ip, + key_hash=key_hash, key_prefix=key_prefix)["batch"] + + +def request_identity() -> dict: + from flask import g, request + from route_helpers import request_api_key + from services.key_controls import client_ip + key = request_api_key() or "" + return {"client_ip": client_ip(request) or "", "key_hash": hashlib.sha256(key.encode()).hexdigest() if key else "", + "key_prefix": (g.authenticated_user or {}).get("api_key_prefix") or ""} + + +def require_item_retention(items: list[dict], user: dict, key_hash: str) -> None: + from services.retention_policy import resolve_policy + for endpoint in {item["url"] for item in items}: + if not resolve_policy(key_id=str(user.get("id") or user["username"]), key_hash=key_hash, + route=endpoint).allows_content: + raise BatchError(400, "retention_conflict", "Zero-content retention cannot persist batch items for this endpoint.") diff --git a/tests/fixtures/secret_firewall_coverage.json b/tests/fixtures/secret_firewall_coverage.json index 3bc276f1..52be50a9 100644 --- a/tests/fixtures/secret_firewall_coverage.json +++ b/tests/fixtures/secret_firewall_coverage.json @@ -53,6 +53,15 @@ "app.route('/v1/free/models', methods=['GET', 'OPTIONS'])", "app.route('/v1/free/providers', methods=['GET', 'OPTIONS'])" ], + "routes/gateway_batches.py": [ + "app.route('/internal/gateway/batch-item', methods=['POST'])", + "app.route('/v1/batches', methods=['GET', 'POST', 'OPTIONS'])", + "app.route('/v1/batches/', methods=['GET', 'OPTIONS'])", + "app.route('/v1/batches//cancel', methods=['POST', 'OPTIONS'])", + "app.route('/v1/files', methods=['GET', 'POST', 'OPTIONS'])", + "app.route('/v1/files/', methods=['GET', 'DELETE', 'OPTIONS'])", + "app.route('/v1/files//content', methods=['GET', 'OPTIONS'])" + ], "routes/gateway_mcp.py": [ "app.add_url_rule(ENDPOINT, 'gateway_mcp', csrf.exempt(api_authenticate_only(required_scope=_entry_scope)(_mcp)), methods=['POST', 'GET', 'DELETE', 'OPTIONS'])" ], @@ -224,6 +233,8 @@ "worker/alert-delivery.mjs", "worker/api-paths.mjs", "worker/auto-routes-d1.mjs", + "worker/batch-files.mjs", + "worker/batch-jobs.mjs", "worker/build-id.mjs", "worker/cascades-d1.mjs", "worker/client-headers.mjs", diff --git a/tests/helpers/load_cloudflare_worker.mjs b/tests/helpers/load_cloudflare_worker.mjs index 677b0cc8..52ae1e66 100644 --- a/tests/helpers/load_cloudflare_worker.mjs +++ b/tests/helpers/load_cloudflare_worker.mjs @@ -746,6 +746,7 @@ export async function loadWorkerModule({ transformSource = source => source } = .replace('from "./worker/intelligence-outbound.mjs";', `from "${new URL("../../worker/intelligence-outbound.mjs", import.meta.url)}";`) .replace('from "./worker/admission-do.mjs";', `from "${new URL("../../worker/admission-do.mjs", import.meta.url)}";`) .replace('from "./worker/gateway-extensions.mjs";', `from "${new URL("../../worker/gateway-extensions.mjs", import.meta.url)}";`) + .replace('from "./worker/batch-jobs.mjs";', `from "${new URL("../../worker/batch-jobs.mjs", import.meta.url)}";`) .replace('from "./worker/client-headers.mjs";', `from "${new URL("../../worker/client-headers.mjs", import.meta.url)}";`) .replace('from "./worker/opencode-session.mjs";', `from "${new URL("../../worker/opencode-session.mjs", import.meta.url)}";`) .replace('from "./worker/opencode/reasoning-request.mjs";', `from "${new URL("../../worker/opencode/reasoning-request.mjs", import.meta.url)}";`) diff --git a/tests/test_gateway_batches.py b/tests/test_gateway_batches.py new file mode 100644 index 00000000..64970c68 --- /dev/null +++ b/tests/test_gateway_batches.py @@ -0,0 +1,353 @@ +"""Owned batch files, explicit admission and real managed execution boundaries.""" +import io +import json + +import pytest +from flask import Flask, g, jsonify +from flask_wtf.csrf import CSRFProtect + +from error_handlers import init_error_handlers +from services import gateway_batches as batches +from routes.gateway_batches import register_gateway_batch_routes + + +def line(custom_id="one", **changes): + return {"custom_id": custom_id, "method": "POST", "url": "/v1/chat/completions", + "body": {"model": "openai:test", "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 10}, **changes} + + +@pytest.fixture +def client(monkeypatch): + monkeypatch.setenv("GATEWAY_BATCHES_ENABLED", "true") + monkeypatch.setenv("BATCH_SPILLOVER_ENABLED", "false") + monkeypatch.setenv("CONTENT_RETENTION_ENABLED", "false") + monkeypatch.setenv("MODEL_PRICING_USD_PER_MILLION", '{"openai:test":{"input":1,"output":2}}') + app = Flask(__name__) + app.config.update(TESTING=True, SECRET_KEY="synthetic-batch-secret") + init_error_handlers(app) + CSRFProtect(app) + import route_helpers + monkeypatch.setattr(route_helpers, "_authenticate_api_request", lambda: None) + monkeypatch.setattr(route_helpers, "_authorize_api_scope", lambda _: None) + monkeypatch.setattr(route_helpers, "_call_accounted", lambda target, args, kwargs: target(*args, **kwargs)) + @app.before_request + def owner(): + g.authenticated_user = {"username": "alice", "scopes": ["chat"]} + register_gateway_batch_routes(app) + app.batch_calls = [] + def call(operation, **fields): + app.batch_calls.append((operation, fields)) + if operation == "file_content": + import base64 + return {"content": base64.b64encode((json.dumps(line()) + "\n").encode()).decode()} + return {"data": [], "object": "list", "id": fields.get("id", "file_test"), + "batch": {"id": "batch_test", "object": "batch", "status": "validating"}} + monkeypatch.setattr(batches, "call", call) + return app.test_client(), app + + +@pytest.mark.parametrize("value", ["", "false", "0", "invalid", "secret-value"]) +def test_disabled_paths_do_not_touch_storage(client, monkeypatch, value): + c, app = client + monkeypatch.setenv("GATEWAY_BATCHES_ENABLED", value) + for path in ("/v1/files", "/v1/files/file_test", "/v1/files/file_test/content", "/v1/batches"): + assert c.get(path).status_code == 404 + assert not app.batch_calls + + +def test_upload_validates_before_storage(client): + c, app = client + response = c.post("/v1/files", data={"purpose": "batch", "file": (io.BytesIO((json.dumps(line()) + "\n").encode()), "input.jsonl")}) + assert response.status_code == 200 + assert app.batch_calls[0][0] == "file_create" + response = c.post("/v1/files", data={"purpose": "batch", "file": (io.BytesIO((json.dumps(line()) + "\n{}\n").encode()), "input.jsonl")}) + assert response.status_code == 400 + assert response.json["error"]["line"] == 2 + assert len(app.batch_calls) == 1 + + +@pytest.mark.parametrize("data,bad_line", [ + (b"not json\n", 1), (b"\xff", 1), (b"", 1), + ((json.dumps(line()) + "\n" + json.dumps(line())).encode(), 2), + (json.dumps(line(method="GET")).encode(), 1), + (json.dumps(line(url="https://example.com")).encode(), 1), + (json.dumps(line(body={"model": "x", "stream": True})).encode(), 1), + (("\n".join(json.dumps(line(str(i))) for i in range(1001))).encode(), 1001), +]) +def test_invalid_jsonl_reports_first_line(data, bad_line): + with pytest.raises(batches.BatchError) as raised: + batches.validate_jsonl(data) + assert raised.value.status == 400 + assert raised.value.line == bad_line + + +def test_byte_limit(): + with pytest.raises(batches.BatchError) as raised: + batches.validate_jsonl(b" " * (10 * 1024 * 1024 + 1)) + assert raised.value.status == 400 + + +@pytest.mark.parametrize("budget", [None, "0", "-1", "NaN", "1e2", 1, "0.00000000001"]) +def test_budget_requires_positive_bounded_decimal_string(budget): + with pytest.raises(batches.BatchError): + batches.budget_units(budget) + + +def test_creation_rejects_unknown_prices(client, monkeypatch): + c, app = client + monkeypatch.setenv("MODEL_PRICING_USD_PER_MILLION", "{}") + response = c.post("/v1/batches", json={"input_file_id": "file_test", "endpoint": "/v1/chat/completions", + "completion_window": "24h", "metadata": {"multillm_budget_usd": "1"}}) + assert response.status_code == 400 + assert response.json["error"]["code"] == "unknown_price" + assert not any(op == "batch_create" for op, _ in app.batch_calls) + + +def test_missing_table_has_clear_json_503(client, monkeypatch): + c, _ = client + monkeypatch.setattr(batches, "call", lambda *a, **kw: (_ for _ in ()).throw(batches.BatchError(503, "gateway_batches_unavailable", "Batch storage is unavailable; apply its D1 migration."))) + response = c.get("/v1/batches") + assert response.status_code == 503 + assert "migration" in response.json["error"]["message"] + + +def test_zero_retention_rejects_writes(client, monkeypatch): + c, app = client + monkeypatch.setenv("CONTENT_RETENTION_ENABLED", "true") + monkeypatch.setenv("CONTENT_RETENTION_POLICY_JSON", '{"default":"zero"}') + for path in ("/v1/files", "/v1/batches"): + response = c.post(path, json={}) + assert response.status_code == 400 + assert response.json["error"]["code"] == "retention_conflict" + assert not app.batch_calls + + + +def test_endpoint_zero_retention_rejects_upload_and_creation(client, monkeypatch): + c, app = client + monkeypatch.setenv("CONTENT_RETENTION_ENABLED", "true") + monkeypatch.setenv("CONTENT_RETENTION_POLICY_JSON", '{"routes":{"/v1/chat/completions":"zero"}}') + uploaded = c.post("/v1/files", data={"purpose": "batch", "file": (io.BytesIO(json.dumps(line()).encode()), "input.jsonl")}) + assert uploaded.status_code == 400 + assert uploaded.json["error"]["code"] == "retention_conflict" + assert not app.batch_calls + created = c.post("/v1/batches", json={"input_file_id": "file_test", "endpoint": "/v1/chat/completions", + "completion_window": "24h", "metadata": {"multillm_budget_usd": "1"}}) + assert created.status_code == 400 + assert created.json["error"]["code"] == "retention_conflict" + assert not any(op == "batch_create" for op, _ in app.batch_calls) + +def test_spillover_is_explicit_and_returns_real_batch(client, monkeypatch): + c, app = client + @app.post("/v1/chat/completions") + def chat(): + from services.gateway_extensions import after_authentication + refused = after_authentication() + return refused if refused is not None else jsonify({"normal": True}) + app.extensions["csrf"].exempt(chat) + headers = {"X-MultiLLM-Priority": "batch", "Prefer": "respond-async"} + assert c.post("/v1/chat/completions", json=line()["body"], headers=headers).json == {"normal": True} + monkeypatch.setenv("BATCH_SPILLOVER_ENABLED", "true") + for partial in ({"Prefer": "respond-async"}, {"X-MultiLLM-Priority": "batch"}): + assert c.post("/v1/chat/completions", json=line()["body"], headers=partial).status_code == 200 + monkeypatch.setenv("FLASK_SECRET_KEY", "synthetic-batch-signing") + response = c.post("/v1/chat/completions", json={**line()["body"], "metadata": {"multillm_budget_usd": "1"}}, headers=headers) + assert response.status_code == 202 + assert response.json["object"] == "batch" + assert "choices" not in response.json + assert any(op == "batch_create" for op, _ in app.batch_calls) + + + +@pytest.mark.parametrize("body", [{"model": "x", "max_tokens": 0}, {"model": "x", "max_output_tokens": 262145}, + {"model": "x", "n": True}, {"model": "x", "n": 17}]) +def test_token_and_completion_bounds(body): + with pytest.raises(batches.BatchError) as raised: + batches.validate_jsonl(json.dumps(line(body=body)).encode()) + assert raised.value.status == 400 + assert raised.value.line == 1 + + +def test_upload_purpose_and_byte_bound_fail_before_storage(client): + c, app = client + for purpose, data in (("assistants", json.dumps(line()).encode()), ("batch", b" " * (batches.MAX_FILE_BYTES + 1))): + response = c.post("/v1/files", data={"purpose": purpose, "file": (io.BytesIO(data), "input.jsonl")}) + assert response.status_code == 400 + assert not app.batch_calls + + +def test_estimate_reserves_all_completions_and_conservative_default(client): + single = batches.estimate_item(line()) + assert batches.estimate_item(line(body={**line()["body"], "n": 2})) > single + assert batches.estimate_item(line(body={"model": "openai:test", "messages": []})) > single + + +def test_private_transport_never_retries_or_follows_redirects(monkeypatch): + import requests + operations = [] + class Session: + trust_env = True + def __enter__(self): + return self + def __exit__(self, *args): + return None + def mount(self, prefix, adapter): + assert prefix == "http://" and adapter.max_retries.total == 0 + def post(self, url, **kwargs): + assert self.trust_env is False + assert kwargs["allow_redirects"] is False + assert kwargs["timeout"] == (3, 30) + operations.append(url) + raise requests.exceptions.ConnectionError("synthetic failure") + monkeypatch.setattr(batches.requests, "Session", Session) + with pytest.raises(batches.BatchError) as raised: + batches.call("file_list", owner="alice") + assert raised.value.status == 503 + assert operations == [batches.ENDPOINT] + +from tests.unified_api_test_case import UnifiedApiTestCase +from unittest.mock import patch +import importlib +import os + + +class RegisteredBatchTests(UnifiedApiTestCase): + def setUp(self): + with patch("config.load_runtime_env"): + super().setUp() + for name in ("CONTENT_RETENTION_ENABLED", "USAGE_RESERVATIONS_ENABLED", "MANAGED_IDEMPOTENCY_ENABLED", + "GENERATION_CACHE_SHARED_ENABLED", "USAGE_LEDGER_ENABLED", "PROMPT_CACHE_AFFINITY_ENABLED", + "BATCH_SPILLOVER_ENABLED"): + os.environ[name] = "false" + os.environ.update(GATEWAY_BATCHES_ENABLED="true", SESSION_TIER_MODE="off", + MODEL_PRICING_USD_PER_MILLION='{"mimo:mimo-v2.5":{"input":1,"output":2}}') + self.routes = importlib.import_module("routes.gateway_batches") + self.service = self.routes.batches + self.routes.register_gateway_batch_routes(self.app) + from services.budget_service import BudgetService + BudgetService.reset() + self.addCleanup(BudgetService.reset) + self.user = importlib.import_module("services.auth_service").AuthService.verify_api_key("admin-test-key", "127.0.0.1") + self.item = line(body={"model": "mimo:mimo-v2.5", "messages": [{"role": "user", "content": "hello"}], "max_tokens": 10}) + self.item["estimate_units"] = self.service.estimate_item(self.item) + self.batch = {"id": "batch_test", "client_ip": "127.0.0.1", "key_hash": "", "key_prefix": ""} + + def internal(self): + from services.media_signing import issue_principal + capability = issue_principal("gateway_batch", "batch_test", self.user["username"], 100) + return self.client.post("/internal/gateway/batch-item", json={"batch_id": "batch_test", "idx": 0, "lease_token": "lease_test"}, + headers={"Authorization": "BatchPrincipal " + capability}) + + def test_registered_internal_dispatch_reuses_managed_accounting_and_deadline(self): + accounting = importlib.import_module("services.request_accounting") + seen = [] + def send(**kwargs): + assert g.authenticated_user["username"] == self.user["username"] + assert 0 < g.generation_deadline.remaining() <= 30 + assert g.gateway_cancellation is not None + seen.append(json.loads(kwargs["data"])) + reply = self._chat_response() + body = json.loads(reply.content) + body["usage"] = {"prompt_tokens": 4, "completion_tokens": 2} + reply._content = json.dumps(body).encode() + return reply + with patch.object(self.service, "call", return_value={"item": self.item, "batch": self.batch}) as start, \ + patch.object(self.app_module.ProxyService, "make_request", side_effect=send), \ + patch.object(accounting, "mark_dispatched", wraps=accounting.mark_dispatched) as mark: + response = self.internal() + assert response.status_code == 200, response.json + assert response.json["status_code"] == 200, response.json + assert response.json["cost_units"] == 80000 + assert response.json["ambiguous"] is False + assert len(seen) == mark.call_count == 1 + assert start.call_args.args == ("item_start",) + assert seen[0]["model"] == "mimo-v2.5" + + def test_registered_execution_enforces_owner_permissions_before_provider(self): + restricted = {**self.user, "allowed_models": ["openai:other"]} + with patch.object(self.service, "call", return_value={"item": self.item, "batch": self.batch}), \ + patch.object(importlib.import_module("routes.media_batches"), "principal_user", return_value=restricted), \ + patch.object(self.app_module.ProxyService, "make_request", side_effect=AssertionError("provider called")) as send: + response = self.internal() + assert response.json["status_code"] == 403, response.json + assert response.json["cost_units"] == 0 + assert send.call_count == 0 + + def test_registered_default_traffic_and_explicit_spillover(self): + headers = {"Authorization": "Bearer admin-test-key", "X-MultiLLM-Priority": "batch", "Prefer": "respond-async"} + with patch.object(self.app_module.ProxyService, "make_request", return_value=self._chat_response()) as send: + before = self.client.post("/v1/chat/completions", json=self.item["body"], headers=headers) + assert before.data == self._chat_response().content + assert send.call_count == 1 + os.environ["BATCH_SPILLOVER_ENABLED"] = "true" + def storage(operation, **fields): + if operation == "file_create": + return {"file": {"id": "file_test"}} + if operation == "file_content": + import base64 + return {"content": base64.b64encode(json.dumps({k:v for k,v in self.item.items() if k != "estimate_units"}).encode()).decode()} + return {"batch": {"id": fields["id"], "object": "batch", "status": "validating"}} + with patch.object(self.service, "call", side_effect=storage), \ + patch.object(self.app_module.ProxyService, "make_request", side_effect=AssertionError("provider called")) as send: + queued = self.client.post("/v1/chat/completions", json={**self.item["body"], "metadata": {"multillm_budget_usd": "1"}}, headers=headers) + assert queued.status_code == 202, queued.json + assert queued.json["object"] == "batch" + assert queued.headers["Location"] == "/v1/batches/" + queued.json["id"] + assert send.call_count == 0 + + + def test_registered_disabled_routes_return_404_even_without_authentication(self): + os.environ["GATEWAY_BATCHES_ENABLED"] = "false" + for path in ("/v1/files", "/v1/files/file_test", "/v1/files/file_test/content", "/v1/batches", + "/v1/batches/batch_test", "/v1/batches/batch_test/cancel", "/internal/gateway/batch-item"): + assert self.client.get(path).status_code in {404, 405} + assert self.client.post("/v1/batches", json={}).status_code == 404 + assert self.client.post("/internal/gateway/batch-item", json={}).status_code == 404 + + def test_registered_expired_or_ip_restricted_owner_never_calls_provider(self): + for restriction, status in (({"expires_at": "2000-01-01T00:00:00Z"}, 401), ({"allowed_ips": ["192.0.2.0/24"]}, 403)): + with self.subTest(restriction=restriction), \ + patch.object(self.service, "call", return_value={"item": self.item, "batch": self.batch}), \ + patch.object(importlib.import_module("routes.media_batches"), "principal_user", return_value={**self.user, **restriction}), \ + patch.object(self.app_module.ProxyService, "make_request", side_effect=AssertionError("provider called")) as send: + response = self.internal() + assert response.json["status_code"] == status, response.json + assert response.json["cost_units"] == 0 + assert send.call_count == 0 + + def test_registered_responses_item_uses_its_managed_translator(self): + item = {**self.item, "url": "/v1/responses", "body": {"model": "mimo:mimo-v2.5", "input": "hello", "max_output_tokens": 10}} + item["estimate_units"] = self.service.estimate_item(item) + reply = self._chat_response() + body = json.loads(reply.content) + body["usage"] = {"prompt_tokens": 4, "completion_tokens": 2} + reply._content = json.dumps(body).encode() + with patch.object(self.service, "call", return_value={"item": item, "batch": self.batch}), \ + patch.object(self.app_module.ProxyService, "make_request", return_value=reply) as send: + response = self.internal() + assert response.json["status_code"] == 200, response.json + assert response.json["body"]["object"] == "response" + assert response.json["ambiguous"] is False + assert response.json["cost_units"] == 80000 + assert send.call_count == 1 + + + def test_registered_owner_budget_is_checked_before_submission(self): + limited = {**self.user, "daily_budget_usd": 0.00000001} + with patch.object(self.service, "call", return_value={"item": self.item, "batch": self.batch}), \ + patch.object(importlib.import_module("routes.media_batches"), "principal_user", return_value=limited), \ + patch.object(self.app_module.ProxyService, "make_request", side_effect=AssertionError("provider called")) as send: + response = self.internal() + assert response.json["status_code"] == 429, response.json + assert response.json["cost_units"] == 0 + assert send.call_count == 0 + + def test_registered_lost_usage_retains_an_unknown_hold(self): + with patch.object(self.service, "call", return_value={"item": self.item, "batch": self.batch}), \ + patch.object(self.app_module.ProxyService, "make_request", return_value=self._chat_response()) as send: + response = self.internal() + assert response.json["status_code"] == 200 + assert response.json["ambiguous"] is True + assert response.json["cost_units"] is None + assert send.call_count == 1 diff --git a/tests/test_gateway_batches_worker.mjs b/tests/test_gateway_batches_worker.mjs new file mode 100644 index 00000000..1c8ac180 --- /dev/null +++ b/tests/test_gateway_batches_worker.mjs @@ -0,0 +1,309 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { readFileSync } from "node:fs"; +import { DatabaseSync } from "node:sqlite"; +import { handleBatchRequest, runScheduledBatches, shouldForwardBatchSpillover, containerBatchDispatch } from "../worker/batch-jobs.mjs"; + +const NOW = 1800000000; +const item = n => ({ custom_id: `item-${n}`, method: "POST", url: "/v1/chat/completions", + body: { model: "openai:test", messages: [{ role: "user", content: "hello" }], max_tokens: 10 } }); +const envelope = { choices: [{ message: { content: "hi" }, finish_reason: "stop" }], usage: { prompt_tokens: 2, completion_tokens: 2 } }; +function database(t, schema = true) { + const sql = new DatabaseSync(":memory:"); + t.after(() => sql.close()); + if (schema) sql.exec(readFileSync(new URL("../intelligence-migrations/0027_gateway_batches.sql", import.meta.url), "utf8")); + const db = { prepare(query) { + let values = []; + const stmt = { bind(...args) { values = args; return stmt; }, + run() { const r = sql.prepare(query).run(...values); return { meta: { changes: Number(r.changes) } }; }, + first() { return sql.prepare(query).get(...values) ?? null; }, + all() { return { results: sql.prepare(query).all(...values) }; } }; + return stmt; + }, async batch(statements) { + sql.exec("BEGIN"); + try { const r = statements.map(s => s.run()); sql.exec("COMMIT"); return r; } + catch (error) { sql.exec("ROLLBACK"); throw error; } + } }; + const objects = new Map(); + const bucket = { + async put(key, value) { objects.set(key, typeof value === "string" ? value : new TextDecoder().decode(value)); }, + async get(key) { const value = objects.get(key); return value === undefined ? null : { text: async () => value, arrayBuffer: async () => new TextEncoder().encode(value).buffer }; }, + async delete(key) { objects.delete(key); }, + }; + const env = { GATEWAY_BATCHES_ENABLED: "true", INTELLIGENCE_DB: db, multillm_media: bucket }; + const call = async (operation, fields = {}) => { + const response = await handleBatchRequest(new Request("http://intelligence.internal/v1/gateway-batches", { + method: "POST", body: JSON.stringify({ version: 1, operation, owner: "alice", ...fields }), headers: { "content-type": "application/json" }, + }), env, { now: () => NOW }); + return { status: response.status, ...await response.json() }; + }; + const create = async (count = 3, budget = 10000000000) => { + const items = Array.from({ length: count }, (_, n) => item(n)); + const uploaded = await call("file_create", { filename: "input.jsonl", content: Buffer.from(items.map(v => JSON.stringify(v)).join("\n")).toString("base64") }); + assert.equal(uploaded.status, 200); + const made = await call("batch_create", { input_file_id: uploaded.file.id, endpoint: "/v1/chat/completions", + completion_window: "24h", metadata: { multillm_budget_usd: "1" }, budget_units: budget, + principal: "signed-principal", client_ip: "127.0.0.1", key_hash: "", key_prefix: "", + estimates: items.map(v => ({ custom_id: v.custom_id, estimate_units: 2000000000 })) }); + assert.equal(made.status, 200); + return made.batch; + }; + return { call, create, env, sql, objects }; +} +const dispatch = async () => ({ status_code: 200, body: envelope, cost_units: 1000000000, ambiguous: false }); +const options = extra => ({ now: () => NOW, clock: () => 0, dispatch, ...extra }); + +test("off and malformed flags never touch storage or submit a request", async () => { + for (const value of [undefined, "", "false", "garbage"]) { + const env = { GATEWAY_BATCHES_ENABLED: value, INTELLIGENCE_DB: { prepare() { assert.fail("disabled storage"); } } }; + assert.equal((await handleBatchRequest(new Request("http://intelligence.internal/v1/gateway-batches", { method: "POST" }), env)).status, 404); + assert.deepEqual(await runScheduledBatches(env, null), { claimed: 0 }); + } +}); + +test("missing schema fails closed and enabled storage never calls a provider", async t => { + const { call, env } = database(t, false); + assert.equal((await call("file_list")).status, 503); + assert.equal((await call("batch_list")).error.code, "gateway_batches_unavailable"); + await assert.rejects(runScheduledBatches(env, null, options({ dispatch: () => assert.fail("provider call") }))); +}); + +test("file shapes, content, foreign-owner privacy and input deletion protection", async t => { + const { call, create } = database(t); + const batch = await create(1); + const id = batch.input_file_id; + const file = (await call("file_get", { id })).file; + assert.equal(file.object, "file"); assert.equal(file.purpose, "batch"); + assert.equal((await call("file_get", { id, owner: "bob" })).status, 404); + assert.equal((await call("file_content", { id, owner: "bob" })).status, 404); + assert.equal((await call("file_delete", { id, owner: "bob" })).status, 404); + assert.equal((await call("batch_get", { id: batch.id, owner: "bob" })).status, 404); + assert.equal((await call("batch_cancel", { id: batch.id, owner: "bob" })).status, 404); + assert.equal((await call("file_delete", { id })).status, 409); + assert.equal((await call("file_list", { owner: "bob" })).data.length, 0); +}); + +test("private uploads reject invalid and duplicate items before R2", async t => { + const { call, objects } = database(t); + for (const text of ["{}", JSON.stringify(item(0)) + "\n" + JSON.stringify(item(0)), "not-json", "\n"]) { + assert.equal((await call("file_create", { filename: "input.jsonl", content: Buffer.from(text).toString("base64") })).status, 400); + } + assert.equal(objects.size, 0); +}); + +test("two-item continuation records complete JSONL by custom_id", async t => { + const { call, create, env } = database(t); + const batch = await create(); + assert.equal(batch.status, "validating"); + assert.equal((await runScheduledBatches(env, null, options())).claimed, 2); + assert.equal((await call("batch_get", { id: batch.id })).batch.status, "in_progress"); + assert.equal((await runScheduledBatches(env, null, options())).claimed, 1); + const completed = (await call("batch_get", { id: batch.id })).batch; + assert.equal(completed.status, "completed"); + assert.deepEqual(completed.request_counts, { total: 3, completed: 3, failed: 0 }); + const output = Buffer.from((await call("file_content", { id: completed.output_file_id })).content, "base64").toString(); + assert.deepEqual(output.trim().split("\n").map(s => JSON.parse(s).custom_id), ["item-0", "item-1", "item-2"]); +}); + +test("concurrent schedules claim each item once with atomic budget holds", async t => { + const { create, env, sql } = database(t); + await create(4); + const seen = []; + await Promise.all([1, 2].map(() => runScheduledBatches(env, null, options({ dispatch: async item => { seen.push(item.custom_id); await Promise.resolve(); return dispatch(); } })))); + assert.equal(new Set(seen).size, seen.length); + assert.equal(seen.length, 4); + assert.equal(sql.prepare("SELECT SUM(held_units) AS n FROM gateway_batch_items").get().n, 0); +}); + +test("ambiguous transport is never retried and holds exhaust remaining work", async t => { + const { call, create, env, sql } = database(t); + const batch = await create(3, 2000000000); + let calls = 0; + await runScheduledBatches(env, null, options({ dispatch: async () => { calls++; throw Error("transport lost"); } })); + await runScheduledBatches(env, null, options({ dispatch: () => assert.fail("retry") })); + assert.equal(calls, 1); + assert.equal(sql.prepare("SELECT SUM(held_units) AS n FROM gateway_batch_items").get().n, 2000000000); + const row = (await call("batch_get", { id: batch.id })).batch; + const errors = Buffer.from((await call("file_content", { id: row.error_file_id })).content, "base64").toString(); + assert.match(errors, /outcome_unknown/); assert.match(errors, /budget_exhausted/); +}); + +test("expired leases fail unknown without redispatch", async t => { + const { create, env, sql } = database(t); + const b = await create(1); + sql.prepare("UPDATE gateway_batch_items SET status='dispatched', lease_until=?, held_units=estimate_units WHERE batch_id=?").run(NOW - 1, b.id); + await runScheduledBatches(env, null, options({ dispatch: () => assert.fail("retry") })); + assert.equal(sql.prepare("SELECT error_code FROM gateway_batch_items").get().error_code, "outcome_unknown"); +}); + +test("cancel race permits the already dispatched item and stops new ones", async t => { + const { call, create, env } = database(t); + const b = await create(3); + let calls = 0; + await runScheduledBatches(env, null, options({ dispatch: async () => { calls++; const cancelled = await call("batch_cancel", { id: b.id }); assert.equal(cancelled.batch.status, "cancelling"); return dispatch(); } })); + const result = (await call("batch_get", { id: b.id })).batch; + assert.equal(calls, 1); assert.equal(result.status, "cancelled"); + assert.equal(result.request_counts.completed, 1); +}); + +test("24-hour expiry preserves holds and prevents new work", async t => { + const { call, create, env, sql } = database(t); + const b = await create(2); + await runScheduledBatches(env, null, options({ now: () => NOW + 86400, dispatch: () => assert.fail("expired dispatch") })); + assert.equal((await call("batch_get", { id: b.id })).batch.status, "expired"); + assert.equal(sql.prepare("SELECT COUNT(*) AS n FROM gateway_batch_items WHERE error_code='expired'").get().n, 2); +}); + +test("scheduler time budget starts no item without a full deadline", async t => { + const { create, env } = database(t); + await create(1); + let clock = 0; + const r = await runScheduledBatches(env, null, options({ clock: () => (clock += 40000), dispatch: () => assert.fail("time budget") })); + assert.equal(r.claimed, 0); +}); + +test("spillover requires both flags and both headers on the managed route", () => { + const request = headers => new Request("https://gateway.test/v1/chat/completions", { method: "POST", headers }); + const env = { GATEWAY_BATCHES_ENABLED: "true", BATCH_SPILLOVER_ENABLED: "true" }; + const headers = { "X-MultiLLM-Priority": "batch", Prefer: "respond-async" }; + assert.equal(shouldForwardBatchSpillover(request(headers), env), true); + assert.equal(shouldForwardBatchSpillover(request({ Prefer: "respond-async" }), env), false); + assert.equal(shouldForwardBatchSpillover(request(headers), { ...env, BATCH_SPILLOVER_ENABLED: "false" }), false); +}); + + +test("a durable item capability starts once and cannot replay canonical content", async t => { + const { call, create, env } = database(t); + const b = await create(1); + await runScheduledBatches(env, null, options({ dispatch: async item => { + const fields = { id: b.id, idx: item.idx, lease_token: item.lease_token }; + assert.equal((await call("item_start", { ...fields, owner: "bob" })).status, 404); + assert.equal((await call("item_start", { ...fields, lease_token: "wrong" })).status, 409); + const started = await call("item_start", fields); + assert.equal(started.status, 200); + assert.equal(started.item.custom_id, "item-0"); + assert.equal(started.item.estimate_units, 2000000000); + assert.equal((await call("item_start", fields)).status, 409); + return dispatch(); + } })); + assert.equal((await call("batch_get", { id: b.id })).batch.status, "completed"); +}); + +test("Container transport sends only signed identity and a durable claim", async () => { + const signal = new AbortController().signal; + const result = await containerBatchDispatch({ idx: 2, lease_token: "lease-test", body: { messages: ["private"] } }, + { id: "batch-test", principal: "synthetic-capability" }, { async fetch(request) { + assert.equal(new URL(request.url).pathname, "/internal/gateway/batch-item"); + assert.equal(request.headers.get("authorization"), "BatchPrincipal synthetic-capability"); + assert.deepEqual(await request.json(), { batch_id: "batch-test", idx: 2, lease_token: "lease-test" }); + return Response.json(await dispatch()); + } }, signal); + assert.deepEqual(result, await dispatch()); + await assert.rejects(containerBatchDispatch({}, {}, { fetch: async () => Response.json({ choices: [] }) }, signal)); +}); + +test("result storage loss retains the claim and unknown hold without resubmission", async t => { + const { create, env, sql } = database(t); + await create(1); + const put = env.multillm_media.put; + env.multillm_media.put = async key => { if (key.includes("/results/")) throw Error("storage unavailable"); }; + await assert.rejects(runScheduledBatches(env, null, options())); + env.multillm_media.put = put; + await runScheduledBatches(env, null, options({ now: () => NOW + 41, dispatch: () => assert.fail("resubmission") })); + const stored = sql.prepare("SELECT * FROM gateway_batch_items").get(); + assert.equal(stored.error_code, "outcome_unknown"); + assert.equal(stored.held_units, 2000000000); +}); + +test("finalization resumes from an immutable checkpoint without duplicating custom ids", async t => { + const { create, call, env, sql } = database(t); + const b = await create(2); + sql.prepare("UPDATE gateway_batch_items SET status='failed',error_code='cancelled'").run(); + let clockCalls = 0; + await runScheduledBatches(env, null, options({ clock: () => ++clockCalls <= 4 ? 0 : 70000 })); + const checkpoint = sql.prepare("SELECT checkpoint FROM gateway_batches").get().checkpoint; + assert.ok(checkpoint); + await runScheduledBatches(env, null, options()); + const completed = (await call("batch_get", { id: b.id })).batch; + const text = Buffer.from((await call("file_content", { id: completed.error_file_id })).content, "base64").toString(); + assert.deepEqual(text.trim().split("\n").map(x => JSON.parse(x).custom_id), ["item-0", "item-1"]); +}); + +test("terminal files remain owned and the released input can be deleted", async t => { + const { create, call, env } = database(t); + const b = await create(1); + await runScheduledBatches(env, null, options()); + const completed = (await call("batch_get", { id: b.id })).batch; + assert.equal((await call("file_content", { id: completed.output_file_id, owner: "bob" })).status, 404); + assert.equal((await call("file_delete", { id: b.input_file_id })).deleted, true); + assert.equal((await call("file_get", { id: b.input_file_id })).status, 404); +}); + +test("migration is additive and repeatable around existing database rows", t => { + const { sql } = database(t, false); + sql.exec("CREATE TABLE existing_rows (value TEXT); INSERT INTO existing_rows VALUES ('preserved');"); + const migration = readFileSync(new URL("../intelligence-migrations/0027_gateway_batches.sql", import.meta.url), "utf8"); + sql.exec(migration); sql.exec(migration); + assert.equal(sql.prepare("SELECT value FROM existing_rows").get().value, "preserved"); +}); + +test("Worker outbound registration reaches the private handler with schema failures", async t => { + const { loadWorkerModule } = await import("./helpers/load_cloudflare_worker.mjs"); + const worker = await loadWorkerModule({ transformSource: source => source.replace('from "./worker/batch-jobs.mjs";', + `from "${new URL("../worker/batch-jobs.mjs", import.meta.url)}";`) }); + const { env } = database(t, false); + const request = new Request("http://intelligence.internal/v1/gateway-batches", { method: "POST", body: JSON.stringify({ version: 1, owner: "alice", operation: "batch_list" }) }); + const response = await worker.MultiLLMProxyContainer.outboundByHost["intelligence.internal"](request, env); + assert.equal(response.status, 503); + assert.match((await response.json()).error.message, /migration/); +}); + + +test("aggregate result caps preserve a compact error for every custom id across recovery", async t => { + const { create, call, env, sql, objects } = database(t); + const b = await create(20); + for (let idx = 0; idx < 20; idx++) { + const key = `batches/results/${b.id}/${idx}/test`; + const result = JSON.stringify({ custom_id: `item-${idx}`, response: { body: { text: "x".repeat(1024 * 1024 - 200) } }, error: null }) + "\n"; + objects.set(key, result); + sql.prepare("UPDATE gateway_batch_items SET status='completed',result_key=? WHERE batch_id=? AND idx=?").run(key, b.id, idx); + } + // Interrupt after the overflow statuses change; rebuilding must keep errors real. + const put = env.multillm_media.put; + env.multillm_media.put = async () => { throw Error("final output unavailable"); }; + await assert.rejects(runScheduledBatches(env, null, options())); + env.multillm_media.put = put; + await runScheduledBatches(env, null, options({ now: () => NOW + 91 })); + const completed = (await call("batch_get", { id: b.id })).batch; + const output = Buffer.from((await call("file_content", { id: completed.output_file_id })).content, "base64"); + const errors = Buffer.from((await call("file_content", { id: completed.error_file_id })).content, "base64"); + assert.ok(output.length <= 16 * 1024 * 1024); + assert.ok(errors.length <= 16 * 1024 * 1024); + const rows = (output.toString() + errors.toString()).trim().split("\n").map(x => JSON.parse(x)); + assert.equal(new Set(rows.map(x => x.custom_id)).size, 20); + assert.equal(rows.filter(x => x.error?.code === "output_file_too_large").length, 4); + assert.equal(completed.request_counts.failed, 4); +}); + + +test("the thirty-second item deadline aborts transport and never retries the unknown outcome", async t => { + const { create, call, env, sql } = database(t); + const b = await create(1); + t.mock.timers.enable({ apis: ["setTimeout"] }); + let started, signal; + const submitted = new Promise(resolve => { started = resolve; }); + const running = runScheduledBatches(env, null, options({ dispatch: (_item, _batch, _container, abort) => { + signal = abort; + started(); + return new Promise(() => {}); + } })); + await submitted; + t.mock.timers.tick(30000); + assert.equal((await running).claimed, 1); + assert.equal(signal.aborted, true); + await runScheduledBatches(env, null, options({ dispatch: () => assert.fail("deadline resubmission") })); + const result = sql.prepare("SELECT error_code,held_units FROM gateway_batch_items").get(); + assert.equal(result.error_code, "outcome_unknown"); + assert.equal(result.held_units, 2000000000); + assert.equal((await call("batch_get", { id: b.id })).batch.request_counts.failed, 1); +}); diff --git a/worker/batch-files.mjs b/worker/batch-files.mjs new file mode 100644 index 00000000..071869d8 --- /dev/null +++ b/worker/batch-files.mjs @@ -0,0 +1,124 @@ +/** Batch content stays in the existing media bucket; D1 holds only ownership and pointers. */ +import { boundedBytes } from "./media-outbound.mjs"; + +export const MAX_FILE_BYTES = 10 * 1024 * 1024; +export const MAX_RESULT_BYTES = 1024 * 1024; +export const MAX_OUTPUT_BYTES = 16 * 1024 * 1024; +export const ENDPOINTS = new Set(["/v1/chat/completions", "/v1/responses"]); +export const record = value => value !== null && typeof value === "object" && !Array.isArray(value); +export const newId = prefix => `${prefix}_${crypto.randomUUID().replaceAll("-", "")}`; +export const reply = (body, status = 200) => Response.json({ version: 1, ...body }, + { status, headers: { "cache-control": "no-store" } }); +export function failure(code, message, status = 400, line) { + return reply({ error: { code, message, ...(line ? { line } : {}) } }, status); +} +export function batchFlag(env, name) { + const value = String(env[name] ?? "").trim().toLowerCase(); + if (["", "false", "0", "no", "off"].includes(value)) return false; + if (["true", "1", "yes", "on"].includes(value)) return true; + if (!warned.has(name)) { warned.add(name); console.warn(JSON.stringify({ event: "invalid_batch_setting", setting: name })); } + return false; +} +const warned = new Set(); + +export function validateJsonl(bytes) { + const bad = (line, message) => { const error = Error(message); error.line = line; throw error; }; + if (bytes.byteLength > MAX_FILE_BYTES) bad(1, "Batch input exceeds 10 MiB."); + let text; + try { text = new TextDecoder("utf-8", { fatal: true }).decode(bytes); } + catch { bad(1, "Batch input must be UTF-8 JSONL."); } + const lines = text.split("\n"); + if (lines.at(-1) === "") lines.pop(); + if (!lines.length) bad(1, "Batch input is empty."); + const items = [], seen = new Set(); + for (let i = 0; i < lines.length; i++) { + if (i >= 1000) bad(i + 1, "Batch input exceeds 1,000 lines."); + let item; + try { item = JSON.parse(lines[i]); } catch { bad(i + 1, "Invalid JSON object."); } + if (!record(item) || typeof item.custom_id !== "string" || !/^[^\x00-\x1f\x7f]{1,64}$/.test(item.custom_id) + || seen.has(item.custom_id) || item.method !== "POST" || !ENDPOINTS.has(item.url) || !record(item.body) + || typeof item.body.model !== "string" || !/^[A-Za-z0-9][A-Za-z0-9._:/+@-]{0,255}$/.test(item.body.model) + || (item.body.stream !== undefined && item.body.stream !== false)) bad(i + 1, "Invalid batch item or duplicate custom_id."); + for (const name of ["max_tokens", "max_completion_tokens", "max_output_tokens"]) { + if (item.body[name] !== undefined && (!Number.isSafeInteger(item.body[name]) || item.body[name] < 1 + || item.body[name] > 262144)) bad(i + 1, "Output token limit must be between 1 and 262144."); + } + if (item.body.n !== undefined && (!Number.isSafeInteger(item.body.n) || item.body.n < 1 || item.body.n > 16)) bad(i + 1, "Completion count must be between 1 and 16."); + seen.add(item.custom_id); items.push(item); + } + return items; +} + +export function fileObject(row) { + return { id: row.id, object: "file", bytes: row.bytes, created_at: row.created_at, + filename: row.filename, purpose: row.purpose, status: "processed", status_details: null }; +} +export const ownedFile = (db, owner, id) => db.prepare("SELECT * FROM gateway_batch_files WHERE owner=? AND id=?").bind(owner, id).first(); + +export async function storeFile(env, owner, filename, content, purpose, now, id = newId("file")) { + const key = `batches/files/${id}`; + const size = new TextEncoder().encode(content).byteLength; + await env.multillm_media.put(key, content, { httpMetadata: { contentType: "application/jsonl" } }); + await env.INTELLIGENCE_DB.prepare(`INSERT INTO gateway_batch_files + (id,owner,filename,purpose,bytes,r2_key,created_at) VALUES (?,?,?,?,?,?,?)`) + .bind(id, owner, filename, purpose, size, key, now).run(); + return fileObject({ id, bytes: size, created_at: now, filename, purpose }); +} + +export async function inputItems(env, file) { + const object = await env.multillm_media.get(file.r2_key); + if (!object) throw Error("batch_input_missing"); + return validateJsonl(new Uint8Array(await object.arrayBuffer())); +} + +export async function handleFileOperation(body, env, now) { + const db = env.INTELLIGENCE_DB; + if (body.operation === "file_create") { + if (typeof body.filename !== "string" || body.filename.length > 255 || typeof body.content !== "string" + || body.content.length > Math.ceil(MAX_FILE_BYTES / 3) * 4 || !/^[A-Za-z0-9+/]*={0,2}$/.test(body.content)) { + return failure("invalid_file", "Send a bounded JSONL file."); + } + let bytes, items; + try { bytes = Uint8Array.from(atob(body.content), c => c.charCodeAt(0)); items = validateJsonl(bytes); } + catch (error) { return failure("invalid_file", error.line ? error.message : "Invalid file encoding.", 400, error.line ?? 1); } + // Preserve the uploaded bytes exactly, including whitespace and CRLF. + return reply({ file: await storeFile(env, body.owner, body.filename || "input.jsonl", + new TextDecoder().decode(bytes), "batch", now), item_count: items.length }); + } + if (body.operation === "file_list") { + const rows = await db.prepare("SELECT * FROM gateway_batch_files WHERE owner=? AND id>? ORDER BY id LIMIT ?") + .bind(body.owner, body.after ?? "", pageLimit(body.limit) + 1).all(); + return reply(listObject(rows.results.map(fileObject), pageLimit(body.limit))); + } + const file = await ownedFile(db, body.owner, body.id ?? ""); + if (!file) return failure("not_found", "File not found.", 404); + if (body.operation === "file_get") return reply({ file: fileObject(file) }); + if (body.operation === "file_content") { + const object = await env.multillm_media.get(file.r2_key); + if (!object) throw Error("batch_file_missing"); + const bytes = new Uint8Array(await object.arrayBuffer()); + let text = ""; + for (let start = 0; start < bytes.length; start += 8192) text += String.fromCharCode(...bytes.subarray(start, start + 8192)); + return reply({ content: btoa(text) }); + } + if (body.operation === "file_delete") { + const active = await db.prepare(`SELECT id FROM gateway_batches WHERE input_file_id=? + AND status IN ('validating','in_progress','finalizing','cancelling') LIMIT 1`).bind(file.id).first(); + if (active) return failure("file_in_use", "An active batch still uses this file.", 409); + await env.multillm_media.delete(file.r2_key); + await db.prepare("DELETE FROM gateway_batch_files WHERE id=? AND owner=?").bind(file.id, body.owner).run(); + return reply({ id: file.id, object: "file", deleted: true }); + } + return failure("invalid_operation", "Unknown file operation."); +} +export const pageLimit = value => Number.isSafeInteger(value) && value >= 1 && value <= 100 ? value : 20; +export function listObject(data, limit) { + const page = data.slice(0, limit); + return { object: "list", data: page, first_id: page[0]?.id ?? null, last_id: page.at(-1)?.id ?? null, has_more: data.length > limit }; +} +export async function readBatchBody(request) { + const bytes = await boundedBytes(request, 16 * 1024 * 1024); + if (!bytes) return null; + try { const body = JSON.parse(new TextDecoder().decode(bytes)); return record(body) ? body : null; } + catch { return null; } +} diff --git a/worker/batch-jobs.mjs b/worker/batch-jobs.mjs new file mode 100644 index 00000000..7e0f2298 --- /dev/null +++ b/worker/batch-jobs.mjs @@ -0,0 +1,315 @@ +/** Durable, single-submission batch execution on the owner's managed Container path. */ +import { batchFlag, record, reply, failure, ENDPOINTS, newId, ownedFile, inputItems, + handleFileOperation, pageLimit, listObject, readBatchBody, MAX_RESULT_BYTES, + MAX_OUTPUT_BYTES, storeFile } from "./batch-files.mjs"; +import { boundedBytes } from "./media-outbound.mjs"; + +const ACTIVE = "('validating','in_progress','cancelling','finalizing')"; +const nowSeconds = () => Math.floor(Date.now() / 1000); +const units = value => Number.isSafeInteger(value) && value >= 0; +const row = (db, id) => db.prepare("SELECT * FROM gateway_batches WHERE id=?").bind(id).first(); +const owned = (db, body) => db.prepare("SELECT * FROM gateway_batches WHERE id=? AND owner=?").bind(body.id ?? "", body.owner).first(); +export const batchesEnabled = env => batchFlag(env, "GATEWAY_BATCHES_ENABLED"); +export function shouldForwardBatchSpillover(request, env) { + return batchesEnabled(env) && batchFlag(env, "BATCH_SPILLOVER_ENABLED") && request.method === "POST" + && new URL(request.url).pathname === "/v1/chat/completions" + && request.headers.get("X-MultiLLM-Priority")?.trim().toLowerCase() === "batch" + && (request.headers.get("Prefer") ?? "").split(",").some(v => v.trim().toLowerCase() === "respond-async"); +} + +async function schema(env) { + if (!env.INTELLIGENCE_DB || !env.multillm_media) throw Error("batch_storage_missing"); + for (const table of ["gateway_batch_files", "gateway_batches", "gateway_batch_items"]) { + await env.INTELLIGENCE_DB.prepare(`SELECT 1 FROM ${table} LIMIT 1`).first(); + } +} + +async function batchObject(db, batch) { + const { results } = await db.prepare("SELECT status,COUNT(*) AS n FROM gateway_batch_items WHERE batch_id=? GROUP BY status").bind(batch.id).all(); + const counts = { total: 0, completed: 0, failed: 0 }; + for (const item of results) { + counts.total += item.n; + if (item.status === "completed") counts.completed += item.n; + if (item.status === "failed") counts.failed += item.n; + } + const result = { id: batch.id, object: "batch", endpoint: batch.endpoint, + errors: null, input_file_id: batch.input_file_id, completion_window: "24h", status: batch.status, + output_file_id: batch.output_file_id ?? null, error_file_id: batch.error_file_id ?? null, + created_at: batch.created_at, expires_at: batch.expires_at, request_counts: counts, metadata: JSON.parse(batch.metadata) }; + for (const name of ["in_progress_at", "finalizing_at", "completed_at", "failed_at", "expired_at", "cancelling_at", "cancelled_at"]) result[name] = batch[name] ?? null; + return result; +} + +async function createBatch(body, env, now) { + const db = env.INTELLIGENCE_DB; + if (!ENDPOINTS.has(body.endpoint) || body.completion_window !== "24h" || !record(body.metadata) + || Object.keys(body.metadata).length > 16 || Object.entries(body.metadata).some(([k,v]) => k.length > 64 || typeof v !== "string" || v.length > 512) + || !units(body.budget_units) || body.budget_units === 0 || body.budget_units > 1e15 + || typeof body.principal !== "string" || body.principal.length > 2048 + || ![body.client_ip, body.key_hash, body.key_prefix].every(v => typeof v === "string" && v.length <= 128)) { + return failure("invalid_batch", "Send an endpoint, 24h window, metadata budget and valid owner principal."); + } + const file = await ownedFile(db, body.owner, body.input_file_id ?? ""); + if (!file || file.purpose !== "batch") return failure("not_found", "Input file not found.", 404); + const items = await inputItems(env, file); + if (!Array.isArray(body.estimates) || body.estimates.length !== items.length + || items.some((v,i) => v.url !== body.endpoint || body.estimates[i]?.custom_id !== v.custom_id + || !units(body.estimates[i]?.estimate_units) || body.estimates[i].estimate_units > 1e15)) { + return failure("invalid_batch", "Every item must match the endpoint and have a known bounded price."); + } + const id = body.id ?? newId("batch"); + if (!/^batch_[a-f0-9]{32}$/.test(id)) return failure("invalid_batch", "Invalid batch id."); + const statements = [db.prepare(`INSERT INTO gateway_batches + (id,owner,input_file_id,endpoint,status,metadata,principal,client_ip,key_hash,key_prefix,budget_units,created_at,expires_at) + VALUES (?,?,?,?,'validating',?,?,?,?,?,?,?,?)`) + .bind(id, body.owner, file.id, body.endpoint, JSON.stringify(body.metadata), body.principal, + body.client_ip, body.key_hash, body.key_prefix, body.budget_units, now, now + 86400)]; + // D1 binds at most 100 values per statement; a single batch is transactional. + for (let start = 0; start < items.length; start += 16) { + const page = items.slice(start, start + 16); + statements.push(db.prepare(`INSERT INTO gateway_batch_items (batch_id,idx,owner,custom_id,estimate_units) VALUES + ${page.map(() => "(?,?,?,?,?)").join(",")}`) + .bind(...page.flatMap((v,i) => [id, start + i, body.owner, v.custom_id, body.estimates[start + i].estimate_units]))); + } + await db.batch(statements); + return reply({ batch: await batchObject(db, await row(db, id)) }); +} + +async function cancelBatch(body, env, now) { + const db = env.INTELLIGENCE_DB, batch = await owned(db, body); + if (!batch) return failure("not_found", "Batch not found.", 404); + await db.batch([ + db.prepare(`UPDATE gateway_batches SET status='cancelling',cancelling_at=COALESCE(cancelling_at,?),terminal_status='cancelled' + WHERE id=? AND status IN ('validating','in_progress')`).bind(now, batch.id), + db.prepare(`UPDATE gateway_batch_items SET status='failed',error_code='cancelled' + WHERE batch_id=? AND status='queued' AND EXISTS (SELECT 1 FROM gateway_batches WHERE id=? AND status='cancelling')`).bind(batch.id, batch.id), + ]); + return reply({ batch: await batchObject(db, await row(db, batch.id)) }); +} + +export async function handleBatchRequest(request, env, { now = nowSeconds } = {}) { + if (new URL(request.url).origin !== "http://intelligence.internal" + || new URL(request.url).pathname !== "/v1/gateway-batches" || request.method !== "POST" || !batchesEnabled(env)) { + return failure("not_found", "Not found.", 404); + } + try { + await schema(env); + const body = await readBatchBody(request); + if (!body || body.version !== 1 || typeof body.owner !== "string" || !/^[^\x00-\x1f\x7f]{1,256}$/.test(body.owner)) return failure("invalid_request", "Invalid batch operation."); + if (body.operation?.startsWith("file_")) return await handleFileOperation(body, env, now()); + if (body.operation === "batch_create") return await createBatch(body, env, now()); + if (body.operation === "batch_cancel") return await cancelBatch(body, env, now()); + const db = env.INTELLIGENCE_DB; + if (body.operation === "item_start") { + const batch = await owned(db, body); + if (!batch || !Number.isSafeInteger(body.idx) || typeof body.lease_token !== "string") return failure("not_found", "Claim not found.", 404); + const updated = await db.prepare(`UPDATE gateway_batch_items SET status='executing' + WHERE batch_id=? AND idx=? AND status='dispatched' AND lease_token=? AND lease_until>?`) + .bind(batch.id, body.idx, body.lease_token, now()).run(); + if (!updated.meta?.changes) return failure("claim_rejected", "The item claim was already used or expired.", 409); + const file = await ownedFile(db, batch.owner, batch.input_file_id); + const items = await inputItems(env, file); + const stored = await db.prepare("SELECT estimate_units FROM gateway_batch_items WHERE batch_id=? AND idx=?").bind(batch.id, body.idx).first(); + return reply({ item: { ...items[body.idx], estimate_units: stored.estimate_units }, + batch: { id: batch.id, client_ip: batch.client_ip, key_hash: batch.key_hash, key_prefix: batch.key_prefix } }); + } + if (body.operation === "batch_list") { + const limit = pageLimit(body.limit); + const { results } = await db.prepare("SELECT * FROM gateway_batches WHERE owner=? AND id>? ORDER BY id LIMIT ?") + .bind(body.owner, body.after ?? "", limit + 1).all(); + return reply(listObject(await Promise.all(results.map(v => batchObject(db, v))), limit)); + } + if (body.operation === "batch_get") { + const batch = await owned(db, body); + return batch ? reply({ batch: await batchObject(db, batch) }) : failure("not_found", "Batch not found.", 404); + } + return failure("invalid_operation", "Unknown batch operation."); + } catch { + return failure("gateway_batches_unavailable", "Batch storage is unavailable; check the media bucket and apply the gateway batch D1 migration.", 503); + } +} + +async function recover(db, now) { + await db.batch([ + db.prepare(`UPDATE gateway_batch_items SET status='failed',error_code='outcome_unknown' + WHERE status IN ('dispatched','executing') AND lease_until<=?`).bind(now), + db.prepare(`UPDATE gateway_batches SET terminal_status='expired',expired_at=? + WHERE expires_at<=? AND status IN ('validating','in_progress')`).bind(now, now), + db.prepare(`UPDATE gateway_batch_items SET status='failed',error_code='expired' WHERE status='queued' + AND batch_id IN (SELECT id FROM gateway_batches WHERE terminal_status='expired')`), + ]); +} + +async function claim(db, batch, index, token, now) { + // Claim and hold are one CAS. Concurrent schedules include all other measured/held spend. + const result = await db.prepare(`UPDATE gateway_batch_items SET status='dispatched',lease_token=?,lease_until=?,held_units=estimate_units + WHERE batch_id=? AND idx=? AND status='queued' AND EXISTS + (SELECT 1 FROM gateway_batches b WHERE b.id=? AND b.status IN ('validating','in_progress') + AND b.terminal_status IS NULL AND b.expires_at>? AND b.budget_units >= gateway_batch_items.estimate_units + + (SELECT COALESCE(SUM(cost_units+held_units),0) FROM gateway_batch_items WHERE batch_id=b.id) + AND (b.budget_units > (SELECT COALESCE(SUM(cost_units+held_units),0) FROM gateway_batch_items WHERE batch_id=b.id)))`) + .bind(token, now + 40, batch.id, index, batch.id, now).run(); + if (!result.meta?.changes) return false; + await db.prepare(`UPDATE gateway_batches SET status='in_progress',in_progress_at=COALESCE(in_progress_at,?) + WHERE id=? AND status='validating'`).bind(now, batch.id).run(); + return true; +} + +async function exhaustBudget(db, id) { + await db.prepare(`UPDATE gateway_batch_items SET status='failed',error_code='budget_exhausted' + WHERE batch_id=? AND status='queued' AND EXISTS (SELECT 1 FROM gateway_batches b WHERE b.id=? AND + (b.budget_units <= (SELECT COALESCE(SUM(cost_units+held_units),0) FROM gateway_batch_items WHERE batch_id=b.id) + OR b.budget_units < gateway_batch_items.estimate_units + + (SELECT COALESCE(SUM(cost_units+held_units),0) FROM gateway_batch_items WHERE batch_id=b.id)))`).bind(id, id).run(); +} + +export async function containerBatchDispatch(item, batch, container, signal) { + if (!container) throw Error("batch_container_unavailable"); + const response = await container.fetch(new Request("http://container/internal/gateway/batch-item", { + method: "POST", headers: { "content-type": "application/json", Authorization: `BatchPrincipal ${batch.principal}` }, + body: JSON.stringify({ batch_id: batch.id, idx: item.idx, lease_token: item.lease_token }), signal, + })); + const bytes = await boundedBytes(response, MAX_RESULT_BYTES + 8192); + if (!response.ok || !bytes) throw Error("batch_dispatch_unknown"); + const outcome = JSON.parse(new TextDecoder().decode(bytes)); + if (!record(outcome) || !Number.isInteger(outcome.status_code) || !record(outcome.body) + || typeof outcome.ambiguous !== "boolean" || (outcome.cost_units !== null && !units(outcome.cost_units))) throw Error("batch_dispatch_unknown"); + return outcome; +} + +async function execute(env, batch, stored, item, dispatch, container, now) { + let outcome; + const controller = new AbortController(); + let timer; + try { + outcome = await Promise.race([ + dispatch({ ...item, idx: stored.idx, lease_token: stored.lease_token, estimate_units: stored.estimate_units }, batch, container, controller.signal), + new Promise((_, reject) => { timer = setTimeout(() => { controller.abort(); reject(Error("batch_deadline")); }, 30000); }), + ]); + } catch { outcome = { status_code: 504, body: {}, ambiguous: true, cost_units: null }; } + finally { clearTimeout(timer); } + if (!record(outcome) || !record(outcome.body) || !Number.isInteger(outcome.status_code) + || (outcome.cost_units !== null && !units(outcome.cost_units))) outcome = { status_code: 502, body: {}, ambiguous: true, cost_units: null }; + const unknown = outcome.ambiguous || outcome.cost_units === null; + const success = !unknown && outcome.status_code === 200 && !outcome.body.error; + const code = unknown ? "outcome_unknown" : success ? null : outcome.error_code ?? "request_failed"; + const result = { id: newId("batch_req"), custom_id: item.custom_id, + response: unknown ? null : { status_code: outcome.status_code, request_id: outcome.request_id ?? null, body: outcome.body }, + error: success ? null : { code, message: unknown ? "The provider outcome is unknown; this item will not be retried." : "The batch item failed." } }; + const content = JSON.stringify(result) + "\n"; + const key = `batches/results/${batch.id}/${stored.idx}/${stored.lease_token}`; + if (new TextEncoder().encode(content).byteLength > MAX_RESULT_BYTES) { + // A billed oversized result is not retried; the hold remains if usage was not measured. + result.response = null; result.error = { code: "result_too_large", message: "Batch result exceeds 1 MiB." }; + } + await env.multillm_media.put(key, JSON.stringify(result) + "\n"); + await env.INTELLIGENCE_DB.prepare(`UPDATE gateway_batch_items SET status=?,result_key=?,error_code=?,cost_units=?,held_units=? + WHERE batch_id=? AND idx=? AND status IN ('dispatched','executing') AND lease_token=? AND lease_until>?`) + .bind(success && !result.error ? "completed" : "failed", key, result.error?.code ?? null, + unknown ? 0 : outcome.cost_units, unknown ? stored.estimate_units : 0, + batch.id, stored.idx, stored.lease_token, now()).run(); +} + +function overflowResult(item) { + return JSON.stringify({ id: newId("batch_req"), custom_id: item.custom_id, response: null, + error: { code: "output_file_too_large", message: "Aggregate result content exceeds its size bound." } }) + "\n"; +} + +async function appendFinalResult(env, batch, item, state) { + let content; + if (item.error_code === "output_file_too_large") content = overflowResult(item); + else if (item.result_key) { + const object = await env.multillm_media.get(item.result_key); + if (!object) throw Error("batch_result_missing"); + content = await object.text(); + } else content = JSON.stringify({ id: newId("batch_req"), custom_id: item.custom_id, response: null, + error: { code: item.error_code, message: "The batch item was not completed." } }) + "\n"; + const side = item.status === "completed" ? "output" : "errors"; + const counter = side === "output" ? "outputBytes" : "errorBytes"; + const size = new TextEncoder().encode(content).byteLength; + // Reserve room for every remaining item's compact error before large errors. + const capacity = side === "errors" ? MAX_OUTPUT_BYTES - 1000 * 1024 : MAX_OUTPUT_BYTES; + if (state[counter] + size > capacity) { + const compact = overflowResult(item); + state.errors += compact; + state.errorBytes += new TextEncoder().encode(compact).byteLength; + await env.INTELLIGENCE_DB.prepare("UPDATE gateway_batch_items SET status='failed',error_code='output_file_too_large' WHERE batch_id=? AND idx=?") + .bind(batch.id, item.idx).run(); + } else { state[side] += content; state[counter] += size; } +} + +async function finalization(env, batch, now, clock, deadline) { + const db = env.INTELLIGENCE_DB; + const active = await db.prepare("SELECT 1 FROM gateway_batch_items WHERE batch_id=? AND status IN ('queued','dispatched','executing') LIMIT 1").bind(batch.id).first(); + if (active || clock() >= deadline) return; + const token = newId("lease"); + const claimed = await db.prepare(`UPDATE gateway_batches SET status='finalizing',finalizing_at=COALESCE(finalizing_at,?),lease_token=?,lease_until=? + WHERE id=? AND status IN ${ACTIVE} AND (lease_until IS NULL OR lease_until<=?)`).bind(now(), token, now() + 90, batch.id, now()).run(); + if (!claimed.meta?.changes) return; + batch = await row(db, batch.id); + let state = { output: "", errors: "" }, cursor = batch.finalize_cursor; + if (batch.checkpoint) { + const object = await env.multillm_media.get(batch.checkpoint); + if (!object) throw Error("batch_checkpoint_missing"); + state = JSON.parse(await object.text()); + } + state.outputBytes = new TextEncoder().encode(state.output).byteLength; + state.errorBytes = new TextEncoder().encode(state.errors).byteLength; + // Bounded pages and a durable checkpoint allow finalization to continue next cron. + let done = false; + for (let page = 0; page < 10 && clock() < deadline; page++) { + const { results } = await db.prepare("SELECT * FROM gateway_batch_items WHERE batch_id=? AND idx>=? ORDER BY idx LIMIT 100").bind(batch.id, cursor).all(); + if (!results.length) { done = true; break; } + for (const item of results) { + if (clock() >= deadline) break; + await appendFinalResult(env, batch, item, state); + cursor = item.idx + 1; + } + } + if (!done) { + const remaining = await db.prepare("SELECT 1 FROM gateway_batch_items WHERE batch_id=? AND idx>=? LIMIT 1").bind(batch.id, cursor).first(); + done = !remaining; + } + if (!done) { + const key = `batches/checkpoints/${batch.id}/${token}`; + await env.multillm_media.put(key, JSON.stringify(state)); + await db.prepare("UPDATE gateway_batches SET checkpoint=?,finalize_cursor=?,lease_until=NULL WHERE id=? AND lease_token=?").bind(key, cursor, batch.id, token).run(); + return; + } + let output = null, errors = null; + if (state.output) output = await storeFile(env, batch.owner, `${batch.id}-output.jsonl`, state.output, "batch_output", now()); + if (state.errors) errors = await storeFile(env, batch.owner, `${batch.id}-errors.jsonl`, state.errors, "batch_output", now()); + const status = batch.terminal_status ?? "completed"; + await db.prepare(`UPDATE gateway_batches SET status=?,output_file_id=?,error_file_id=?,completed_at=?, + cancelled_at=CASE WHEN ?='cancelled' THEN ? ELSE NULL END,lease_until=NULL WHERE id=? AND lease_token=?`) + .bind(status, output?.id ?? null, errors?.id ?? null, now(), status, now(), batch.id, token).run(); +} + +export async function runScheduledBatches(env, container, { now = nowSeconds, clock = () => performance.now(), + dispatch = containerBatchDispatch } = {}) { + if (!batchesEnabled(env)) return { claimed: 0 }; + const deadline = clock() + 65000, db = env.INTELLIGENCE_DB; + await schema(env); + await recover(db, now()); + const { results: batches } = await db.prepare(`SELECT * FROM gateway_batches WHERE status IN ${ACTIVE} ORDER BY created_at,id LIMIT 20`).all(); + let claimed = 0; + for (const batch of batches) { + if (clock() >= deadline) break; + if (["validating", "in_progress"].includes(batch.status) && !batch.terminal_status && batch.expires_at > now()) { + const file = await ownedFile(db, batch.owner, batch.input_file_id); + const items = file ? await inputItems(env, file) : null; + if (!items) throw Error("batch_input_missing"); + const { results } = await db.prepare("SELECT * FROM gateway_batch_items WHERE batch_id=? AND status='queued' ORDER BY idx LIMIT 1000").bind(batch.id).all(); + for (const item of results) { + if (claimed >= 2 || clock() + 30000 > deadline) break; + const token = newId("lease"); + if (!await claim(db, batch, item.idx, token, now())) continue; + claimed++; + await execute(env, batch, { ...item, lease_token: token }, items[item.idx], dispatch, container, now); + } + await exhaustBudget(db, batch.id); + } + await finalization(env, await row(db, batch.id), now, clock, deadline); + } + return { claimed }; +} diff --git a/worker/container-env.mjs b/worker/container-env.mjs index 1a7994db..2d06acb6 100644 --- a/worker/container-env.mjs +++ b/worker/container-env.mjs @@ -40,6 +40,8 @@ const DIRECT_ENV_KEYS = [ "CONNECTION_PROFILES_DB_PATH", "CONFIG_REVISION_SYNC_ENABLED", "CONFIG_SYNC_TTL_SECONDS", + "GATEWAY_BATCHES_ENABLED", + "BATCH_SPILLOVER_ENABLED", "GATEWAY_ALERTS_ENABLED", "GATEWAY_ALERT_WEBHOOK_ALLOWLIST", "CONFIG_SECURITY_TTL_SECONDS", diff --git a/worker/cors-policy.mjs b/worker/cors-policy.mjs index 4e5b33fb..cc67bdcc 100644 --- a/worker/cors-policy.mjs +++ b/worker/cors-policy.mjs @@ -1,6 +1,6 @@ export const CORS_ALLOWED_METHODS = "GET, POST, PUT, DELETE, PATCH, OPTIONS"; export const CORS_DEFAULT_HEADERS = - "X-MultiLLM-Tool-Repair, X-MultiLLM-Image-QA, X-MultiLLM-Cascade, X-MultiLLM-Conversion-Report, X-MultiLLM-Retention, X-MultiLLM-Deadline-Ms, Authorization, X-Api-Key, X-Goog-Api-Key, X-MultiLLM-Api-Key, X-Roleplay-Session-ID, Anthropic-Version, Anthropic-Beta, Anthropic-Dangerous-Direct-Browser-Access, Content-Type, Accept, Origin, X-Requested-With, OpenAI-Beta, OpenAI-Organization, OpenAI-Project, Idempotency-Key, Moderation, Moderation-Model, Redaction, X-Client-Request-ID, X-App-Name, X-Billing-Mode, X-BYOK-Provider, X-Encryption-Key, X-Encryption-Passphrase, X-Fal-Object-Lifecycle-Preference, X-PAYMENT, X-Prompt-Caching-Cut-After, X-Provider, X-Team-ID, X-Use-BYOK, x-x402"; + "X-MultiLLM-Tool-Repair, X-MultiLLM-Image-QA, X-MultiLLM-Cascade, X-MultiLLM-Conversion-Report, X-MultiLLM-Retention, X-MultiLLM-Deadline-Ms, Authorization, X-Api-Key, X-Goog-Api-Key, X-MultiLLM-Api-Key, X-Roleplay-Session-ID, Anthropic-Version, Anthropic-Beta, Anthropic-Dangerous-Direct-Browser-Access, Content-Type, Accept, Origin, X-Requested-With, OpenAI-Beta, OpenAI-Organization, OpenAI-Project, Idempotency-Key, X-MultiLLM-Priority, Prefer, Moderation, Moderation-Model, Redaction, X-Client-Request-ID, X-App-Name, X-Billing-Mode, X-BYOK-Provider, X-Encryption-Key, X-Encryption-Passphrase, X-Fal-Object-Lifecycle-Preference, X-PAYMENT, X-Prompt-Caching-Cut-After, X-Provider, X-Team-ID, X-Use-BYOK, x-x402"; export const CORS_EXPOSE_HEADERS = "X-MultiLLM-Secret-Scan, X-MultiLLM-Retention, X-MultiLLM-Roleplay-Recovery, " + - "X-MultiLLM-Tool-Repair, X-MultiLLM-Image-QA, X-MultiLLM-Cascade, X-MultiLLM-Conversion-Fidelity, X-MultiLLM-Conversion-Fields, Retry-After, X-Request-ID, X-MultiLLM-RateLimit-Limit, X-MultiLLM-RateLimit-Remaining, X-MultiLLM-RateLimit-Reset, X-MultiLLM-Idempotency, X-MultiLLM-Optimization, X-MultiLLM-Optimization-Mode, X-MultiLLM-Estimated-Input-Before, X-MultiLLM-Estimated-Input-After, X-MultiLLM-Image-Prompts-Compacted, X-MultiLLM-Messages-Summarized, X-MultiLLM-Optimization-Target-Met, X-MultiLLM-Summary, X-MultiLLM-Optimization-Cache-Hits, X-MultiLLM-Optimization-Cache-Misses, X-MultiLLM-Prompt-Cache, X-MultiLLM-Prompt-Cache-Mode, X-MultiLLM-Prompt-Cache-Estimated-Tokens, X-MultiLLM-Provider, X-MultiLLM-Model, X-MultiLLM-Credential-Attempts, X-MultiLLM-Route-Decision, X-MultiLLM-Circuit-State, X-MultiLLM-Auto-Route, X-MultiLLM-Auto-Selected-Model, X-MultiLLM-Auto-Attempts, X-MultiLLM-Auto-Selected-Priority, X-MultiLLM-Auto-Ordering, X-MultiLLM-Auto-Failover-Reasons, X-MultiLLM-Cache, X-MultiLLM-Cache-Backend, X-MultiLLM-Usage-Basis, X-MultiLLM-Provider-Calls, Age, X-MultiLLM-Transport-Failure, X-MultiLLM-Token-Count, X-MultiLLM-Latency-Ms, X-MultiLLM-Estimated-Cost-USD, X-MultiLLM-Cost-Basis, X-MultiLLM-Reasoning-Normalized, X-Roleplay-Session-ID, X-Roleplay-Trace-ID, X-Roleplay-Session-Source, X-Roleplay-Provider, X-Roleplay-Model, X-Roleplay-Selection, X-Roleplay-Memory, X-Roleplay-Estimated-Input-Tokens, X-Roleplay-Max-Output-Tokens, X-Roleplay-Fallback-Count, X-Roleplay-State-Cache, X-Roleplay-Credential-Check, Server-Timing, WWW-Authenticate, X-PAYMENT-RESPONSE, X-Poll-After, X-NanoGPT-Advisor-ID, X-NanoGPT-Data-Endpoint, X-NanoGPT-Direct-Endpoint, X-NanoGPT-Inline-Moderation-Cost-USD, X-NanoGPT-Inline-Moderation-Flagged, X-NanoGPT-Inline-Moderation-Model"; + "X-MultiLLM-Tool-Repair, X-MultiLLM-Image-QA, X-MultiLLM-Cascade, X-MultiLLM-Conversion-Fidelity, X-MultiLLM-Conversion-Fields, Retry-After, Location, X-Request-ID, X-MultiLLM-RateLimit-Limit, X-MultiLLM-RateLimit-Remaining, X-MultiLLM-RateLimit-Reset, X-MultiLLM-Idempotency, X-MultiLLM-Optimization, X-MultiLLM-Optimization-Mode, X-MultiLLM-Estimated-Input-Before, X-MultiLLM-Estimated-Input-After, X-MultiLLM-Image-Prompts-Compacted, X-MultiLLM-Messages-Summarized, X-MultiLLM-Optimization-Target-Met, X-MultiLLM-Summary, X-MultiLLM-Optimization-Cache-Hits, X-MultiLLM-Optimization-Cache-Misses, X-MultiLLM-Prompt-Cache, X-MultiLLM-Prompt-Cache-Mode, X-MultiLLM-Prompt-Cache-Estimated-Tokens, X-MultiLLM-Provider, X-MultiLLM-Model, X-MultiLLM-Credential-Attempts, X-MultiLLM-Route-Decision, X-MultiLLM-Circuit-State, X-MultiLLM-Auto-Route, X-MultiLLM-Auto-Selected-Model, X-MultiLLM-Auto-Attempts, X-MultiLLM-Auto-Selected-Priority, X-MultiLLM-Auto-Ordering, X-MultiLLM-Auto-Failover-Reasons, X-MultiLLM-Cache, X-MultiLLM-Cache-Backend, X-MultiLLM-Usage-Basis, X-MultiLLM-Provider-Calls, Age, X-MultiLLM-Transport-Failure, X-MultiLLM-Token-Count, X-MultiLLM-Latency-Ms, X-MultiLLM-Estimated-Cost-USD, X-MultiLLM-Cost-Basis, X-MultiLLM-Reasoning-Normalized, X-Roleplay-Session-ID, X-Roleplay-Trace-ID, X-Roleplay-Session-Source, X-Roleplay-Provider, X-Roleplay-Model, X-Roleplay-Selection, X-Roleplay-Memory, X-Roleplay-Estimated-Input-Tokens, X-Roleplay-Max-Output-Tokens, X-Roleplay-Fallback-Count, X-Roleplay-State-Cache, X-Roleplay-Credential-Check, Server-Timing, WWW-Authenticate, X-PAYMENT-RESPONSE, X-Poll-After, X-NanoGPT-Advisor-ID, X-NanoGPT-Data-Endpoint, X-NanoGPT-Direct-Endpoint, X-NanoGPT-Inline-Moderation-Cost-USD, X-NanoGPT-Inline-Moderation-Flagged, X-NanoGPT-Inline-Moderation-Model"; diff --git a/worker/d1-schema.mjs b/worker/d1-schema.mjs index 366c2e78..fe8959de 100644 --- a/worker/d1-schema.mjs +++ b/worker/d1-schema.mjs @@ -10,7 +10,7 @@ export const REQUIRED_D1_TABLES = Object.freeze(["intelligence_policy", "intelli "usage_events", "usage_daily", "usage_batches", "media_jobs", "media_job_items", "shadow_eval_samples", "shadow_eval_results", "shadow_eval_config", "shadow_eval_policy_backups", "prompt_templates", - "config_snapshot_revisions", "config_snapshots", "config_snapshot_applications", "control_revisions", "tool_grants", "generation_cache", "usage_reservation_budgets", "usage_reservations", "usage_reservation_transitions", "session_tiers", "gateway_alert_rules", "gateway_alert_events", "managed_idempotency"]); + "config_snapshot_revisions", "config_snapshots", "config_snapshot_applications", "control_revisions", "tool_grants", "generation_cache", "usage_reservation_budgets", "usage_reservations", "usage_reservation_transitions", "session_tiers", "gateway_alert_rules", "gateway_alert_events", "managed_idempotency", "gateway_batch_files", "gateway_batches", "gateway_batch_items"]); /** * Readiness for the D1 schema. A deploy that skipped `wrangler d1 migrations apply` reports diff --git a/worker/intelligence-outbound.mjs b/worker/intelligence-outbound.mjs index 0e80edd3..8c324469 100644 --- a/worker/intelligence-outbound.mjs +++ b/worker/intelligence-outbound.mjs @@ -12,6 +12,7 @@ import { handleUsageLedgerRequest } from "./usage-ledger-d1.mjs"; import { handleShadowEvalRequest } from "./shadow-eval-d1.mjs"; import { handleMediaJobsRequest } from "./media-jobs.mjs"; import { handleReservationsRequest } from "./reservations-d1.mjs"; +import { handleBatchRequest } from "./batch-jobs.mjs"; import { handleManagedStateRequest } from "./managed-state-dispatch.mjs"; /** Domain operations reachable only through the container's private outbound handler. */ @@ -25,6 +26,7 @@ export function handleIntelligenceOutbound(request, env) { { status: 405, headers: { Allow: "POST" } }); } if (url.pathname === "/v1/reservations") return handleReservationsRequest(request, env); + if (url.pathname === "/v1/gateway-batches") return handleBatchRequest(request, env); if (url.pathname === "/v1/admission") return handleAdmissionRequest(request, env); if (url.pathname === "/v1/store") return handleIntelligenceStoreRequest(request, env); if (url.pathname === "/v1/auth") return handleIntelligenceAuthRequest(request, env);