From f3499b27644eb73269971651ef6c74ca1fa7ef58 Mon Sep 17 00:00:00 2001 From: Abhyudaya Baiju Tharakan Date: Fri, 9 Oct 2026 05:59:06 +0530 Subject: [PATCH] feat(idempotency): add opt-in managed request idempotency Accept Idempotency-Key on managed nonstreaming chat behind MANAGED_IDEMPOTENCY_ENABLED (default off). An atomic D1 claim (migration 0023) precedes admission and dispatch, binds the key to the principal, path, normalized body and routing policy, and records the provider handoff durably. Concurrent duplicates get 409 request_in_progress, a changed request under the same key gets 422, and a completed response replays from R2 with X-MultiLLM-Idempotency: replayed and no new provider call. With the flag off, the intelligence gateway keeps rejecting the header and raw provider routes are unchanged. --- docs/managed-idempotency.md | 72 ++++ intelligence-migrations/0023_idempotency.sql | 15 + middleware/idempotency.py | 192 +++++++++++ package.json | 2 +- route_helpers.py | 2 +- routes/intelligence.py | 19 +- services/idempotency_store.py | 201 +++++++++++ tests/fixtures/secret_firewall_coverage.json | 2 + tests/test_managed_idempotency.py | 345 +++++++++++++++++++ tests/test_managed_idempotency_worker.mjs | 166 +++++++++ worker/container-env.mjs | 1 + worker/cors-policy.mjs | 2 +- worker/d1-schema.mjs | 2 +- worker/idempotency-d1.mjs | 133 +++++++ worker/intelligence-outbound.mjs | 2 + worker/managed-state-dispatch.mjs | 14 + 16 files changed, 1163 insertions(+), 7 deletions(-) create mode 100644 docs/managed-idempotency.md create mode 100644 intelligence-migrations/0023_idempotency.sql create mode 100644 middleware/idempotency.py create mode 100644 services/idempotency_store.py create mode 100644 tests/test_managed_idempotency.py create mode 100644 tests/test_managed_idempotency_worker.mjs create mode 100644 worker/idempotency-d1.mjs create mode 100644 worker/managed-state-dispatch.mjs diff --git a/docs/managed-idempotency.md b/docs/managed-idempotency.md new file mode 100644 index 00000000..1b039997 --- /dev/null +++ b/docs/managed-idempotency.md @@ -0,0 +1,72 @@ +# Managed request idempotency + +`MANAGED_IDEMPOTENCY_ENABLED` defaults to `false`. Empty values retain that default; +malformed values disable the feature and emit one warning without the value. +Without an `Idempotency-Key`, existing requests and responses are unchanged. +Raw provider routes bypass idempotency claiming and keep their existing header +forwarding rules: raw OpenAI chat drops `Idempotency-Key`; providers whose header +allowlists support it submit the caller's key unchanged once. When disabled, +the intelligence gateway keeps its existing `idempotency_not_supported` rejection. + +Enable the setting in the Worker and Container after applying +`intelligence-migrations/0023_idempotency.sql`. The private authority requires +`INTELLIGENCE_DB` and the existing `multillm_media` R2 binding. Missing migrations, +bindings or private authority return HTTP 503 `idempotency_store_unavailable` +before generation. There is no process-local fallback. + +For a managed nonstreaming chat request, send a stable key: + +```http +POST /intelligence/v1/chat/completions +Idempotency-Key: order-123 +Content-Type: application/json + +{"messages":[{"role":"user","content":"Summarize this order."}],"stream":false} +``` + +Keys contain 1 to 128 ASCII letters, digits or `._:/+@-`. The identity is scoped to +the authenticated principal, HTTP method and path. A normalized JSON digest binds +the body, requested model, routing policy and retention revision to that key. +Object key ordering and insignificant whitespace do not change the digest. +Changing the request or policy under a claimed key returns HTTP 422 +`idempotency_key_conflict`. Different principals and paths have separate claims. + +An atomic claim precedes admission and dispatch. Concurrent identical submissions +receive HTTP 409 `request_in_progress` with `Retry-After: 1` while the owner runs. +A completed response replays the stored status, body bytes and bounded content +headers, with `X-MultiLLM-Idempotency: replayed`. Replay does not reserve usage or +call a provider. Streaming keyed requests return HTTP 400 +`idempotency_stream_unsupported` before claiming or dispatching. + +A timeout, abandoned request, failed completion write or lost outcome blocks the +key. Later submissions receive HTTP 409 `outcome_unknown`; they never start another +generation. A dispatch acknowledgement is recorded before provider handoff. Claims +that remain unfinished for one hour become unknown when accessed. Existing +provider refusal/fallback rules are unchanged; idempotency grants no new retry +permission. An unknown outcome can still incur provider charges and retain the +existing usage hold. A new key can create a separate billable generation and is +not a safe way to recover an unknown outcome. + +Responses are bounded to 1 MiB, with at most 32 selected headers and 16 KiB of header +metadata. Response bytes live under `managed-idempotency/` in R2; D1 contains only +hashed identity, ownership, state, expiry and an integrity-checked object pointer. +Successful responses above the bound are returned normally but cannot be replayed; +the key becomes unknown. Errors and partial results are never stored as successes. +The zero-content retention policy also prevents response storage and replay. + +Replay content expires 24 hours after claim. Expiry, missing objects or failed +integrity checks block replay rather than authorizing another generation. Hashed +claim tombstones remain to prevent ambiguous key reuse. Expired objects are removed +when their key is accessed; operators should also configure an R2 lifecycle rule for +this prefix to remove expired and orphaned content. Expiry does not erase usage +records or release unknown usage holds. No prompt or response content is logged. + +The Flask registration point is `middleware.idempotency.register_idempotency`. +Register its named `idempotency_request_hook` after authentication, retention and +secret policy, and before admission. Intelligence dispatch finalizes only after +validated nonstream completion. Other managed chat dispatchers supply their resolved +policy through `policy_revision` and call `dispatch_with_idempotency` with an explicit +completion classifier. The optional `settlement` callback receives classified state, +without response content or permission to generate. The private managed-state +dispatcher uses fixed domain paths and explicitly supplied reservation/tool-grant +handlers; unregistered domains return 404. diff --git a/intelligence-migrations/0023_idempotency.sql b/intelligence-migrations/0023_idempotency.sql new file mode 100644 index 00000000..72726308 --- /dev/null +++ b/intelligence-migrations/0023_idempotency.sql @@ -0,0 +1,15 @@ +-- Durable claims remain as tombstones when replay content expires. +CREATE TABLE IF NOT EXISTS managed_idempotency ( + scope TEXT PRIMARY KEY, + request_digest TEXT NOT NULL, + owner TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('pending', 'completed', 'unknown')), + handed_off INTEGER NOT NULL DEFAULT 0 CHECK (handed_off IN (0, 1)), + created_at INTEGER NOT NULL, + pending_until INTEGER NOT NULL, + expires_at INTEGER NOT NULL, + response_key TEXT, + response_digest TEXT +); +CREATE INDEX IF NOT EXISTS managed_idempotency_expiry + ON managed_idempotency (status, expires_at); diff --git a/middleware/idempotency.py b/middleware/idempotency.py new file mode 100644 index 00000000..d6edc78f --- /dev/null +++ b/middleware/idempotency.py @@ -0,0 +1,192 @@ +"""Named post-authentication idempotency hook and explicit managed dispatch boundary.""" +from __future__ import annotations + +import logging + +from flask import Response, current_app, g, request + +from services.idempotency_store import (IdempotencyStore, decode_response, encode_response, + idempotency_enabled, request_fingerprint, unavailable) +from services.intelligence_contract import GatewayError + +logger = logging.getLogger(__name__) + + +def _error_response(error): + from routes.intelligence import error_response + return error_response(error) + + +def _settle(claim, state): + callback = current_app.extensions.get("managed_idempotency_settlement") + if callback is not None: + try: + callback(claim, state) + except Exception: + logger.warning("Managed idempotency settlement collaborator unavailable") + + +def _status_response(claim): + if claim.status == "completed": + response = decode_response(claim.response) + _settle(claim, "replayed") + return response + if claim.status == "pending": + raise GatewayError("request_in_progress", "The same managed request is still in progress.", 409, retry_after="1") + if claim.status == "conflict": + raise GatewayError("idempotency_key_conflict", "The key is already bound to a different request or policy revision.", 422) + if claim.status == "unknown": + raise GatewayError("outcome_unknown", "The earlier request may have generated output; it will not be submitted again.", 409) + return None + + +def begin_managed_request(payload, revision): + """Claim once, after auth/policy and before admission or provider work.""" + if not idempotency_enabled() or request.headers.get("Idempotency-Key") is None: + return None + if payload.get("stream"): + raise GatewayError("idempotency_stream_unsupported", "Streaming managed responses do not support idempotency replay.") + existing = getattr(g, "managed_idempotency_claim", None) + if existing is not None: + return None + from services.retention_policy import request_policy + retention = request_policy() + principal = (getattr(g, "authenticated_user", None) or {}).get("username") + scope, digest = request_fingerprint(principal, request.method, request.path, + request.headers["Idempotency-Key"], payload, + {"policy": revision, "retention": retention.revision, "mode": retention.mode, + "tool_repair": request.headers.get("X-MultiLLM-Tool-Repair", "")}) + store = current_app.extensions.get("managed_idempotency_store") + if store is None: + # A keyed enabled request cannot silently run without the registrar. + raise unavailable() + claim = store.claim(scope, digest) + response = _status_response(claim) + if response is not None: + return response + g.managed_idempotency_claim = claim + g.managed_idempotency_retention = retention + g.managed_idempotency_handed_off = False + g.managed_idempotency_finalized = False + return None + + +def idempotency_request_hook(): + """Static request-identity collaborator registered before admission.""" + if not idempotency_enabled() or request.method != "POST" or request.headers.get("Idempotency-Key") is None: + return None + dedicated = request.path == "/intelligence/v1/chat/completions" + if not dedicated and request.path != "/v1/chat/completions": + return None + payload = request.get_json(silent=True) + if not isinstance(payload, dict): + return None + model = payload.get("model", "") + intelligence = dedicated or model == "auto:intelligence" or "routing" in payload + managed = intelligence or isinstance(model, str) and model.startswith(("auto:", "cascade:")) + if not managed: + return None + payload = dict(payload) + if dedicated: + payload.setdefault("model", "auto:intelligence") + try: + if intelligence: + from routes.intelligence import load_policy + from services.intelligence_contract import ChatRequest + policy = load_policy() + if len(request.get_data(cache=True)) > policy["max_request_bytes"]: + raise GatewayError("request_too_large", "The intelligence request exceeds the size limit.", 413) + try: + ChatRequest.parse(payload, policy) + except (ValueError, TypeError, AttributeError, OverflowError): + raise GatewayError("invalid_routing_request", "The request does not satisfy the version-one chat and routing contract.") from None + g.managed_idempotency_policy = policy + revision = policy + else: + resolver = current_app.extensions.get("managed_idempotency_policy_revision") + if resolver is None: + raise unavailable() + revision = resolver(payload) + if revision is None: + raise unavailable() + return begin_managed_request(payload, revision) + except GatewayError as error: + return _error_response(error) + except Exception: + return _error_response(unavailable()) + + +def mark_managed_handoff(): + claim = getattr(g, "managed_idempotency_claim", None) + if claim is None: + return + if getattr(g, "managed_idempotency_handed_off", False): + raise unavailable() + # Record the dispatch boundary durably before any provider side effect. + g.managed_idempotency_handed_off = True + if not current_app.extensions["managed_idempotency_store"].transition(claim, "handoff"): + raise unavailable() + + +def finish_managed_response(response, *, complete=False): + """The caller supplies positive completion proof; errors never become replays.""" + claim = getattr(g, "managed_idempotency_claim", None) + if claim is None or getattr(g, "managed_idempotency_finalized", False): + return response + g.managed_idempotency_finalized = True + store = current_app.extensions["managed_idempotency_store"] + retention = g.managed_idempotency_retention + document = encode_response(response) if complete and retention.allows_content else None + try: + if document is not None and store.transition(claim, "complete", response=document): + _settle(claim, "completed") + return response + store.transition(claim, "unknown") + except GatewayError: + # A lost completion write cannot grant replay permission. Best-effort + # ambiguity marking leaves the original pending claim blocked on failure. + try: + store.transition(claim, "unknown") + except GatewayError: + logger.warning("Managed idempotency finalization unavailable; request remains blocked") + _settle(claim, "unknown") + return response + + +def dispatch_with_idempotency(dispatch, *, completed=None): + """Inject a managed dispatcher and its completion proof after policy eligibility.""" + try: + mark_managed_handoff() + except GatewayError as error: + return _error_response(error) + try: + response = current_app.make_response(dispatch()) + except BaseException: + finish_managed_response(Response(status=503)) + raise + proof = completed(response) if completed is not None else False + return finish_managed_response(response, complete=proof) + + +def register_idempotency(app, *, store=None, policy_revision=None, settlement=None): + """Mount through the Flask registrar after retention/secret policy, before admission. + + policy_revision supplies the resolved auto/cascade policy. settlement is an + explicitly injected usage collaborator and must never authorize generation. + """ + app.extensions["managed_idempotency_store"] = store or IdempotencyStore() + if policy_revision is not None: + app.extensions["managed_idempotency_policy_revision"] = policy_revision + if settlement is not None: + app.extensions["managed_idempotency_settlement"] = settlement + app.extensions.setdefault("gateway_after_authentication", []).append(idempotency_request_hook) + + @app.after_request + def finalize_managed_failure(response): + # Success is finalized by the explicit dispatch wrapper, not guessed here. + return finish_managed_response(response) + + @app.teardown_request + def abandon_managed_request(error): + if getattr(g, "managed_idempotency_claim", None) is not None and not getattr(g, "managed_idempotency_finalized", False): + finish_managed_response(Response(status=503)) diff --git a/package.json b/package.json index fa5f6aff..e8bbc182 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "lint": "ruff check . --select E9,F63,F7,F82", "typecheck": "mypy --ignore-missing-imports --follow-imports=skip --disable-error-code import-untyped --disable-error-code var-annotated --disable-error-code arg-type --disable-error-code assignment --disable-error-code operator app.py routes services providers streaming", "test": "python -m pytest -q", - "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs", + "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_managed_idempotency_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs", "db:migrate": "python scripts/validate_sqlite_schema.py --apply", "db:validate": "python scripts/validate_sqlite_schema.py", "security:secrets": "python scripts/check_static_secrets.py", diff --git a/route_helpers.py b/route_helpers.py index cd72a0b8..d704dcc4 100644 --- a/route_helpers.py +++ b/route_helpers.py @@ -37,7 +37,7 @@ + ", ".join([*CLIENT_HEADER_NAMES.values(), *OPENCODE_CLIENT_HEADER_NAMES.values()]) ) CORS_EXPOSE_HEADERS = ( - "Retry-After, X-Request-ID, X-MultiLLM-RateLimit-Limit, X-MultiLLM-RateLimit-Remaining, X-MultiLLM-RateLimit-Reset, X-MultiLLM-Optimization, X-MultiLLM-Tool-Repair, X-MultiLLM-Image-QA, X-MultiLLM-Cascade, X-MultiLLM-Conversion-Fidelity, X-MultiLLM-Conversion-Fields, " + "Retry-After, X-Request-ID, X-MultiLLM-RateLimit-Limit, X-MultiLLM-RateLimit-Remaining, X-MultiLLM-RateLimit-Reset, X-MultiLLM-Idempotency, X-MultiLLM-Optimization, X-MultiLLM-Tool-Repair, X-MultiLLM-Image-QA, X-MultiLLM-Cascade, X-MultiLLM-Conversion-Fidelity, X-MultiLLM-Conversion-Fields, " "X-MultiLLM-Optimization-Mode, X-MultiLLM-Estimated-Input-Before, " "X-MultiLLM-Estimated-Input-After, X-MultiLLM-Image-Prompts-Compacted, " "X-MultiLLM-Messages-Summarized, X-MultiLLM-Optimization-Target-Met, " diff --git a/routes/intelligence.py b/routes/intelligence.py index 93b8bbae..562308e3 100644 --- a/routes/intelligence.py +++ b/routes/intelligence.py @@ -11,7 +11,9 @@ from werkzeug.exceptions import RequestEntityTooLarge from error_handlers import get_request_id +from middleware.idempotency import begin_managed_request, finish_managed_response, mark_managed_handoff from route_helpers import api_auth_required +from services.idempotency_store import idempotency_enabled from services.intelligence_cancellation import CallerCancellation from services.intelligence_contract import ChatRequest, GatewayError from services.intelligence_gateway import ChatGateway @@ -55,7 +57,7 @@ def load_policy(): def reject_idempotency(): - if request.headers.get("Idempotency-Key") is not None: + if request.headers.get("Idempotency-Key") is not None and not idempotency_enabled(): raise GatewayError( "idempotency_not_supported", "Version one does not accept Idempotency-Key; uncertain generations must not be replayed.", @@ -158,7 +160,8 @@ def dispatch_intelligence_chat(app, auth, metrics, proxy, payload): gateway = None try: reject_idempotency() - policy = load_policy() + policy = getattr(g, "managed_idempotency_policy", None) or load_policy() + identity_policy = policy from services.judge_routing import judge_candidate_allowed policy = {**policy, "candidates": [candidate for candidate in policy["candidates"] @@ -181,6 +184,9 @@ def dispatch_intelligence_chat(app, auth, metrics, proxy, payload): "invalid_routing_request", "The request does not satisfy the version-one chat and routing contract.", ) from None + replay = begin_managed_request(payload, identity_policy) + if replay is not None: + return replay gateway = ChatGateway( parsed, policy, @@ -203,11 +209,18 @@ def dispatch_intelligence_chat(app, auth, metrics, proxy, payload): if cooldown_settings().enabled: gateway.prepare_credentials() return _cascade_stream(gateway, cascade_deadline) if cascade_deadline is not None else stream_response(gateway) + try: + mark_managed_handoff() + except GatewayError: + # No provider was contacted when the durable dispatch acknowledgement failed. + gateway.settle() + raise result = list(gateway.events()) response = Response(json.dumps(result[-1]), content_type="application/json") if parsed.payload.get("tools"): response.headers[HEADER] = summary_header(gateway.tool_repair.buffer.report) - return response + # Exhausting events proves validated nonstream completion, including settlement. + return finish_managed_response(response, complete=True) except (ModelCooldownExhausted, ModelCooldownCapacity): if gateway is not None: gateway.cancel() diff --git a/services/idempotency_store.py b/services/idempotency_store.py new file mode 100644 index 00000000..dc828a27 --- /dev/null +++ b/services/idempotency_store.py @@ -0,0 +1,201 @@ +"""Durable, single-submission managed request identity and response storage.""" +from __future__ import annotations + +import base64 +import hashlib +import json +import logging +import os +import queue +import re +import threading +import time +import uuid +from dataclasses import dataclass +from functools import lru_cache + +import requests +from requests.adapters import HTTPAdapter + +from services import intelligence_d1_store as private_store +from services.intelligence_contract import GatewayError + +MAX_RESPONSE_BYTES = 1_048_576 +MAX_DOCUMENT_BYTES = 1_420_000 +TTL_SECONDS = 86_400 +PRIVATE_URL = "http://intelligence.internal/v1/managed-state/idempotency" +REPLAY_HEADER = "X-MultiLLM-Idempotency" +RESPONSE_HEADERS = frozenset({"content-type", "content-length", "cache-control", "x-multillm-model", + "x-multillm-tool-repair"}) +_KEY = re.compile(r"[A-Za-z0-9._:/+@-]{1,128}\Z") +_HASH = re.compile(r"[0-9a-f]{64}\Z") +_SLOTS = threading.BoundedSemaphore(8) +logger = logging.getLogger(__name__) + + +@lru_cache(maxsize=1) +def _warn_invalid_flag(): + logger.warning("Invalid MANAGED_IDEMPOTENCY_ENABLED; managed idempotency disabled") + + +def idempotency_enabled(): + value = os.environ.get("MANAGED_IDEMPOTENCY_ENABLED", "").strip().lower() + if value not in {"", "0", "false", "no", "off", "1", "true", "yes", "on"}: + _warn_invalid_flag() + return False + return value in {"1", "true", "yes", "on"} + + +def unavailable(): + return GatewayError("idempotency_store_unavailable", + "The durable idempotency authority is unavailable; do not resubmit with a new key.", 503) + + +def _canonical(value): + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False).encode("utf-8") + + +def request_fingerprint(principal, method, path, key, body, revision): + if not isinstance(key, str) or not _KEY.fullmatch(key): + raise GatewayError("invalid_idempotency_key", "Idempotency-Key requires 1 to 128 safe ASCII characters.") + if not isinstance(principal, str) or not principal or len(principal) > 256: + raise unavailable() + try: + scope = hashlib.sha256(_canonical([principal, method, path, key])).hexdigest() + digest = hashlib.sha256(_canonical([body, revision])).hexdigest() + except (ValueError, TypeError, UnicodeError, RecursionError): + raise GatewayError("invalid_request", "The managed request cannot be normalized.") from None + return scope, digest + + +def _submit(document, stopped, deadline, results): + try: + with requests.Session() as session: + session.trust_env = False + session.mount("http://", HTTPAdapter(max_retries=0)) + if stopped.is_set() or time.monotonic() >= deadline: + raise unavailable() + with session.post(PRIVATE_URL, data=document, + headers={"Content-Type": "application/json", "Accept": "application/json", "Accept-Encoding": "identity"}, + timeout=(2, 3), allow_redirects=False, stream=True) as response: + result = private_store._decode_response(response, stopped, deadline, (200,), MAX_DOCUMENT_BYTES) + results.put_nowait((True, result)) + except Exception: + results.put_nowait((False, unavailable())) + finally: + _SLOTS.release() + + +def private_call(body): + """One fixed private submission, with bounded bytes, concurrency and wall time.""" + if os.environ.get("INTELLIGENCE_STORAGE_BACKEND", "").strip() != "d1": + raise unavailable() + try: + document = _canonical(body) + except (ValueError, TypeError, UnicodeError, RecursionError): + raise unavailable() from None + if len(document) > MAX_DOCUMENT_BYTES or not _SLOTS.acquire(blocking=False): + raise unavailable() + stopped, results = threading.Event(), queue.Queue(maxsize=1) + deadline = time.monotonic() + 5 + worker = threading.Thread(target=_submit, args=(document, stopped, deadline, results), daemon=True, + name="managed-idempotency-store") + try: + worker.start() + except Exception: + _SLOTS.release() + raise unavailable() from None + try: + success, result = results.get(timeout=max(0, deadline - time.monotonic())) + if not success: + raise result + return result + except queue.Empty: + raise unavailable() from None + finally: + stopped.set() + + +def encode_response(response): + if response.is_streamed or not 200 <= response.status_code < 300: + return None + body = response.get_data() + if len(body) > MAX_RESPONSE_BYTES: + return None + headers = [[name, value] for name, value in response.headers if name.lower() in RESPONSE_HEADERS] + if len(_canonical(headers)) > 16_384: + return None + return {"status": response.status_code, "headers": headers, "body": base64.b64encode(body).decode("ascii")} + + +def decode_response(document): + from flask import Response + try: + if not isinstance(document, dict) or set(document) != {"status", "headers", "body"}: + raise ValueError() + if type(document["status"]) is not int or not 200 <= document["status"] < 300: + raise ValueError() + headers = document["headers"] + if not isinstance(headers, list) or len(headers) > 32 or len(_canonical(headers)) > 16_384: + raise ValueError() + for pair in headers: + if not isinstance(pair, list) or len(pair) != 2 or not all(isinstance(value, str) for value in pair): + raise ValueError() + if pair[0].lower() not in RESPONSE_HEADERS or any(c in pair[1] for c in "\r\n\x00"): + raise ValueError() + encoded = document["body"] + if not isinstance(encoded, str) or len(encoded) > (MAX_RESPONSE_BYTES + 2) // 3 * 4: + raise ValueError() + body = base64.b64decode(encoded, validate=True) + if len(body) > MAX_RESPONSE_BYTES: + raise ValueError() + response = Response(body, status=document["status"], headers=headers) + response.headers[REPLAY_HEADER] = "replayed" + return response + except (ValueError, TypeError, KeyError, UnicodeError): + raise unavailable() from None + + +@dataclass(frozen=True) +class Claim: + scope: str + digest: str + owner: str + status: str + response: dict | None = None + + +class IdempotencyStore: + def __init__(self, call=None): + self.call = call or private_call + + def _request(self, operation, scope, digest, owner, **values): + if not _HASH.fullmatch(scope) or not _HASH.fullmatch(digest) or not re.fullmatch(r"[0-9a-f]{32}", owner): + raise unavailable() + try: + document = self.call({"version": 1, "operation": operation, "scope": scope, "digest": digest, "owner": owner, **values}) + if not isinstance(document, dict) or set(document) != {"version", "result"} or type(document["version"]) is not int or document["version"] != 1: + raise unavailable() + result = document["result"] + if not isinstance(result, dict): + raise unavailable() + return result + except Exception: + raise unavailable() from None + + def claim(self, scope, digest, *, pending_seconds=3600): + owner = uuid.uuid4().hex + result = self._request("claim", scope, digest, owner, pending_seconds=pending_seconds) + status = result.get("status") + if status not in {"claimed", "pending", "conflict", "completed", "unknown"}: + raise unavailable() + expected = {"status", "response"} if status == "completed" else {"status"} + if set(result) != expected: + raise unavailable() + return Claim(scope, digest, owner, status, result.get("response")) + + def transition(self, claim, operation, **values): + result = self._request(operation, claim.scope, claim.digest, claim.owner, **values) + if set(result) != {"changed"} or type(result["changed"]) is not bool: + raise unavailable() + return result["changed"] diff --git a/tests/fixtures/secret_firewall_coverage.json b/tests/fixtures/secret_firewall_coverage.json index ee82f1ec..4e33c656 100644 --- a/tests/fixtures/secret_firewall_coverage.json +++ b/tests/fixtures/secret_firewall_coverage.json @@ -235,6 +235,7 @@ "worker/gateway-lifecycle.mjs", "worker/gpt-image-moderation.mjs", "worker/health-schedule.mjs", + "worker/idempotency-d1.mjs", "worker/intelligence-auth-d1.mjs", "worker/intelligence-d1.mjs", "worker/intelligence-outbound.mjs", @@ -287,6 +288,7 @@ "worker/knowledge/skills-validation.mjs", "worker/knowledge/skills.mjs", "worker/log.mjs", + "worker/managed-state-dispatch.mjs", "worker/media-files.mjs", "worker/media-jobs.mjs", "worker/media-outbound.mjs", diff --git a/tests/test_managed_idempotency.py b/tests/test_managed_idempotency.py new file mode 100644 index 00000000..ef2579e5 --- /dev/null +++ b/tests/test_managed_idempotency.py @@ -0,0 +1,345 @@ +"""Managed generation contracts through registered Flask routes and injected authority.""" +import base64 +import importlib +import json +import sqlite3 +import threading +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from unittest.mock import patch + +import pytest +import requests +from flask import Flask, Response, g + +from services import idempotency_store as store_module +from middleware import idempotency as middleware +from tests.intelligence_fixtures import IntelligenceApiTestCase, completion, upstream + +MIGRATION = Path("intelligence-migrations/0023_idempotency.sql") + + +class Authority: + def __init__(self): + self.rows = {} + self.lock = threading.Lock() + self.calls = [] + self.fail = None + + def __call__(self, body): + with self.lock: + self.calls.append(body) + if self.fail == body["operation"]: + raise requests.Timeout("private detail") + scope, digest, owner = body["scope"], body["digest"], body["owner"] + row = self.rows.get(scope) + operation = body["operation"] + if operation == "claim": + if row is None: + self.rows[scope] = {"digest": digest, "owner": owner, "status": "pending"} + result = {"status": "claimed"} + elif row["digest"] != digest: + result = {"status": "conflict"} + elif row["status"] == "completed": + result = {"status": "completed", "response": row["response"]} + else: + result = {"status": row["status"]} + else: + changed = row is not None and row["owner"] == owner and row["digest"] == digest and row["status"] == "pending" + if changed and operation != "handoff": + row["status"] = "completed" if operation == "complete" else "unknown" + if operation == "complete": + row["response"] = body["response"] + result = {"changed": changed} + return {"version": 1, "result": result} + + +@pytest.fixture(autouse=True) +def pin_settings(monkeypatch): + for name in ("MANAGED_IDEMPOTENCY_ENABLED", "CONTENT_RETENTION_POLICY_ENABLED", "ADMISSION_ENABLED", "CONFIG_REVISION_SYNC_ENABLED"): + monkeypatch.setenv(name, "false") + monkeypatch.setenv("INTELLIGENCE_STORAGE_BACKEND", "") + monkeypatch.setenv("NANOGPT_API_KEY", "") + + +def test_registered_hook_claims_before_admission_and_dispatch(): + authority = Authority() + app = Flask(__name__) + middleware.register_idempotency(app, store=store_module.IdempotencyStore(authority), policy_revision=lambda body: "revision-1") + observed = [] + app.extensions["gateway_after_authentication"].append(lambda: observed.append("admission")) + @app.post("/v1/chat/completions") + def managed(): + g.authenticated_user = {"username": "caller"} + for hook in app.extensions["gateway_after_authentication"]: + rejection = hook() + if rejection is not None: + return rejection + return middleware.dispatch_with_idempotency( + lambda: Response(b'{"choices":[1]}', content_type="application/json"), + completed=lambda response: response.status_code == 200) + with patch.dict("os.environ", {"MANAGED_IDEMPOTENCY_ENABLED": "true"}): + with app.test_client() as client: + first = client.post("/v1/chat/completions", json={"model": "auto:test"}, headers={"Idempotency-Key": "one"}) + second = client.post("/v1/chat/completions", json={"model": "auto:test"}, headers={"Idempotency-Key": "one"}) + assert first.status_code == second.status_code == 200 + assert first.data == second.data and second.headers["X-MultiLLM-Idempotency"] == "replayed" + assert observed == ["admission"] + assert [call["operation"] for call in authority.calls] == ["claim", "handoff", "complete", "claim"] + + +def test_migration_is_additive_and_preserves_existing_rows(): + db = sqlite3.connect(":memory:") + db.execute("CREATE TABLE old_usage (amount INTEGER)") + db.execute("INSERT INTO old_usage VALUES (5)") + db.executescript(MIGRATION.read_text()) + db.executescript(MIGRATION.read_text()) + assert db.execute("SELECT amount FROM old_usage").fetchone() == (5,) + assert db.execute("SELECT count(*) FROM managed_idempotency").fetchone() == (0,) + + +@pytest.mark.parametrize("flag", ["", "false", "bad-value", "0", "off"]) +def test_disabled_flag_and_malformed_warning_are_bounded(monkeypatch, caplog, flag): + monkeypatch.setenv("MANAGED_IDEMPOTENCY_ENABLED", flag) + assert not store_module.idempotency_enabled() + assert not store_module.idempotency_enabled() + assert "bad-value" not in caplog.text + + +@pytest.mark.parametrize("key", ["", "x" * 129, "has space", "newline\n", "ΓΌ"]) +def test_invalid_key_is_rejected_without_storage(key): + with pytest.raises(Exception) as raised: + store_module.request_fingerprint("p", "POST", "/v1/chat/completions", key, {}, "r") + assert raised.value.code == "invalid_idempotency_key" + + +def test_principal_path_body_and_revision_identity(): + fingerprint = store_module.request_fingerprint + first = fingerprint("alice", "POST", "/a", "key", {"b": 1, "a": 2}, "r") + assert first == fingerprint("alice", "POST", "/a", "key", {"a": 2, "b": 1}, "r") + assert first[0] != fingerprint("bob", "POST", "/a", "key", {"b": 1, "a": 2}, "r")[0] + assert first[0] != fingerprint("alice", "POST", "/b", "key", {"b": 1, "a": 2}, "r")[0] + assert first[0] == fingerprint("alice", "POST", "/a", "key", {}, "r2")[0] + assert first[1] != fingerprint("alice", "POST", "/a", "key", {"b": 1, "a": 2}, "r2")[1] + + +class ManagedIdempotencyTests(IntelligenceApiTestCase): + def setUp(self): + # The shared fixture must not load deployment environment files. + with patch("config.load_runtime_env"): + super().setUp() + import os + os.environ.update({"MANAGED_IDEMPOTENCY_ENABLED": "true", "INTELLIGENCE_STORAGE_BACKEND": "", "CONFIG_REVISION_SYNC_ENABLED": "false"}) + self.route = importlib.import_module("routes.intelligence") + self.middleware = importlib.import_module("middleware.idempotency") + self.authority = Authority() + self.middleware.register_idempotency(self.app, store=store_module.IdempotencyStore(self.authority)) + self.headers["Idempotency-Key"] = "test-key" + self.seed() + + def test_completed_replay_preserves_body_headers_and_only_one_provider_call(self): + with self.requests(return_value=upstream(completion())) as send: + first, replay = self.post(), self.post() + assert first.status_code == replay.status_code == 200 and send.call_count == 1 + assert first.data == replay.data + assert first.headers["Content-Type"] == replay.headers["Content-Type"] + assert replay.headers["X-MultiLLM-Idempotency"] == "replayed" + assert [call["operation"] for call in self.authority.calls] == ["claim", "handoff", "complete", "claim"] + + def test_body_model_and_policy_conflicts_do_not_dispatch(self): + with self.requests(return_value=upstream(completion())) as send: + assert self.post().status_code == 200 + for changes in ({"messages": [{"role": "user", "content": "changed"}]}, {"model": "openai:small", "routing": {}}): + conflict = self.post(**changes) + assert conflict.status_code == 422 and conflict.json["error"]["code"] == "idempotency_key_conflict" + with patch.object(self.route, "load_policy", return_value={**self.route.load_policy(), "deadline_ms": 1000}): + # The hook reads the same policy source as managed dispatch. + conflict = self.post() + assert conflict.status_code == 422 and send.call_count == 1 + + def test_concurrent_submissions_have_one_provider_call(self): + entered, release = threading.Event(), threading.Event() + def generate(*args, **kwargs): + entered.set() + assert release.wait(3) + return upstream(completion()) + def post(): + with self.app.test_client() as client: + return client.post("/v1/chat/completions", headers=self.headers, json={"model": "auto:intelligence", "messages": [{"role": "user", "content": "test"}]}) + with self.requests(side_effect=generate) as send, ThreadPoolExecutor(max_workers=2) as executor: + future = executor.submit(post) + try: + assert entered.wait(3) + duplicate = post() + assert duplicate.status_code == 409 and duplicate.json["error"]["code"] == "request_in_progress" + assert duplicate.headers["Retry-After"] == "1" + finally: + release.set() + assert future.result(3).status_code == 200 + assert send.call_count == 1 + + def test_ambiguous_outcome_never_generates_again(self): + with self.requests(side_effect=requests.Timeout("private detail")) as send: + first, retry = self.post(), self.post() + assert first.status_code == 502 and retry.status_code == 409 and send.call_count == 1 + assert retry.json["error"]["code"] == "outcome_unknown" + assert "Retry-After" not in retry.headers and b"private detail" not in first.data + + def test_streaming_rejected_before_claim(self): + with self.requests() as send: + response = self.post(stream=True) + assert response.status_code == 400 and response.json["error"]["code"] == "idempotency_stream_unsupported" + assert self.authority.calls == [] and send.call_count == 0 + + def test_missing_authority_fails_closed(self): + self.authority.fail = "claim" + with self.requests() as send: + response = self.post() + assert response.status_code == 503 and response.json["error"]["code"] == "idempotency_store_unavailable" + assert send.call_count == 0 + + def test_lost_handoff_ack_never_dispatches(self): + from services.control_plane_backup import capture + self.authority.fail = "handoff" + with self.requests() as send: + first, retry = self.post(), self.post() + assert first.status_code == 503 and retry.status_code == 409 and send.call_count == 0 + assert [(row["state"], row["charged"]) for row in capture()["tables"]["intelligence_reservations"]] == [("settled", 0)] + + def test_lost_completion_or_oversized_body_blocks_retry(self): + for failure in ("complete", None): + self.authority.rows.clear() + self.authority.fail = failure + content = "ok" if failure else "x" * store_module.MAX_RESPONSE_BYTES + with self.requests(return_value=upstream(completion(content))) as send: + first, retry = self.post(), self.post() + assert first.status_code == 200 and retry.status_code == 409 and send.call_count == 1 + assert retry.json["error"]["code"] == "outcome_unknown" + + def test_no_key_and_default_off_preserve_existing_contract(self): + self.headers.pop("Idempotency-Key") + with self.requests(return_value=upstream(completion())) as send: + assert self.post().status_code == 200 and send.call_count == 1 + assert self.authority.calls == [] + self.headers["Idempotency-Key"] = "one" + with patch.dict("os.environ", {"MANAGED_IDEMPOTENCY_ENABLED": "false"}), self.requests() as send: + response = self.post() + assert response.status_code == 400 and response.json["error"]["code"] == "idempotency_not_supported" + assert send.call_count == 0 and self.authority.calls == [] + + def test_raw_openai_header_filter_and_body_are_unchanged(self): + with self.requests(return_value=upstream(completion())) as send: + response = self.post(model="openai:small") + assert response.status_code == 200 and send.call_count == 1 and self.authority.calls == [] + assert "idempotency-key" not in {name.lower() for name in send.call_args.kwargs["headers"]} + assert json.loads(send.call_args.kwargs["data"])["messages"] == [{"role": "user", "content": "test"}] + + def test_dedicated_route_replays_after_durable_handoff_before_provider(self): + def generate(*args, **kwargs): + assert [call["operation"] for call in self.authority.calls] == ["claim", "handoff"] + return upstream(completion()) + with self.requests(side_effect=generate) as send: + first = self.client.post("/intelligence/v1/chat/completions", headers=self.headers, + json={"messages": [{"role": "user", "content": "test"}]}) + replay = self.client.post("/intelligence/v1/chat/completions", headers=self.headers, + json={"messages": [{"role": "user", "content": "test"}]}) + assert first.status_code == replay.status_code == 200 and send.call_count == 1 + assert replay.data == first.data and replay.headers["X-MultiLLM-Idempotency"] == "replayed" + + def test_incomplete_success_body_never_becomes_a_replay(self): + partial = completion() + partial["choices"][0].pop("finish_reason") + with self.requests(return_value=upstream(partial)) as send: + first, retry = self.post(routing={"max_escalations": 0}), self.post(routing={"max_escalations": 0}) + assert first.status_code == 502 and retry.status_code == 409 and send.call_count == 1 + assert retry.json["error"]["code"] == "outcome_unknown" + assert not any(call["operation"] == "complete" for call in self.authority.calls) + + def test_no_key_response_bytes_headers_and_provider_payload_match_flag_off(self): + self.headers.pop("Idempotency-Key") + with patch.object(self.route.time, "time", return_value=1000), patch.object(self.route.time, "perf_counter", return_value=1000), self.requests(side_effect=lambda **kwargs: upstream(completion())) as send: + enabled = self.post() + enabled_payload = send.call_args.kwargs["data"] + with patch.dict("os.environ", {"MANAGED_IDEMPOTENCY_ENABLED": "false"}): + disabled = self.post() + disabled_payload = send.call_args.kwargs["data"] + assert enabled.status_code == disabled.status_code == 200 + assert enabled.data == disabled.data and list(enabled.headers) == list(disabled.headers) + assert enabled_payload == disabled_payload and self.authority.calls == [] + + def test_unauthenticated_submission_never_claims(self): + response = self.client.post("/v1/chat/completions", json={"model": "auto:intelligence"}, headers={"Idempotency-Key": "one"}) + assert response.status_code == 401 and self.authority.calls == [] + + def test_retention_zero_stores_no_response(self): + from services import retention_policy + with patch.object(retention_policy, "request_policy", return_value=retention_policy.RetentionPolicy("zero", True, "r")), self.requests(return_value=upstream(completion())) as send: + first, retry = self.post(), self.post() + assert first.status_code == 200 and retry.status_code == 409 and send.call_count == 1 + assert not any("response" in call for call in self.authority.calls) + + +@pytest.mark.parametrize("document", [{"version": 1, "result": {"changed": 1}}, {"version": True, "result": {}}, {"version": 1, "result": {"status": "free_to_retry"}}, {"version": 1, "result": {"status": "claimed", "extra": "private"}}]) +def test_malformed_authority_responses_never_authorize_dispatch(document): + store = store_module.IdempotencyStore(lambda body: document) + with pytest.raises(Exception) as raised: + store.claim("a" * 64, "b" * 64) + assert raised.value.code == "idempotency_store_unavailable" + + +def test_private_client_uses_fixed_no_retry_transport_and_accepts_full_response_bound(monkeypatch): + monkeypatch.setenv("INTELLIGENCE_STORAGE_BACKEND", "d1") + calls = [] + document = {"version": 1, "result": {"status": "completed", "response": {"status": 200, + "headers": [["Content-Type", "application/json"]], "body": base64.b64encode(b"x" * store_module.MAX_RESPONSE_BYTES).decode()}}} + class Reply: + status_code = 200 + headers = {"Content-Type": "application/json"} + def __enter__(self): + return self + def __exit__(self, *args): + pass + def iter_content(self, chunk_size): + yield json.dumps(document).encode() + class Session: + trust_env = True + def __enter__(self): + return self + def __exit__(self, *args): + pass + def mount(self, prefix, adapter): + assert prefix == "http://" and adapter.max_retries.total == 0 + def post(self, url, **kwargs): + calls.append((url, kwargs, self.trust_env)) + return Reply() + monkeypatch.setattr(store_module.requests, "Session", Session) + result = store_module.private_call({"version": 1, "operation": "claim"}) + assert result == document and len(calls) == 1 + url, options, trust_env = calls[0] + assert url == store_module.PRIVATE_URL and not trust_env + assert options["allow_redirects"] is False and options["stream"] is True and options["timeout"] == (2, 3) + replay = store_module.decode_response(document["result"]["response"]) + assert len(replay.data) == store_module.MAX_RESPONSE_BYTES + + +def test_dispatch_wrapper_requires_explicit_completion_and_notifies_settlement(): + authority = Authority() + notifications = [] + app = Flask(__name__) + middleware.register_idempotency(app, store=store_module.IdempotencyStore(authority), policy_revision=lambda body: "r", + settlement=lambda claim, state: notifications.append(state)) + @app.post("/v1/chat/completions") + def managed(): + g.authenticated_user = {"username": "caller"} + for hook in app.extensions["gateway_after_authentication"]: + refusal = hook() + if refusal is not None: + return refusal + return middleware.dispatch_with_idempotency(lambda: Response("unproven", status=200)) + with patch.dict("os.environ", {"MANAGED_IDEMPOTENCY_ENABLED": "true"}): + with app.test_client() as client: + first = client.post("/v1/chat/completions", json={"model": "auto:test"}, headers={"Idempotency-Key": "one"}) + retry = client.post("/v1/chat/completions", json={"model": "auto:test"}, headers={"Idempotency-Key": "one"}) + assert first.status_code == 200 and retry.status_code == 409 + assert retry.json["error"]["code"] == "outcome_unknown" and notifications == ["unknown"] + assert not any("response" in call for call in authority.calls) diff --git a/tests/test_managed_idempotency_worker.mjs b/tests/test_managed_idempotency_worker.mjs new file mode 100644 index 00000000..8324a75e --- /dev/null +++ b/tests/test_managed_idempotency_worker.mjs @@ -0,0 +1,166 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { readFileSync } from "node:fs"; +import { DatabaseSync } from "node:sqlite"; +import { handleIntelligenceOutbound } from "../worker/intelligence-outbound.mjs"; +import { handleIdempotencyRequest } from "../worker/idempotency-d1.mjs"; +import { handleManagedStateRequest } from "../worker/managed-state-dispatch.mjs"; + +const MIGRATION = new URL("../intelligence-migrations/0023_idempotency.sql", import.meta.url); +const scope = "a".repeat(64), digest = "b".repeat(64), owner = "c".repeat(32); +const body = (operation, changes = {}) => ({ version: 1, operation, scope, digest, owner, ...changes }); +function fixture(t, missing = false, dispatch = handleIntelligenceOutbound) { + const sqlite = new DatabaseSync(":memory:"); + t.after(() => sqlite.close()); + sqlite.exec("CREATE TABLE existing_usage (amount INTEGER); INSERT INTO existing_usage VALUES (9)"); + if (!missing) { sqlite.exec(readFileSync(MIGRATION, "utf8")); sqlite.exec(readFileSync(MIGRATION, "utf8")); } + const wrap = (sql, values = []) => ({ bind: (...values) => wrap(sql, values), + async first() { return sqlite.prepare(sql).get(...values) ?? null; }, + async all() { return { results: sqlite.prepare(sql).all(...values) }; }, + async run() { const result = sqlite.prepare(sql).run(...values); return { meta: { changes: Number(result.changes) } }; } }); + const objects = new Map(); + const bucket = { async put(key, value) { objects.set(key, value); }, async get(key) { + const value = objects.get(key); return value === undefined ? null : { size: new TextEncoder().encode(value).length, async text() { return value; } }; + }, async delete(key) { objects.delete(key); } }; + const env = { MANAGED_IDEMPOTENCY_ENABLED: "true", INTELLIGENCE_DB: { prepare: wrap, async batch(statements) { + sqlite.exec("BEGIN"); try { const results = []; for (const statement of statements) results.push(await statement.all()); sqlite.exec("COMMIT"); return results; } + catch (error) { sqlite.exec("ROLLBACK"); throw error; } + } }, multillm_media: bucket }; + const call = async (payload, target = env, path = "/v1/managed-state/idempotency") => { + const response = await dispatch(new Request(`http://intelligence.internal${path}`, { + method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(payload) }), target); + return { status: response.status, data: await response.json() }; + }; + return { sqlite, env, objects, call }; +} +const response = { status: 200, headers: [["Content-Type", "application/json"]], body: btoa('{"choices":[1]}') }; + +test("private registered path atomically admits one owner and preserves old data", async t => { + const { call, sqlite } = fixture(t); + const claims = await Promise.all([call(body("claim")), call(body("claim", { owner: "d".repeat(32) }))]); + assert.deepEqual(claims.map(item => item.data.result.status).sort(), ["claimed", "pending"]); + assert.equal(sqlite.prepare("SELECT amount FROM existing_usage").get().amount, 9); + assert.equal((await call(body("claim", { digest: "f".repeat(64) }))).data.result.status, "conflict"); + assert.equal((await call(body("claim", { scope: "e".repeat(64) }))).data.result.status, "claimed"); +}); + +test("completion CAS persists exact bounded response and replays across instances", async t => { + const { call, objects } = fixture(t); + await call(body("claim")); + assert.equal((await call(body("handoff"))).data.result.changed, true); + assert.equal((await call(body("complete", { response }))).data.result.changed, true); + assert.equal(objects.size, 1); + const replay = await call(body("claim", { owner: "e".repeat(32) })); + assert.deepEqual(replay.data.result, { status: "completed", response }); + assert.equal((await call(body("unknown"))).data.result.changed, false); +}); + +test("wrong owner and repeated handoff never authorize another submission", async t => { + const { call } = fixture(t); + await call(body("claim")); + assert.equal((await call(body("handoff", { owner: "d".repeat(32) }))).data.result.changed, false); + assert.equal((await call(body("handoff"))).data.result.changed, true); + assert.equal((await call(body("handoff"))).data.result.changed, false); + assert.equal((await call(body("complete", { owner: "d".repeat(32), response }))).data.result.changed, false); +}); + +test("ambiguity, stalled claims and expired results never reclaim a key", async t => { + const { call, sqlite, objects } = fixture(t); + await call(body("claim")); + await call(body("handoff")); + await call(body("unknown")); + assert.equal((await call(body("claim"))).data.result.status, "unknown"); + sqlite.prepare("UPDATE managed_idempotency SET expires_at=0, pending_until=0").run(); + assert.equal((await call(body("claim"))).data.result.status, "unknown"); + await call(body("claim", { scope: "e".repeat(64) })); + sqlite.prepare("UPDATE managed_idempotency SET pending_until=0 WHERE scope=?").run("e".repeat(64)); + assert.equal((await call(body("claim", { scope: "e".repeat(64) }))).data.result.status, "unknown"); + assert.equal(objects.size, 0); +}); + +test("expired completion and missing R2 response are unknown rather than regenerated", async t => { + const { call, sqlite, objects } = fixture(t); + await call(body("claim")); await call(body("handoff")); await call(body("complete", { response })); + objects.clear(); + assert.equal((await call(body("claim"))).data.result.status, "unknown"); + assert.equal(sqlite.prepare("SELECT status FROM managed_idempotency").get().status, "unknown"); + const scope2 = "e".repeat(64); + await call(body("claim", { scope: scope2 })); await call(body("handoff", { scope: scope2 })); await call(body("complete", { scope: scope2, response })); + sqlite.prepare("UPDATE managed_idempotency SET expires_at=0 WHERE scope=?").run(scope2); + assert.equal((await call(body("claim", { scope: scope2 }))).data.result.status, "unknown"); + assert.equal(objects.size, 0); +}); + +test("response limits and unsafe headers fail before content storage", async t => { + const { call, objects } = fixture(t); + await call(body("claim")); await call(body("handoff")); + for (const invalid of [ { ...response, body: btoa("x".repeat(1048577)) }, { ...response, headers: [["Set-Cookie", "private"]] }, + { ...response, status: 502 }, { ...response, body: "invalid-base64!" } ]) { + assert.equal((await call(body("complete", { response: invalid }))).status, 400); + } + assert.equal(objects.size, 0); +}); + +test("disabled flags perform no storage; missing migration and R2 fail closed", async t => { + const { call, env } = fixture(t, true); + for (const flag of [undefined, "", "false", "garbage"]) assert.equal((await call(body("claim"), { MANAGED_IDEMPOTENCY_ENABLED: flag })).status, 404); + assert.equal((await call(body("claim"))).status, 503); + assert.equal((await call(body("claim"), { ...env, multillm_media: undefined })).status, 503); +}); + +test("private boundary rejects unreviewed domains and accepts only injected sibling handlers", async () => { + const request = path => new Request(`http://intelligence.internal${path}`, { method: "POST" }); + assert.equal(await handleManagedStateRequest(request("/v1/state/models"), {}), null); + for (const path of ["/v1/managed-state/reservations", "/v1/managed-state/tool-grants", "/v1/managed-state/unknown"]) + assert.equal((await handleManagedStateRequest(request(path), {})).status, 404); + const called = []; + const handlers = { reservations: async () => { called.push("reservations"); return Response.json({ ok: true }); }, + toolGrants: async () => { called.push("grants"); return Response.json({ ok: true }); } }; + assert.equal((await handleManagedStateRequest(request("/v1/managed-state/reservations"), {}, handlers)).status, 200); + assert.equal((await handleManagedStateRequest(request("/v1/managed-state/tool-grants"), {}, handlers)).status, 200); + assert.deepEqual(called, ["reservations", "grants"]); +}); + + +test("durable authority claims, handoff, exact replay and principal isolation without hub registration", async t => { + const { call, env, sqlite, objects } = fixture(t, false, handleIdempotencyRequest); + const claims = await Promise.all([call(body("claim")), call(body("claim", { owner: "d".repeat(32) }))]); + assert.deepEqual(claims.map(item => item.data.result.status).sort(), ["claimed", "pending"]); + assert.equal((await call(body("claim", { digest: "f".repeat(64) }))).data.result.status, "conflict"); + assert.equal((await call(body("claim", { scope: "e".repeat(64) }))).data.result.status, "claimed"); + assert.equal((await call(body("handoff"))).data.result.changed, true); + assert.equal((await call(body("handoff"))).data.result.changed, false); + assert.equal((await call(body("complete", { response }))).data.result.changed, true); + assert.deepEqual((await call(body("claim"))).data.result, { status: "completed", response }); + assert.equal(objects.size, 1); + sqlite.prepare("UPDATE managed_idempotency SET expires_at=0").run(); + assert.equal((await call(body("claim"))).data.result.status, "unknown"); + assert.equal(objects.size, 0); + assert.equal((await call(body("handoff"))).data.result.changed, false); + assert.equal((await call(body("claim"), { ...env, multillm_media: undefined })).status, 503); +}); + +test("authority validates one-MiB boundary, integrity, failure and pending expiry without hub registration", async t => { + const { call, sqlite, objects, env } = fixture(t, false, handleIdempotencyRequest); + await call(body("claim")); await call(body("handoff")); + const atLimit = { ...response, body: btoa("x".repeat(1048576)) }; + assert.equal((await call(body("complete", { response: { ...atLimit, body: btoa("x".repeat(1048577)) } }))).status, 400); + assert.equal((await call(body("complete", { response: atLimit }))).status, 200); + assert.deepEqual((await call(body("claim"))).data.result.response, atLimit); + objects.set([...objects.keys()][0], "corrupt content"); + assert.equal((await call(body("claim"))).data.result.status, "unknown"); + const other = "e".repeat(64); + await call(body("claim", { scope: other })); + sqlite.prepare("UPDATE managed_idempotency SET pending_until=0 WHERE scope=?").run(other); + assert.equal((await call(body("claim", { scope: other }))).data.result.status, "unknown"); + for (const invalid of [{ ...response, headers: [["Set-Cookie", "private"]] }, { ...response, status: 502 }, { ...response, body: "bad!" }]) + assert.equal((await call(body("complete", { response: invalid }))).status, 400); + for (const flag of [undefined, "", "false", "invalid"]) assert.equal((await call(body("claim"), { MANAGED_IDEMPOTENCY_ENABLED: flag })).status, 404); + env.multillm_media.put = async () => { throw new Error("private storage detail"); }; + const third = "f".repeat(64); + await call(body("claim", { scope: third })); await call(body("handoff", { scope: third })); + const failed = await call(body("complete", { scope: third, response })); + assert.equal(failed.status, 503); + assert.ok(!JSON.stringify(failed.data).includes("private storage detail")); + assert.equal((await call(body("claim", { scope: third }))).data.result.status, "pending"); +}); diff --git a/worker/container-env.mjs b/worker/container-env.mjs index 241cf668..558b8e9c 100644 --- a/worker/container-env.mjs +++ b/worker/container-env.mjs @@ -27,6 +27,7 @@ const DIRECT_ENV_KEYS = [ "INTELLIGENCE_NANOGPT_SUBSCRIPTION_API_KEY", "WORKBENCH_WORKER_URL", "MCP_CONTRACT_DIGESTS_ENABLED", + "MANAGED_IDEMPOTENCY_ENABLED", "CONNECTION_PROFILES_DB_PATH", "CONFIG_REVISION_SYNC_ENABLED", "CONFIG_SYNC_TTL_SECONDS", diff --git a/worker/cors-policy.mjs b/worker/cors-policy.mjs index fbe48ac9..6cc24350 100644 --- a/worker/cors-policy.mjs +++ b/worker/cors-policy.mjs @@ -3,4 +3,4 @@ export const CORS_DEFAULT_HEADERS = "X-MultiLLM-Tool-Repair, X-MultiLLM-Image-QA, X-MultiLLM-Cascade, X-MultiLLM-Conversion-Report, X-MultiLLM-Retention, Authorization, X-Api-Key, X-Goog-Api-Key, X-MultiLLM-Api-Key, X-Roleplay-Session-ID, Anthropic-Version, Anthropic-Beta, Anthropic-Dangerous-Direct-Browser-Access, Content-Type, Accept, Origin, X-Requested-With, OpenAI-Beta, OpenAI-Organization, OpenAI-Project, Idempotency-Key, Moderation, Moderation-Model, Redaction, X-Client-Request-ID, X-App-Name, X-Billing-Mode, X-BYOK-Provider, X-Encryption-Key, X-Encryption-Passphrase, X-Fal-Object-Lifecycle-Preference, X-PAYMENT, X-Prompt-Caching-Cut-After, X-Provider, X-Team-ID, X-Use-BYOK, x-x402"; export const CORS_EXPOSE_HEADERS = "X-MultiLLM-Secret-Scan, X-MultiLLM-Retention, X-MultiLLM-Roleplay-Recovery, " + - "X-MultiLLM-Tool-Repair, X-MultiLLM-Image-QA, X-MultiLLM-Cascade, X-MultiLLM-Conversion-Fidelity, X-MultiLLM-Conversion-Fields, Retry-After, X-Request-ID, X-MultiLLM-RateLimit-Limit, X-MultiLLM-RateLimit-Remaining, X-MultiLLM-RateLimit-Reset, X-MultiLLM-Optimization, X-MultiLLM-Optimization-Mode, X-MultiLLM-Estimated-Input-Before, X-MultiLLM-Estimated-Input-After, X-MultiLLM-Image-Prompts-Compacted, X-MultiLLM-Messages-Summarized, X-MultiLLM-Optimization-Target-Met, X-MultiLLM-Summary, X-MultiLLM-Optimization-Cache-Hits, X-MultiLLM-Optimization-Cache-Misses, X-MultiLLM-Prompt-Cache, X-MultiLLM-Prompt-Cache-Mode, X-MultiLLM-Prompt-Cache-Estimated-Tokens, X-MultiLLM-Provider, X-MultiLLM-Model, X-MultiLLM-Credential-Attempts, X-MultiLLM-Route-Decision, X-MultiLLM-Circuit-State, X-MultiLLM-Auto-Route, X-MultiLLM-Auto-Selected-Model, X-MultiLLM-Auto-Attempts, X-MultiLLM-Auto-Selected-Priority, X-MultiLLM-Auto-Ordering, X-MultiLLM-Auto-Failover-Reasons, X-MultiLLM-Cache, X-MultiLLM-Transport-Failure, X-MultiLLM-Token-Count, X-MultiLLM-Latency-Ms, X-MultiLLM-Estimated-Cost-USD, X-MultiLLM-Cost-Basis, X-MultiLLM-Reasoning-Normalized, X-Roleplay-Session-ID, X-Roleplay-Trace-ID, X-Roleplay-Session-Source, X-Roleplay-Provider, X-Roleplay-Model, X-Roleplay-Selection, X-Roleplay-Memory, X-Roleplay-Estimated-Input-Tokens, X-Roleplay-Max-Output-Tokens, X-Roleplay-Fallback-Count, X-Roleplay-State-Cache, X-Roleplay-Credential-Check, Server-Timing, WWW-Authenticate, X-PAYMENT-RESPONSE, X-Poll-After, X-NanoGPT-Advisor-ID, X-NanoGPT-Data-Endpoint, X-NanoGPT-Direct-Endpoint, X-NanoGPT-Inline-Moderation-Cost-USD, X-NanoGPT-Inline-Moderation-Flagged, X-NanoGPT-Inline-Moderation-Model"; + "X-MultiLLM-Tool-Repair, X-MultiLLM-Image-QA, X-MultiLLM-Cascade, X-MultiLLM-Conversion-Fidelity, X-MultiLLM-Conversion-Fields, Retry-After, X-Request-ID, X-MultiLLM-RateLimit-Limit, X-MultiLLM-RateLimit-Remaining, X-MultiLLM-RateLimit-Reset, X-MultiLLM-Idempotency, X-MultiLLM-Optimization, X-MultiLLM-Optimization-Mode, X-MultiLLM-Estimated-Input-Before, X-MultiLLM-Estimated-Input-After, X-MultiLLM-Image-Prompts-Compacted, X-MultiLLM-Messages-Summarized, X-MultiLLM-Optimization-Target-Met, X-MultiLLM-Summary, X-MultiLLM-Optimization-Cache-Hits, X-MultiLLM-Optimization-Cache-Misses, X-MultiLLM-Prompt-Cache, X-MultiLLM-Prompt-Cache-Mode, X-MultiLLM-Prompt-Cache-Estimated-Tokens, X-MultiLLM-Provider, X-MultiLLM-Model, X-MultiLLM-Credential-Attempts, X-MultiLLM-Route-Decision, X-MultiLLM-Circuit-State, X-MultiLLM-Auto-Route, X-MultiLLM-Auto-Selected-Model, X-MultiLLM-Auto-Attempts, X-MultiLLM-Auto-Selected-Priority, X-MultiLLM-Auto-Ordering, X-MultiLLM-Auto-Failover-Reasons, X-MultiLLM-Cache, X-MultiLLM-Transport-Failure, X-MultiLLM-Token-Count, X-MultiLLM-Latency-Ms, X-MultiLLM-Estimated-Cost-USD, X-MultiLLM-Cost-Basis, X-MultiLLM-Reasoning-Normalized, X-Roleplay-Session-ID, X-Roleplay-Trace-ID, X-Roleplay-Session-Source, X-Roleplay-Provider, X-Roleplay-Model, X-Roleplay-Selection, X-Roleplay-Memory, X-Roleplay-Estimated-Input-Tokens, X-Roleplay-Max-Output-Tokens, X-Roleplay-Fallback-Count, X-Roleplay-State-Cache, X-Roleplay-Credential-Check, Server-Timing, WWW-Authenticate, X-PAYMENT-RESPONSE, X-Poll-After, X-NanoGPT-Advisor-ID, X-NanoGPT-Data-Endpoint, X-NanoGPT-Direct-Endpoint, X-NanoGPT-Inline-Moderation-Cost-USD, X-NanoGPT-Inline-Moderation-Flagged, X-NanoGPT-Inline-Moderation-Model"; diff --git a/worker/d1-schema.mjs b/worker/d1-schema.mjs index 4a0a37b0..4e32ab66 100644 --- a/worker/d1-schema.mjs +++ b/worker/d1-schema.mjs @@ -10,7 +10,7 @@ export const REQUIRED_D1_TABLES = Object.freeze(["intelligence_policy", "intelli "usage_events", "usage_daily", "usage_batches", "media_jobs", "media_job_items", "shadow_eval_samples", "shadow_eval_results", "shadow_eval_config", "shadow_eval_policy_backups", "prompt_templates", - "config_snapshot_revisions", "config_snapshots", "config_snapshot_applications", "control_revisions"]); + "config_snapshot_revisions", "config_snapshots", "config_snapshot_applications", "control_revisions", "managed_idempotency"]); /** * Readiness for the D1 schema. A deploy that skipped `wrangler d1 migrations apply` reports diff --git a/worker/idempotency-d1.mjs b/worker/idempotency-d1.mjs new file mode 100644 index 00000000..cafcbad1 --- /dev/null +++ b/worker/idempotency-d1.mjs @@ -0,0 +1,133 @@ +/** Atomic managed request claims and bounded complete response objects. */ +import { boundedBody } from "./control-users-d1.mjs"; + +export const MAX_RESPONSE_BYTES = 1_048_576; +const MAX_DOCUMENT_BYTES = 1_420_000; +const TTL_SECONDS = 86_400; +const PREFIX = "managed-idempotency/"; +const HASH = /^[0-9a-f]{64}$/; +const OWNER = /^[0-9a-f]{32}$/; +const HEADERS = new Set(["content-type", "content-length", "cache-control", "x-multillm-model", "x-multillm-tool-repair"]); +const encoder = new TextEncoder(); +const object = value => value !== null && typeof value === "object" && !Array.isArray(value); +const exact = (body, fields) => object(body) && Object.keys(body).length === fields.length && fields.every(name => Object.hasOwn(body, name)); +const reply = (result, status = 200) => Response.json(status === 200 ? { version: 1, result } + : { version: 1, error: { code: result, message: "Managed idempotency authority could not accept the operation." } }, + { status, headers: { "cache-control": "no-store" } }); +let warned = false; + +export function idempotencyEnabled(env) { + const flag = String(env.MANAGED_IDEMPOTENCY_ENABLED ?? "").trim().toLowerCase(); + if (!["", "0", "false", "no", "off", "1", "true", "yes", "on"].includes(flag)) { + if (!warned) { warned = true; console.warn("Invalid MANAGED_IDEMPOTENCY_ENABLED; managed idempotency disabled"); } + return false; + } + return ["1", "true", "yes", "on"].includes(flag); +} + +function validResponse(value) { + if (!exact(value, ["status", "headers", "body"]) || !Number.isInteger(value.status) || value.status < 200 || value.status >= 300 + || !Array.isArray(value.headers) || value.headers.length > 32 || encoder.encode(JSON.stringify(value.headers)).length > 16_384 + || !value.headers.every(pair => Array.isArray(pair) && pair.length === 2 && pair.every(v => typeof v === "string") + && HEADERS.has(pair[0].toLowerCase()) && !/[\r\n\x00]/.test(pair[1])) + || typeof value.body !== "string" || value.body.length > Math.ceil(MAX_RESPONSE_BYTES / 3) * 4 + || !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value.body)) return false; + try { return atob(value.body).length <= MAX_RESPONSE_BYTES; } catch { return false; } +} + +function validBody(body) { + if (!object(body) || body.version !== 1 || !["claim", "handoff", "complete", "unknown"].includes(body.operation) + || typeof body.scope !== "string" || !HASH.test(body.scope) || typeof body.digest !== "string" || !HASH.test(body.digest) + || typeof body.owner !== "string" || !OWNER.test(body.owner)) return false; + const fields = ["version", "operation", "scope", "digest", "owner"]; + if (body.operation === "claim" && Object.hasOwn(body, "pending_seconds")) { + fields.push("pending_seconds"); + if (!Number.isInteger(body.pending_seconds) || body.pending_seconds < 1 || body.pending_seconds > TTL_SECONDS) return false; + } + if (body.operation === "complete") { fields.push("response"); if (!validResponse(body.response)) return false; } + return exact(body, fields); +} + +const hash = async text => Array.from(new Uint8Array(await crypto.subtle.digest("SHA-256", encoder.encode(text)))) + .map(byte => byte.toString(16).padStart(2, "0")).join(""); +const responseKey = row => typeof row.response_key === "string" + && new RegExp(`^${PREFIX}${row.scope}/${row.owner}/[0-9a-f]{32}$`).test(row.response_key); + +async function block(db, row, bucket) { + await db.prepare("UPDATE managed_idempotency SET status='unknown', response_key=NULL, response_digest=NULL WHERE scope=? AND owner=? AND status=?") + .bind(row.scope, row.owner, row.status).run(); + if (responseKey(row)) await bucket.delete(row.response_key); + return { status: "unknown" }; +} + +async function replay(db, row, bucket, now) { + if (row.expires_at <= now || !responseKey(row) || !HASH.test(row.response_digest ?? "")) return block(db, row, bucket); + const stored = await bucket.get(row.response_key); + if (!stored || stored.size > MAX_DOCUMENT_BYTES) return block(db, row, bucket); + const document = await stored.text(); + if (encoder.encode(document).length > MAX_DOCUMENT_BYTES || await hash(document) !== row.response_digest) return block(db, row, bucket); + let response; + try { response = JSON.parse(document); } catch { return block(db, row, bucket); } + if (!validResponse(response)) return block(db, row, bucket); + return { status: "completed", response }; +} + +async function claim(db, bucket, body, now) { + const inserted = await db.prepare(`INSERT INTO managed_idempotency + (scope, request_digest, owner, status, created_at, pending_until, expires_at) + VALUES (?, ?, ?, 'pending', ?, ?, ?) ON CONFLICT(scope) DO NOTHING`) + .bind(body.scope, body.digest, body.owner, now, now + (body.pending_seconds ?? 3600), now + TTL_SECONDS).run(); + if (inserted.meta.changes === 1) return { status: "claimed" }; + const row = await db.prepare("SELECT * FROM managed_idempotency WHERE scope=?").bind(body.scope).first(); + if (!row) throw new Error("Claim disappeared"); + if (row.request_digest !== body.digest) return { status: "conflict" }; + if (row.status === "completed") return replay(db, row, bucket, now); + if (row.status === "pending" && row.pending_until <= now) return block(db, row, bucket); + return { status: row.status === "pending" ? "pending" : "unknown" }; +} + +async function complete(db, bucket, body, now) { + const row = await db.prepare("SELECT * FROM managed_idempotency WHERE scope=?").bind(body.scope).first(); + if (!row || row.owner !== body.owner || row.request_digest !== body.digest || row.status !== "pending" || !row.handed_off) return { changed: false }; + if (row.expires_at <= now || row.pending_until <= now) { await block(db, row, bucket); return { changed: false }; } + const key = `${PREFIX}${body.scope}/${body.owner}/${crypto.randomUUID().replaceAll("-", "")}`; + const document = JSON.stringify(body.response); + await bucket.put(key, document, { httpMetadata: { contentType: "application/json" }, customMetadata: { expires_at: String(row.expires_at) } }); + // Each completion has its own object, so a losing CAS cannot overwrite the winner. + const changed = await db.prepare(`UPDATE managed_idempotency SET status='completed', response_key=?, response_digest=? + WHERE scope=? AND request_digest=? AND owner=? AND status='pending' AND handed_off=1 AND pending_until>? AND expires_at>?`) + .bind(key, await hash(document), body.scope, body.digest, body.owner, now, now).run(); + if (changed.meta.changes !== 1) await bucket.delete(key); + return { changed: changed.meta.changes === 1 }; +} + +export async function handleIdempotencyRequest(request, env) { + const url = new URL(request.url); + if (request.method !== "POST" || url.origin !== "http://intelligence.internal" || url.pathname !== "/v1/managed-state/idempotency" + || url.search || url.hash || url.username || url.password) return reply("not_found", 404); + if (!idempotencyEnabled(env)) return reply("not_found", 404); + let body; + try { + if (request.headers.get("content-type")?.split(";", 1)[0].trim() !== "application/json") throw new Error(); + body = JSON.parse(await boundedBody(request, MAX_DOCUMENT_BYTES)); + if (!validBody(body)) throw new Error(); + } catch { return reply("invalid_idempotency_request", 400); } + const db = env.INTELLIGENCE_DB, bucket = env.multillm_media; + if (!db || !bucket) return reply("idempotency_store_unavailable", 503); + const now = Math.floor(Date.now() / 1000); + try { + if (body.operation === "claim") return reply(await claim(db, bucket, body, now)); + if (body.operation === "complete") return reply(await complete(db, bucket, body, now)); + const query = body.operation === "handoff" + ? "UPDATE managed_idempotency SET handed_off=1 WHERE scope=? AND request_digest=? AND owner=? AND status='pending' AND handed_off=0 AND pending_until>? AND expires_at>?" + : "UPDATE managed_idempotency SET status='unknown' WHERE scope=? AND request_digest=? AND owner=? AND status='pending'"; + const args = [body.scope, body.digest, body.owner]; + if (body.operation === "handoff") args.push(now, now); + const changed = await db.prepare(query).bind(...args).run(); + return reply({ changed: changed.meta.changes === 1 }); + } catch { + // Neither caller identities, content nor underlying storage errors belong in logs. + console.warn("Managed idempotency authority unavailable"); + return reply("idempotency_store_unavailable", 503); + } +} diff --git a/worker/intelligence-outbound.mjs b/worker/intelligence-outbound.mjs index 7ba67c2b..cafb3c30 100644 --- a/worker/intelligence-outbound.mjs +++ b/worker/intelligence-outbound.mjs @@ -11,6 +11,7 @@ import { handleRouteHealthRequest } from "./route-health-d1.mjs"; import { handleUsageLedgerRequest } from "./usage-ledger-d1.mjs"; import { handleShadowEvalRequest } from "./shadow-eval-d1.mjs"; import { handleMediaJobsRequest } from "./media-jobs.mjs"; +import { handleManagedStateRequest } from "./managed-state-dispatch.mjs"; /** Domain operations reachable only through the container's private outbound handler. */ export function handleIntelligenceOutbound(request, env) { @@ -33,5 +34,6 @@ export function handleIntelligenceOutbound(request, env) { if (url.pathname === "/v1/usage") return handleUsageLedgerRequest(request, env); if (url.pathname === "/v1/shadow-eval") return handleShadowEvalRequest(request, env); if (url.pathname === "/v1/media-jobs") return handleMediaJobsRequest(request, env); + if (url.pathname.startsWith("/v1/managed-state/")) return handleManagedStateRequest(request, env); return Response.json({ error: { code: "not_found", message: "Storage operation not found." } }, { status: 404 }); } diff --git a/worker/managed-state-dispatch.mjs b/worker/managed-state-dispatch.mjs new file mode 100644 index 00000000..4286437d --- /dev/null +++ b/worker/managed-state-dispatch.mjs @@ -0,0 +1,14 @@ +/** Fixed private managed-state domains; sibling authorities are injected explicitly. */ +import { handleIdempotencyRequest } from "./idempotency-d1.mjs"; + +export function handleManagedStateRequest(request, env, { reservations, toolGrants } = {}) { + const url = new URL(request.url); + if (!url.pathname.startsWith("/v1/managed-state/")) return null; + if (url.origin !== "http://intelligence.internal" || url.search || url.hash || url.username || url.password || request.method !== "POST") { + return Response.json({ version: 1, error: { code: "invalid_store_target", message: "Invalid managed storage target." } }, { status: 400 }); + } + if (url.pathname === "/v1/managed-state/idempotency") return handleIdempotencyRequest(request, env); + if (url.pathname === "/v1/managed-state/reservations" && reservations) return reservations(request, env); + if (url.pathname === "/v1/managed-state/tool-grants" && toolGrants) return toolGrants(request, env); + return Response.json({ version: 1, error: { code: "not_found", message: "Managed storage operation not found." } }, { status: 404 }); +}