diff --git a/app.py b/app.py index a73a07c2..27d3287b 100644 --- a/app.py +++ b/app.py @@ -147,7 +147,7 @@ def create_app() -> Flask: register_gateway_mcp_routes(app, csrf) register_documentation_routes(app, AuthService, ProxyService) register_usage_routes(app, csrf) - register_gateway_extensions(app, callbacks=gateway_callbacks()) + register_gateway_extensions(app, callbacks=gateway_callbacks(csrf=csrf)) # Restore the dashboard's recent requests from the durable ledger in the background. usage_ledger.start(MetricsService.get_instance()) diff --git a/docs/responses-state.md b/docs/responses-state.md new file mode 100644 index 00000000..306f209d --- /dev/null +++ b/docs/responses-state.md @@ -0,0 +1,93 @@ +# Hosted Responses continuation + +`HOSTED_RESPONSES_ENABLED` enables principal-owned continuation on the managed +`POST /v1/responses` route. It defaults to off; an empty or malformed value is +off. A malformed value produces one warning without the supplied value. When +off, existing provider requests, response bytes, headers and storage behavior +are preserved, and retrieval and deletion retain their existing routing behavior. + +Enable the flag in the Worker and Flask Container, select the existing D1 +intelligence storage backend, and apply the additive `hosted_responses` migration. +The authority requires `INTELLIGENCE_DB` and the existing `multillm_media` R2 +bucket. Missing bindings or a missing table produce a JSON 503 before provider +dispatch for a hosted request. Native Worker generation routes do not serve +hosted Responses; managed requests are forwarded to Flask. + +## Requests + +This body opts into gateway storage: + +```json +{"model":"provider:model","input":"Summarize this text","store":true,"gateway_state":true} +``` + +A classified completed response returns an ID beginning with the reserved +gateway prefix `gwresp_`, followed by 32 hexadecimal characters. The gateway +removes `gateway_state` before dispatch and disables provider-side storage for +the opted-in generation. The returned response has `store: true`. + +Continue using that ID and a new input, including tool results: + +```json +{"model":"provider:model","previous_response_id":"gwresp_0123456789abcdef0123456789abcdef","input":[{"type":"function_call_output","call_id":"call_1","output":"result"}],"store":true,"gateway_state":true} +``` + +The gateway rebuilds the original input, completed output items and new input +in that order. It preserves item order and tool call/result boundaries. Each +stored child is self-contained and has its own expiry. A gateway parent can +also be used without `gateway_state: true`; the generation then uses the +retained context and does not create another hosted record. + +Without hosted opt-in or a gateway parent, requests keep the existing behavior. +For example, `{"store":true,"previous_response_id":"resp_provider",...}` +continues through the provider unchanged. Combining hosted opt-in with a +provider-owned parent returns 400: supply full input or use a gateway parent, +because the gateway cannot retain unseen provider history. Hosted opt-in +requires the literal boolean `store: true`. + +## Retrieval and deletion + +Authenticated `GET /v1/responses/` returns the stored completed response. +Authenticated `DELETE /v1/responses/` removes its metadata and R2 body and +returns `{"id":"gwresp_...","object":"response.deleted","deleted":true}`. +Both operations accept only gateway IDs owned by the authenticated principal. +An absent, expired, malformed or foreign ID returns JSON 404. A foreign ID does +not reveal content or authorize deletion. Responses use `Cache-Control: no-store`. + +Deletion is explicit. TTL expiry is the only automatic deletion condition; +expired records are removed when accessed. Failed deletions retain an +inaccessible metadata marker so a subsequent explicit DELETE can finish. +Missing or corrupt bodies return 503 without regeneration or automatic repair. + +## Completion, limits and cost + +State is published only after completed terminal status and settled existing +usage accounting, with measured or explicitly unknown usage. Failed, +incomplete, cancelled, deadline-expired and ambiguous outcomes cannot be read +or continued as completed state. Existing conservative accounting decisions +remain authoritative, including uncertain streaming handoffs under a deadline. +An unverified completed outcome returns `response_outcome_unknown` without +submitting the generation again. Storage acknowledgement lost after dispatch +also does not grant permission to repeat a provider call. + +Streaming stores after a terminal `response.completed` event, clean EOF and +accounting settlement. The terminal event is held until storage succeeds. +An interrupted stream has no completed hosted state; errors are returned as +SSE error events once streaming has begun. There is no SSE reattachment. +Enabled managed idempotency can replay a completed nonstreaming response with +the same gateway ID; an unknown earlier outcome remains blocked. Streaming +idempotency replay is unsupported. + +Each stored input plus response is limited to 1 MiB of UTF-8 JSON, expires +24 hours after creation, and belongs to a chain of at most 16 generations. +Oversized input and excessive depth return 400 before dispatch. An oversized +generated state returns 502 after dispatch and still incurs provider usage. +Storage errors return 503 and never trigger a provider retry. D1 stores owner, +status, provider/model, parent, policy revision, expiry, body pointer, byte +length and SHA-256; R2 bodies use the `responses-state/` prefix. The bounded +private hop is `POST http://intelligence.internal/v1/managed-state/responses`. + +Zero-content retention returns 400 `retention_conflict` before dispatch for +hosted opt-in and gateway continuation. It also forbids retrieval; explicit +deletion remains available. Hosted storage introduces D1/R2 storage operations +in addition to existing provider costs. No prompt or response content is logged. diff --git a/intelligence-migrations/0028_responses_state.sql b/intelligence-migrations/0028_responses_state.sql new file mode 100644 index 00000000..6abe4f7d --- /dev/null +++ b/intelligence-migrations/0028_responses_state.sql @@ -0,0 +1,17 @@ +-- Only a completed row can expose its immutable response body. +CREATE TABLE IF NOT EXISTS hosted_responses ( + id TEXT PRIMARY KEY, + owner TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('writing', 'failed', 'completed', 'deleting')), + provider TEXT NOT NULL, + model TEXT NOT NULL, + parent_id TEXT, + policy_revision TEXT NOT NULL, + depth INTEGER NOT NULL CHECK (depth BETWEEN 1 AND 16), + created_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL, + body_key TEXT NOT NULL, + body_bytes INTEGER NOT NULL CHECK (body_bytes BETWEEN 1 AND 1048576), + body_sha256 TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS hosted_responses_owner_expiry ON hosted_responses (owner, expires_at); diff --git a/package.json b/package.json index 5f15a7d8..57251df1 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "lint": "ruff check . --select E9,F63,F7,F82", "typecheck": "mypy --ignore-missing-imports --follow-imports=skip --disable-error-code import-untyped --disable-error-code var-annotated --disable-error-code arg-type --disable-error-code assignment --disable-error-code operator app.py routes services providers streaming", "test": "python -m pytest -q", - "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_managed_idempotency_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_gateway_alerts_worker.mjs tests/test_session_tiers_worker.mjs tests/test_usage_reservations_worker.mjs tests/test_shared_generation_cache_worker.mjs tests/test_deferred_tools_worker.mjs tests/test_generation_deadline_worker.mjs tests/test_wave4_worker.mjs tests/test_gateway_batches_worker.mjs tests/test_semantic_generation_cache_worker.mjs tests/test_canary_traffic_worker.mjs tests/test_context_pages_worker.mjs", + "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_managed_idempotency_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_gateway_alerts_worker.mjs tests/test_session_tiers_worker.mjs tests/test_usage_reservations_worker.mjs tests/test_shared_generation_cache_worker.mjs tests/test_deferred_tools_worker.mjs tests/test_generation_deadline_worker.mjs tests/test_wave4_worker.mjs tests/test_responses_state_worker.mjs tests/test_gateway_batches_worker.mjs tests/test_semantic_generation_cache_worker.mjs tests/test_canary_traffic_worker.mjs tests/test_context_pages_worker.mjs", "db:migrate": "python scripts/validate_sqlite_schema.py --apply", "db:validate": "python scripts/validate_sqlite_schema.py", "security:secrets": "python scripts/check_static_secrets.py", diff --git a/routes/responses_state.py b/routes/responses_state.py new file mode 100644 index 00000000..57cd4592 --- /dev/null +++ b/routes/responses_state.py @@ -0,0 +1,231 @@ +"""Hosted Responses routes and bounded SSE completion observation.""" +from __future__ import annotations + +import json +from functools import wraps + +from flask import Response, abort, current_app, g, request +from werkzeug.exceptions import HTTPException +from werkzeug.routing import Map + +from route_helpers import api_auth_required +from services import responses_state as state +from services.generation_deadline import current_deadline, GenerationDeadlineExceeded, error_event, error_response as deadline_error +from services.intelligence_contract import GatewayError + + +def error_response(error): + return Response(json.dumps({"error": {"code": error.code, "message": error.message}}), + status=error.status, content_type="application/json", headers={"Cache-Control": "no-store"}) + + +def defer_hosted_idempotency(finalizer): + """Hosted completion settles after accounting and deadline response checks.""" + @wraps(finalizer) + def finish(response): + if getattr(g, "hosted_responses_turn", None) is not None: + return response + return finalizer(response) + return finish + + +def responses_state_hook(): + if not state.enabled() or request.path != "/v1/responses" or request.method != "POST": + return None + payload = request.get_json(silent=True) + if not isinstance(payload, dict): + return None + try: + turn = state.prepare(payload) + if turn is None: + return None + g.hosted_responses_turn = turn + from middleware.idempotency import begin_managed_request + replay = begin_managed_request(payload, turn.revision) + if replay is not None: + g.hosted_responses_turn = None + return replay + except GatewayError as error: + return error_response(error) + except (ValueError, TypeError, UnicodeError, RecursionError): + return error_response(GatewayError("invalid_request", "Hosted Responses input must be valid finite JSON.")) + + +def _event_data(frame): + lines = frame.replace(b"\r\n", b"\n").split(b"\n") + return lines, b"\n".join(line[5:].lstrip(b" ") for line in lines if line.startswith(b"data:")) + + +def _event(frame, turn): + lines, data = _event_data(frame) + if not data or data == b"[DONE]": + return frame + b"\n\n", None, False + try: + event = json.loads(data) + except (ValueError, RecursionError): + raise GatewayError("invalid_response_stream", "Hosted Responses received an invalid event stream.", 502) from None + if not isinstance(event, dict): + raise GatewayError("invalid_response_stream", "Hosted Responses received an invalid event stream.", 502) + body = event.get("response") + if isinstance(body, dict): + event["response"] = state.public_body(body, turn) + if "response_id" in event: + event["response_id"] = turn.id + name = event.get("type", "") + if name == "response.completed" and not state.complete_body(body): + raise GatewayError("invalid_response_stream", "Hosted Responses received an invalid completion event.", 502) + terminal = name in {"response.completed", "response.incomplete", "response.failed", "error"} + prefix = b"\n".join(line for line in lines if not line.startswith(b"data:")) + encoded = prefix + b"\ndata: " + json.dumps(event, ensure_ascii=False, separators=(",", ":")).encode() + b"\n\n" + return encoded, body if name == "response.completed" else None, terminal + + +def hosted_stream(source, turn, headers): + """Hold terminal bytes until EOF accounting, rejecting interrupted or ambiguous streams.""" + pending = b"" + terminal = None + completed = None + try: + for chunk in source: + pending += chunk.encode() if isinstance(chunk, str) else chunk + while True: + # SSE allows LF and CRLF separators, including separators split across reads. + split = pending.find(b"\n\n") + crlf = pending.find(b"\r\n\r\n") + if crlf >= 0 and (split < 0 or crlf < split): + split, width = crlf, 4 + else: + width = 2 + if split < 0: + if len(pending) > state.MAX_DOCUMENT_BYTES: + raise GatewayError("response_state_too_large", "Hosted Responses event exceeds 1 MiB.", 502) + break + if split > state.MAX_DOCUMENT_BYTES: + raise GatewayError("response_state_too_large", "Hosted Responses event exceeds 1 MiB.", 502) + frame, pending = pending[:split], pending[split + width:] + encoded, body, is_terminal = _event(frame, turn) + if is_terminal: + if terminal is not None: + raise GatewayError("invalid_response_stream", "Hosted Responses received multiple terminal events.", 502) + terminal, completed = encoded, body + elif terminal is None: + yield encoded + elif _event_data(frame)[1] not in (b"", b"[DONE]"): + raise GatewayError("invalid_response_stream", "Hosted Responses received data after completion.", 502) + if pending.strip() or terminal is None: + raise GatewayError("incomplete_response_stream", "Hosted Responses stream ended without a terminal event.", 502) + close = getattr(source, "close", None) + if close is not None: + close() + if completed is not None and not state.persist(turn, completed, headers): + raise GatewayError("response_outcome_unknown", "Hosted Responses completion could not be verified.", 502) + yield terminal + except GatewayError as error: + yield ("event: error\ndata: " + json.dumps({"type": "error", "code": error.code, "message": error.message}) + "\n\n").encode() + except GenerationDeadlineExceeded: + yield error_event("responses") + except Exception: + yield b'event: error\ndata: {"type":"error","code":"response_outcome_unknown","message":"Hosted Responses stream was interrupted."}\n\n' + finally: + close = getattr(source, "close", None) + if close is not None: + close() + + +def finish_hosted_response(response): + turn = getattr(g, "hosted_responses_turn", None) + if turn is None: + return response + from middleware.idempotency import finish_managed_response + if not turn.keep: + from services.managed_turn import completed_response + return finish_managed_response(response, complete=completed_response(response)) + turn.cancellation = getattr(g, "gateway_cancellation", None) or turn.cancellation + turn.deadline = current_deadline() or turn.deadline + if response.status_code != 200: + return finish_managed_response(response) + if response.mimetype == "text/event-stream": + response.response = hosted_stream(response.response, turn, response.headers) + response.headers.pop("Content-Length", None) + return response + try: + if response.is_streamed: + # The existing passthrough iterator must finish accounting before storage. + source = response.response + data = bytearray() + try: + for chunk in source: + data.extend(chunk.encode() if isinstance(chunk, str) else chunk) + if len(data) > state.MAX_DOCUMENT_BYTES: + raise GatewayError("response_state_too_large", "Hosted Responses state exceeds 1 MiB.", 502) + finally: + close = getattr(source, "close", None) + if close is not None: + close() + response.set_data(bytes(data)) + body = json.loads(response.get_data()) + stored = state.persist(turn, body, response.headers) + if state.complete_body(body) and not stored: + raise GatewayError("response_outcome_unknown", "Hosted Responses completion could not be verified.", 502) + if stored: + response.set_data(json.dumps(state.public_body(body, turn), ensure_ascii=False, separators=(",", ":"))) + return finish_managed_response(response, complete=stored) + except GatewayError as error: + return finish_managed_response(error_response(error)) + except GenerationDeadlineExceeded: + return finish_managed_response(deadline_error()) + except (ValueError, RecursionError): + return finish_managed_response(error_response(GatewayError("invalid_response", "Hosted Responses received an invalid response.", 502))) + except Exception: + return finish_managed_response(error_response(GatewayError("response_outcome_unknown", "Hosted Responses completion was interrupted.", 502))) + + +def register_responses_state(app, csrf): + original_routes = Map([rule.empty() for rule in app.url_map.iter_rules()], + default_subdomain=app.url_map.default_subdomain, + host_matching=app.url_map.host_matching, + strict_slashes=app.url_map.strict_slashes, + merge_slashes=app.url_map.merge_slashes, + converters=app.url_map.converters) + app.extensions["responses_state_store"] = state.ResponsesStateStore() + app.extensions.setdefault("gateway_after_authentication", []).append(responses_state_hook) + app.after_request(finish_hosted_response) + + @app.route("/v1/responses/", methods=["GET", "DELETE"]) + @csrf.exempt + def stored_response(response_id): + if not state.enabled(): + abort(404) + return authorized_response(response_id) + + def preserve_disabled_routing(): + if request.endpoint != "stored_response" or state.enabled(): + return None + # Restore the pre-existing rule before CSRF, policy, authentication and accounting. + try: + request.url_rule, request.view_args = original_routes.bind_to_environ(request.environ).match(return_rule=True) + except HTTPException as error: + request.url_rule, request.view_args = None, None + request.routing_exception = error + return None + + app.before_request_funcs.setdefault(None, []).insert(0, preserve_disabled_routing) + + @api_auth_required + def authorized_response(response_id): + try: + if not state.gateway_id(response_id): + raise state.not_found() + if request.method == "GET" and not state.request_policy().allows_content: + raise GatewayError("retention_conflict", "Zero-content retention forbids reading hosted Responses state.") + owner = state.principal_owner() + store = current_app.extensions["responses_state_store"] + if request.method == "DELETE": + store.delete(owner, response_id) + body = {"id": response_id, "object": "response.deleted", "deleted": True} + else: + body = store.get(owner, response_id)["document"]["response"] + return Response(json.dumps(body, ensure_ascii=False, separators=(",", ":")), content_type="application/json", + headers={"Cache-Control": "no-store"}) + except GatewayError as error: + return error_response(error) diff --git a/routes/unified.py b/routes/unified.py index 0e99e1da..0422bd82 100644 --- a/routes/unified.py +++ b/routes/unified.py @@ -92,7 +92,7 @@ from services.prompt_cache_affinity import affinity_auto_request, affinity_candidate from services.managed_turn import managed_pipeline, emit_managed_request, with_managed_idempotency, check_managed_candidate from services.managed_dispatch import execute_managed_attempt, observe_managed_response - +from services.responses_state import with_responses_state logger = logging.getLogger(__name__) RAW_CHAT_PASSTHROUGH_PROVIDERS = RAW_PASSTHROUGH_PROVIDERS @@ -823,6 +823,7 @@ def _dispatch_translated_protocol( dispatch_chat=dispatch_unified_chat_completion, ) +@with_responses_state @managed_pipeline def dispatch_protocol_request( app, diff --git a/services/gateway_extensions.py b/services/gateway_extensions.py index 06b8fe83..41b0fcf6 100644 --- a/services/gateway_extensions.py +++ b/services/gateway_extensions.py @@ -40,10 +40,17 @@ def cooldown_failure(error): app.register_error_handler(error, cooldown_failure) -def gateway_callbacks(): +def register_hosted_responses(app, *, csrf=None): + from flask_wtf.csrf import CSRFProtect + from routes.responses_state import register_responses_state + register_responses_state(app, csrf or app.extensions.get("csrf") or CSRFProtect()) + + +def gateway_callbacks(*, csrf=None): + from functools import partial from middleware.admission import register_admission from middleware.rate_limit_headers import register_rate_limit_headers - return (register_retention, register_deadline, register_managed_idempotency, + return (register_retention, partial(register_hosted_responses, csrf=csrf), register_deadline, register_managed_idempotency, register_admission, register_cooldown_errors, register_rate_limit_headers) @@ -84,6 +91,9 @@ def register_managed_idempotency(app): from middleware.idempotency import register_idempotency from services.idempotency_store import IdempotencyStore register_idempotency(app, store=IdempotencyStore(), policy_revision=managed_policy_revision) + from routes.responses_state import defer_hosted_idempotency + finalizers = app.after_request_funcs[None] + finalizers[-1] = defer_hosted_idempotency(finalizers[-1]) def register_gateway_extensions(app, *, callbacks=(), revision_sync=None, security_refreshers=None): diff --git a/services/protocol_translation/responses.py b/services/protocol_translation/responses.py index 2ecd7218..8688bc6f 100644 --- a/services/protocol_translation/responses.py +++ b/services/protocol_translation/responses.py @@ -714,7 +714,7 @@ def response_object( request = request if isinstance(request, Mapping) else {} reasoning = as_mapping(request.get("reasoning")) text = as_mapping(request.get("text")) - return { + response = { "id": response_id, "object": "response", "created_at": created_at, @@ -738,6 +738,8 @@ def response_object( "usage": usage, "metadata": request.get("metadata") or {}, } + from services.responses_state import rewrite_translated_response + return rewrite_translated_response(response) # --------------------------------------------------------------------------- diff --git a/services/responses_state.py b/services/responses_state.py new file mode 100644 index 00000000..d0d4107d --- /dev/null +++ b/services/responses_state.py @@ -0,0 +1,324 @@ +"""Principal-owned Responses history and completion-only persistence.""" +from __future__ import annotations + +import copy +import hashlib +import json +import logging +import os +import queue +import re +import threading +import time +import uuid +from dataclasses import dataclass +from functools import lru_cache, wraps + +import requests +from flask import current_app, g, has_request_context, request +from requests.adapters import HTTPAdapter + +from services import intelligence_d1_store as private_store +from services.generation_deadline import current_deadline +from services.intelligence_contract import GatewayError +from services.retention_policy import request_policy + +ID_PREFIX = "gwresp_" +MAX_STATE_BYTES = 1_048_576 +MAX_DOCUMENT_BYTES = MAX_STATE_BYTES + 16_384 +TTL_SECONDS = 86_400 +MAX_DEPTH = 16 +PRIVATE_URL = "http://intelligence.internal/v1/managed-state/responses" +_ID = re.compile(r"gwresp_[0-9a-f]{32}\Z") +_SLOTS = threading.BoundedSemaphore(8) +logger = logging.getLogger(__name__) + + +@lru_cache(maxsize=1) +def _warn_invalid_flag(): + logger.warning("Invalid HOSTED_RESPONSES_ENABLED; hosted Responses disabled") + + +def enabled(): + flag = os.environ.get("HOSTED_RESPONSES_ENABLED", "").strip().lower() + if flag not in {"", "0", "false", "off", "no", "1", "true", "on", "yes"}: + _warn_invalid_flag() + return False + return flag in {"1", "true", "on", "yes"} + + +def now(): + return int(time.time()) + + +def gateway_id(value): + return isinstance(value, str) and value.startswith(ID_PREFIX) + + +def unavailable(): + return GatewayError("responses_store_unavailable", + "Hosted Responses storage is unavailable; a generation will not be resubmitted.", 503) + + +def not_found(): + return GatewayError("response_not_found", "The stored response was not found.", 404) + + +def canonical(value): + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False).encode() + + +def principal_owner(): + user = getattr(g, "authenticated_user", None) or {} + principal = user.get("id") or user.get("username") + if not isinstance(principal, (str, int)) or not str(principal): + raise unavailable() + return hashlib.sha256(canonical([user.get("tenant_id"), str(principal)])).hexdigest() + + +def _submit(document, stopped, deadline, results): + try: + with requests.Session() as session: + session.trust_env = False + session.mount("http://", HTTPAdapter(max_retries=0)) + if stopped.is_set() or time.monotonic() >= deadline: + raise unavailable() + with session.post(PRIVATE_URL, data=document, headers={"Content-Type": "application/json", + "Accept": "application/json", "Accept-Encoding": "identity"}, + timeout=(2, 3), allow_redirects=False, stream=True) as response: + result = private_store._decode_response(response, stopped, deadline, (200,), MAX_DOCUMENT_BYTES) + results.put_nowait((True, result)) + except Exception: + results.put_nowait((False, unavailable())) + finally: + _SLOTS.release() + + +def private_call(body): + """A fixed private hop, bounded in bytes, concurrency and wall time, with no retries.""" + if os.environ.get("INTELLIGENCE_STORAGE_BACKEND", "").strip() != "d1": + raise unavailable() + try: + document = canonical(body) + except (ValueError, TypeError, UnicodeError, RecursionError): + raise unavailable() from None + if len(document) > MAX_DOCUMENT_BYTES or not _SLOTS.acquire(blocking=False): + raise unavailable() + stopped, results = threading.Event(), queue.Queue(maxsize=1) + deadline = time.monotonic() + 5 + worker = threading.Thread(target=_submit, args=(document, stopped, deadline, results), + daemon=True, name="responses-state-store") + try: + worker.start() + except Exception: + _SLOTS.release() + raise unavailable() from None + try: + success, result = results.get(timeout=max(0, deadline - time.monotonic())) + if not success: + raise result + return result + except queue.Empty: + raise unavailable() from None + finally: + stopped.set() + + +class ResponsesStateStore: + def __init__(self, call=None): + self.call = call or private_call + + def operation(self, operation, **values): + try: + document = self.call({"version": 1, "operation": operation, **values}) + if (not isinstance(document, dict) or set(document) != {"version", "result"} + or type(document["version"]) is not int or document["version"] != 1 + or not isinstance(document["result"], dict)): + raise unavailable() + return document["result"] + except Exception: + raise unavailable() from None + + def probe(self): + if self.operation("probe") != {"ready": True}: + raise unavailable() + + def get(self, owner, response_id): + if not isinstance(response_id, str) or not _ID.fullmatch(response_id): + raise not_found() + result = self.operation("get", owner=owner, id=response_id) + if set(result) != {"state"}: + raise unavailable() + row = result["state"] + if row is None: + raise not_found() + if (not isinstance(row, dict) or row.get("id") != response_id or row.get("owner") != owner + or type(row.get("depth")) is not int or not 1 <= row["depth"] <= MAX_DEPTH + or type(row.get("expires_at")) is not int or not isinstance(row.get("policy_revision"), str) + or not valid_document(row.get("document"), response_id)): + raise unavailable() + if row["expires_at"] <= now(): + raise not_found() + return row + + def put(self, turn, document, provider, model): + deadline = {} + if turn.deadline is not None: + turn.deadline.check() + deadline = {"deadline_ms": int(time.time() * 1000) + turn.deadline.remaining_ms()} + result = self.operation("put", owner=turn.owner, id=turn.id, parent_id=turn.parent_id, + depth=turn.depth, provider=provider, model=model, + policy_revision=turn.revision, document=document, **deadline) + if result != {"stored": True}: + raise unavailable() + + def fail(self, turn): + if self.operation("fail", owner=turn.owner, id=turn.id) != {"failed": True}: + raise unavailable() + + def delete(self, owner, response_id): + if not isinstance(response_id, str) or not _ID.fullmatch(response_id): + raise not_found() + if self.operation("delete", owner=owner, id=response_id) != {"deleted": True}: + raise not_found() + + +def complete_body(body): + from services.managed_turn import completed_envelope + return (isinstance(body, dict) and body.get("object") == "response" + and body.get("status") == "completed" and completed_envelope(body) + and isinstance(body.get("output"), list) + and all(isinstance(item, dict) and item.get("status", "completed") == "completed" + for item in body["output"])) + + +def valid_document(document, response_id): + try: + return (isinstance(document, dict) and set(document) == {"input", "response"} + and isinstance(document["input"], list) and all(isinstance(item, dict) for item in document["input"]) + and complete_body(document["response"]) and document["response"].get("id") == response_id + and len(canonical(document)) <= MAX_STATE_BYTES) + except (ValueError, TypeError, UnicodeError, RecursionError): + return False + + +def input_items(value): + if isinstance(value, str): + return [{"role": "user", "content": value}] + if not isinstance(value, list) or not all(isinstance(item, dict) for item in value): + raise GatewayError("invalid_request", "Hosted Responses input must be a string or an array of items.") + return copy.deepcopy(value) + + +@dataclass +class HostedTurn: + owner: str + id: str + parent_id: str | None + depth: int + revision: str + input: list + payload: dict + store: ResponsesStateStore + keep: bool + accounting: object = None + cancellation: object = None + deadline: object = None + finalized: bool = False + + +def prepare(payload): + """Resolve retained input after authentication, before admission or dispatch.""" + opted = payload.get("gateway_state") is True + parent = payload.get("previous_response_id") + if not opted and not gateway_id(parent): + return None + retention = request_policy() + if not retention.allows_content: + raise GatewayError("retention_conflict", "Zero-content retention forbids hosted Responses state.") + if opted and payload.get("store") is not True: + raise GatewayError("invalid_request", "gateway_state requires store: true.") + if opted and parent and not gateway_id(parent): + raise GatewayError("invalid_request", "Hosted state requires a gateway-owned parent or full input.") + store = current_app.extensions.get("responses_state_store") + if store is None: + raise unavailable() + owner = principal_owner() + history, depth = [], 1 + if gateway_id(parent): + row = store.get(owner, parent) + depth = row["depth"] + 1 + history = [*row["document"]["input"], *row["document"]["response"]["output"]] + if depth > MAX_DEPTH: + raise GatewayError("response_chain_limit", "Hosted Responses chains are limited to 16 generations.") + history.extend(input_items(payload.get("input", ""))) + if len(canonical({"input": history, "response": {}})) > MAX_STATE_BYTES: + raise GatewayError("response_state_too_large", "Hosted Responses state exceeds 1 MiB.") + store.probe() + cleaned = {key: copy.deepcopy(value) for key, value in payload.items() + if key not in {"gateway_state", "previous_response_id"}} + cleaned["input"] = history + if opted: + cleaned["store"] = False + from services.managed_turn import idempotency_policy_revision + revision = hashlib.sha256(canonical({"policy": idempotency_policy_revision(cleaned), + "retention": retention.revision})).hexdigest() + return HostedTurn(owner, ID_PREFIX + uuid.uuid4().hex, parent if gateway_id(parent) else None, + depth, revision, history, cleaned, store, opted) + + +def with_responses_state(dispatch): + @wraps(dispatch) + def wrapped(app, auth, metrics, proxy, payload, *args, **kwargs): + turn = getattr(g, "hosted_responses_turn", None) if has_request_context() else None + if turn is None or request.path != "/v1/responses": + return dispatch(app, auth, metrics, proxy, payload, *args, **kwargs) + from middleware.idempotency import mark_managed_handoff + mark_managed_handoff() + turn.accounting = getattr(g, "usage_context", None) + result = dispatch(app, auth, metrics, proxy, turn.payload, *args, **kwargs) + turn.cancellation = getattr(g, "gateway_cancellation", None) + turn.deadline = current_deadline() + return result + return wrapped + + +def public_body(body, turn): + return {**body, "id": turn.id, "store": True, "previous_response_id": turn.parent_id} + + +def persist(turn, body, headers): + """Require terminal completion and settled accounting; never rebuild by generation.""" + if turn.finalized: + return False + turn.finalized = True + if not complete_body(body): + return False + if (turn.accounting is None or not turn.accounting.finished or turn.accounting.ambiguous + or turn.cancellation is not None and turn.cancellation.lost): + return False + if turn.deadline is not None: + turn.deadline.check() + body = public_body(body, turn) + document = {"input": turn.input, "response": body} + if not valid_document(document, turn.id): + raise GatewayError("response_state_too_large", "Hosted Responses state exceeds 1 MiB.", 502) + model = (headers.get("X-MultiLLM-Auto-Selected-Model") or headers.get("X-MultiLLM-Selected-Model") + or getattr(turn.accounting, "selected", None) or turn.payload.get("model") or "unknown") + provider = model.split(":", 1)[0] if ":" in model else "unknown" + turn.store.put(turn, document, provider, str(body.get("model") or model)) + try: + if turn.deadline is not None: + turn.deadline.check() + if turn.cancellation is not None and turn.cancellation.lost: + raise GatewayError("response_outcome_unknown", "Hosted Responses completion could not be verified.", 502) + except Exception: + turn.store.fail(turn) + raise + return True + + +def rewrite_translated_response(body): + """Small translator call site; persistence belongs to the route lifecycle.""" + turn = getattr(g, "hosted_responses_turn", None) if has_request_context() else None + return public_body(body, turn) if turn is not None and turn.keep else body diff --git a/tests/fixtures/secret_firewall_coverage.json b/tests/fixtures/secret_firewall_coverage.json index 19ccd979..58b93152 100644 --- a/tests/fixtures/secret_firewall_coverage.json +++ b/tests/fixtures/secret_firewall_coverage.json @@ -155,6 +155,9 @@ "routes/public_pages.py": [ "app.get('/about')" ], + "routes/responses_state.py": [ + "app.route('/v1/responses/', methods=['GET', 'DELETE'])" + ], "routes/shadow_eval.py": [ "app.get('/admin/workbench/shadow/league')", "app.get('/admin/workbench/shadow/samples')", @@ -327,6 +330,7 @@ "worker/prompt-templates-d1.mjs", "worker/request-telemetry.mjs", "worker/reservations-d1.mjs", + "worker/responses-state-d1.mjs", "worker/retention-policy.mjs", "worker/roleplay/candidate-context.mjs", "worker/roleplay/capacity.mjs", diff --git a/tests/test_responses_state.py b/tests/test_responses_state.py new file mode 100644 index 00000000..2fb8dde5 --- /dev/null +++ b/tests/test_responses_state.py @@ -0,0 +1,527 @@ +"""Hosted continuation through registered routes with synthetic storage and transports.""" +import copy +import importlib +import json +import sqlite3 +import threading +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + +from services import responses_state as state +from tests.unified_api_test_case import UnifiedApiTestCase + + +BODY = {"id": "resp_provider", "object": "response", "status": "completed", "model": "grok-4.6", + "output": [{"type": "function_call", "id": "fc_1", "call_id": "call_1", "name": "lookup", + "arguments": '{"q":"one"}', "status": "completed"}], + "usage": {"input_tokens": 4, "output_tokens": 2}} + + +class Authority: + def __init__(self): + self.rows = {} + self.calls = [] + self.fail = None + + def __call__(self, body): + self.calls.append(copy.deepcopy(body)) + operation = body["operation"] + if operation == self.fail: + raise OSError("private storage detail") + if operation == "probe": + result = {"ready": True} + else: + key = (body["owner"], body["id"]) + if operation == "get": + row = self.rows.get(key) + result = {"state": copy.deepcopy(row) if row and row["expires_at"] > state.now() and row.get("status") != "failed" else None} + elif operation == "delete": + row = self.rows.pop(key, None) + result = {"deleted": bool(row and row["expires_at"] > state.now())} + elif operation == "fail": + row = self.rows.get(key) + if row: + row["status"] = "failed" + result = {"failed": row is not None} + else: + self.rows.setdefault(key, {name: copy.deepcopy(body[name]) for name in + ("id", "owner", "provider", "model", "parent_id", "policy_revision", "depth", "document")}) + self.rows[key]["expires_at"] = state.now() + state.TTL_SECONDS + result = {"stored": True} + return {"version": 1, "result": result} + + +@pytest.fixture +def gateway(monkeypatch): + for name in ("HOSTED_RESPONSES_ENABLED", "CONTENT_RETENTION_ENABLED", "MANAGED_IDEMPOTENCY_ENABLED", + "ADMISSION_ENABLED", "CONFIG_REVISION_SYNC_ENABLED", "USAGE_RESERVATIONS_ENABLED", + "PROTOCOL_EXTRAS_ENABLED", "SESSION_TIERS_ENABLED", "PROMPT_CACHE_AFFINITY_ENABLED"): + monkeypatch.setenv(name, "false") + monkeypatch.setenv("INTELLIGENCE_STORAGE_BACKEND", "") + monkeypatch.setenv("NANOGPT_API_KEY", "") + with patch("config.load_runtime_env"), patch("requests.sessions.Session.request", side_effect=AssertionError("live HTTP forbidden")): + fixture = UnifiedApiTestCase() + fixture.setUp() + authority = Authority() + fixture.app.extensions["responses_state_store"] = state.ResponsesStateStore(authority) + fixture.authority = authority + fixture.headers = {"Authorization": "Bearer admin-test-key", "X-Request-ID": "responses-state-test"} + fixture.route = importlib.import_module("routes.unified") + try: + yield fixture + finally: + fixture.tearDown() + + +def upstream(body=BODY, status=200): + import requests + response = requests.Response() + response.status_code = status + response._content = json.dumps(body).encode() + response.headers["Content-Type"] = "application/json" + return response + + +def post(gateway, monkeypatch, changes=None, result=BODY, *, enabled=True): + monkeypatch.setenv("HOSTED_RESPONSES_ENABLED", "true" if enabled else "false") + body = {"model": "opencode:grok-4.6", "input": "one", "store": True, "gateway_state": True, **(changes or {})} + with patch.object(gateway.app_module.ProxyService, "make_request", return_value=upstream(result)) as send: + response = gateway.client.post("/v1/responses", json=body, headers=gateway.headers) + response.get_data() + return response, send + + +@pytest.mark.parametrize("flag", ["false", "", "off", "malformed"]) +def test_disabled_post_preserves_provider_bytes_options_headers_and_storage(gateway, monkeypatch, flag): + monkeypatch.setattr(gateway.route.time, "perf_counter", lambda: 1000) + monkeypatch.setattr(gateway.route.time, "time", lambda: 1000) + hooks = gateway.app.extensions["gateway_after_authentication"] + from routes.responses_state import responses_state_hook, finish_hosted_response + hook_index = hooks.index(responses_state_hook) + hooks.remove(responses_state_hook) + finalizers = gateway.app.after_request_funcs[None] + finalizer_index = finalizers.index(finish_hosted_response) + finalizers.remove(finish_hosted_response) + body = {"model": "opencode:grok-4.6", "input": "one", "store": True, + "gateway_state": True, "previous_response_id": "resp_native"} + with patch.object(gateway.app_module.ProxyService, "make_request", return_value=upstream()): + baseline = gateway.client.post("/v1/responses", json=body, headers=gateway.headers) + baseline.get_data() + hooks.insert(hook_index, responses_state_hook) + finalizers.insert(finalizer_index, finish_hosted_response) + monkeypatch.setenv("HOSTED_RESPONSES_ENABLED", flag) + body = {"model": "opencode:grok-4.6", "input": "one", "store": True, + "gateway_state": True, "previous_response_id": "resp_native"} + with patch.object(gateway.app_module.ProxyService, "make_request", return_value=upstream()) as send: + response = gateway.client.post("/v1/responses", json=body, headers=gateway.headers) + assert response.status_code == 200 and response.data == json.dumps(BODY).encode() + sent = json.loads(send.call_args.kwargs["data"]) + assert sent == {**body, "model": "grok-4.6"} + assert gateway.authority.calls == [] + assert response.data == baseline.data + assert list(response.headers) == list(baseline.headers) + + +def test_invalid_flag_warns_once_without_value(monkeypatch, caplog): + state._warn_invalid_flag.cache_clear() + monkeypatch.setenv("HOSTED_RESPONSES_ENABLED", "private-invalid-value") + assert not state.enabled() and not state.enabled() + assert caplog.text.count("Invalid HOSTED_RESPONSES_ENABLED") == 1 + assert "private-invalid-value" not in caplog.text + + +def test_enabled_without_opt_in_and_native_parent_are_unchanged(gateway, monkeypatch): + response, send = post(gateway, monkeypatch, {"gateway_state": False, "previous_response_id": "resp_native"}) + assert response.data == json.dumps(BODY).encode() and response.status_code == 200 + assert json.loads(send.call_args.kwargs["data"])["previous_response_id"] == "resp_native" + assert gateway.authority.calls == [] + + +def test_store_get_continue_preserve_tool_item_order_and_metadata(gateway, monkeypatch): + first, send = post(gateway, monkeypatch) + assert first.status_code == 200 and first.json["id"].startswith(state.ID_PREFIX) + payload = json.loads(send.call_args.kwargs["data"]) + assert "gateway_state" not in payload and payload["store"] is False + response_id = first.json["id"] + stored = gateway.client.get(f"/v1/responses/{response_id}", headers=gateway.headers) + assert stored.json == first.json and stored.headers["Cache-Control"] == "no-store" + tool_result = {"type": "function_call_output", "call_id": "call_1", "output": "answer"} + second, send = post(gateway, monkeypatch, {"previous_response_id": response_id, "input": [tool_result]}) + sent = json.loads(send.call_args.kwargs["data"]) + assert sent["input"] == [{"role": "user", "content": "one"}, *BODY["output"], tool_result] + assert "previous_response_id" not in sent + record = next(row for row in gateway.authority.rows.values() if row["id"] == second.json["id"]) + assert record["depth"] == 2 and record["parent_id"] == response_id + assert record["provider"] == "opencode" and record["policy_revision"] + + +@pytest.mark.parametrize("kind", ["absent", "foreign", "expired"]) +def test_absent_foreign_expired_parent_404_without_provider(gateway, monkeypatch, kind): + first, _ = post(gateway, monkeypatch) + response_id = first.json["id"] + if kind == "absent": + response_id = state.ID_PREFIX + "f" * 32 + else: + row = next(iter(gateway.authority.rows.values())) + if kind == "foreign": + old = next(iter(gateway.authority.rows)) + gateway.authority.rows[("b" * 64, old[1])] = gateway.authority.rows.pop(old) + else: + row["expires_at"] = 0 + response, send = post(gateway, monkeypatch, {"previous_response_id": response_id}) + assert response.status_code == 404 and response.json["error"]["code"] == "response_not_found" + assert send.call_count == 0 + assert gateway.client.get(f"/v1/responses/{response_id}", headers=gateway.headers).status_code == 404 + assert gateway.client.delete(f"/v1/responses/{response_id}", headers=gateway.headers).status_code == 404 + + +def test_explicit_delete_removes_state_without_dispatch(gateway, monkeypatch): + first, _ = post(gateway, monkeypatch) + response_id = first.json["id"] + response = gateway.client.delete(f"/v1/responses/{response_id}", headers=gateway.headers) + assert response.status_code == 200 and response.json["deleted"] is True + assert gateway.authority.rows == {} + assert gateway.client.get(f"/v1/responses/{response_id}", headers=gateway.headers).status_code == 404 + + +def test_disabled_get_delete_match_existing_routing_bytes(gateway, monkeypatch): + monkeypatch.setenv("HOSTED_RESPONSES_ENABLED", "false") + monkeypatch.setattr(gateway.route.time, "perf_counter", lambda: 1000) + for method in (gateway.client.get, gateway.client.delete): + existing = method("/v1/responses/absent/route", headers=gateway.headers) + hosted = method("/v1/responses/" + state.ID_PREFIX + "a" * 32, headers=gateway.headers) + assert hosted.status_code == existing.status_code == 400 + assert hosted.data == existing.data and list(hosted.headers) == list(existing.headers) + assert gateway.authority.calls == [] + + +@pytest.mark.parametrize("changes", [{"gateway_state": True}, {"gateway_state": False, "previous_response_id": state.ID_PREFIX + "a" * 32}]) +def test_zero_retention_refuses_before_storage_and_provider(gateway, monkeypatch, changes): + monkeypatch.setenv("CONTENT_RETENTION_ENABLED", "true") + monkeypatch.setenv("CONTENT_RETENTION_POLICY_JSON", '{"default":"zero"}') + response, send = post(gateway, monkeypatch, changes) + assert response.status_code == 400 and response.json["error"]["code"] == "retention_conflict" + assert send.call_count == 0 and gateway.authority.calls == [] + + +@pytest.mark.parametrize("failure", ["probe", "put"]) +def test_storage_failure_is_visible_and_never_retried(gateway, monkeypatch, failure): + gateway.authority.fail = failure + response, send = post(gateway, monkeypatch) + assert response.status_code == 503 and response.json["error"]["code"] == "responses_store_unavailable" + assert send.call_count == (0 if failure == "probe" else 1) + assert gateway.authority.rows == {} and b"private storage detail" not in response.data + + +@pytest.mark.parametrize("status", ["failed", "incomplete", "cancelled", "in_progress", None]) +def test_nonterminal_and_failed_outcomes_store_nothing(gateway, monkeypatch, status): + response, send = post(gateway, monkeypatch, result={**BODY, "status": status}) + assert send.call_count == 1 and gateway.authority.rows == {} + assert response.status_code == 200 + + +def test_chain_depth_and_input_size_fail_before_provider(gateway, monkeypatch): + first, _ = post(gateway, monkeypatch) + next(iter(gateway.authority.rows.values()))["depth"] = 16 + for changes, code in [({"previous_response_id": first.json["id"]}, "response_chain_limit"), + ({"input": "x" * state.MAX_STATE_BYTES}, "response_state_too_large")]: + response, send = post(gateway, monkeypatch, changes) + assert response.status_code == 400 and response.json["error"]["code"] == code and send.call_count == 0 + + +def test_oversized_completed_output_has_no_replayable_body(gateway, monkeypatch): + result = {**BODY, "output": [{"type": "message", "role": "assistant", "content": "x" * state.MAX_STATE_BYTES}]} + response, send = post(gateway, monkeypatch, result=result) + assert response.status_code == 502 and response.json["error"]["code"] == "response_state_too_large" + assert send.call_count == 1 and gateway.authority.rows == {} + + +def test_translated_chat_completion_can_continue(gateway, monkeypatch): + result = {"id": "chatcmpl-one", "choices": [{"message": {"role": "assistant", "content": "answer"}, "finish_reason": "stop"}]} + response, _ = post(gateway, monkeypatch, {"model": "opencode:kimi-k2.6"}, result) + assert response.status_code == 200 and response.json["id"].startswith(state.ID_PREFIX) + continued, send = post(gateway, monkeypatch, {"model": "opencode:kimi-k2.6", "previous_response_id": response.json["id"], "input": "two"}, result) + assert continued.status_code == 200 + assert [item["role"] for item in json.loads(send.call_args.kwargs["data"])["messages"]] == ["user", "assistant", "user"] + + +@pytest.mark.parametrize("terminal", ["completed", "incomplete", "missing"]) +def test_stream_persists_only_completed_terminal_after_accounting(gateway, monkeypatch, terminal): + import io + import requests + monkeypatch.setenv("HOSTED_RESPONSES_ENABLED", "true") + response = requests.Response() + response.status_code = 200 + frames = 'event: response.created\ndata: ' + json.dumps({"type": "response.created", "response": {**BODY, "status": "in_progress"}}) + '\n\n' + if terminal != "missing": + frames += f'event: response.{terminal}\ndata: ' + json.dumps({"type": f"response.{terminal}", "response": {**BODY, "status": terminal}}) + '\n\n' + response.raw = io.BytesIO(frames.encode()) + response.headers["Content-Type"] = "text/event-stream" + with patch.object(gateway.app_module.ProxyService, "make_request", return_value=response) as send: + streamed = gateway.client.post("/v1/responses", json={"model": "opencode:grok-4.6", "input": "one", "gateway_state": True, "store": True, "stream": True}, headers=gateway.headers, buffered=False) + assert gateway.authority.rows == {} + data = streamed.get_data(as_text=True) + assert send.call_count == 1 + assert (len(gateway.authority.rows) == 1) == (terminal == "completed") + assert "resp_provider" not in data and state.ID_PREFIX in data + + +def test_migration_is_additive_and_reentrant(): + db = sqlite3.connect(":memory:") + db.executescript("CREATE TABLE old_rows (n INTEGER); INSERT INTO old_rows VALUES (7)") + migration = Path("intelligence-migrations/0028_responses_state.sql").read_text() + db.executescript(migration) + db.executescript(migration) + assert db.execute("SELECT n FROM old_rows").fetchone() == (7,) + assert db.execute("SELECT count(*) FROM hosted_responses").fetchone() == (0,) + db.close() + + +class IdempotencyAuthority: + def __init__(self): + self.rows, self.calls, self.lock = {}, [], threading.Lock() + + def __call__(self, body): + with self.lock: + self.calls.append(body["operation"]) + row = self.rows.get(body["scope"]) + if body["operation"] == "claim": + if row is None: + row = {"digest": body["digest"], "owner": body["owner"], "status": "pending"} + self.rows[body["scope"]] = row + result = {"status": "claimed"} + elif row["digest"] != body["digest"]: + result = {"status": "conflict"} + elif row["status"] == "completed": + result = {"status": "completed", "response": row["response"]} + else: + result = {"status": row["status"]} + else: + changed = bool(row and row["owner"] == body["owner"] and row["status"] == "pending") + if changed and body["operation"] != "handoff": + row["status"] = "completed" if body["operation"] == "complete" else "unknown" + if row["status"] == "completed": + row["response"] = body["response"] + result = {"changed": changed} + return {"version": 1, "result": result} + + +@pytest.mark.parametrize("complete", [True, False]) +def test_keyed_completion_replays_gateway_id_and_unknown_never_resubmits(gateway, monkeypatch, complete): + store = importlib.import_module("services.idempotency_store") + authority = IdempotencyAuthority() + gateway.app.extensions["managed_idempotency_store"] = store.IdempotencyStore(authority) + monkeypatch.setenv("MANAGED_IDEMPOTENCY_ENABLED", "true") + gateway.headers["Idempotency-Key"] = "responses-key" + result = BODY if complete else {**BODY, "status": "incomplete"} + first, send = post(gateway, monkeypatch, result=result) + assert send.call_count == 1 + second, send = post(gateway, monkeypatch, result=result) + assert send.call_count == 0 + if complete: + assert second.status_code == 200 and second.data == first.data + assert second.headers["X-MultiLLM-Idempotency"] == "replayed" + assert len(gateway.authority.rows) == 1 + assert authority.calls == ["claim", "handoff", "complete", "claim"] + else: + assert second.status_code == 409 and second.json["error"]["code"] == "outcome_unknown" + assert not gateway.authority.rows + + +def test_concurrent_duplicate_claim_permits_one_generation(gateway, monkeypatch): + store = importlib.import_module("services.idempotency_store") + gateway.app.extensions["managed_idempotency_store"] = store.IdempotencyStore(IdempotencyAuthority()) + monkeypatch.setenv("MANAGED_IDEMPOTENCY_ENABLED", "true") + monkeypatch.setenv("HOSTED_RESPONSES_ENABLED", "true") + body = {"model": "opencode:grok-4.6", "input": "one", "store": True, "gateway_state": True} + started, release = threading.Event(), threading.Event() + + def provider(*args, **kwargs): + started.set() + assert release.wait(5) + return upstream() + + def request_response(): + with gateway.app.test_client() as client: + response = client.post("/v1/responses", json=body, + headers={**gateway.headers, "Idempotency-Key": "concurrent-key"}) + response.get_data() + return response + + with patch.object(gateway.app_module.ProxyService, "make_request", side_effect=provider) as send: + with ThreadPoolExecutor(max_workers=2) as pool: + first = pool.submit(request_response) + try: + assert started.wait(5) + second = pool.submit(request_response).result(timeout=5) + assert second.status_code == 409 and second.json["error"]["code"] == "request_in_progress" + finally: + release.set() + assert first.result(timeout=5).status_code == 200 + assert send.call_count == 1 and len(gateway.authority.rows) == 1 + + +def test_gateway_parent_without_storage_opt_in_runs_ephemeral(gateway, monkeypatch): + first, _ = post(gateway, monkeypatch) + response, send = post(gateway, monkeypatch, {"gateway_state": False, "store": False, + "previous_response_id": first.json["id"], "input": "two"}) + assert response.json == BODY and response.status_code == 200 + payload = json.loads(send.call_args.kwargs["data"]) + assert payload["input"] == [{"role": "user", "content": "one"}, *BODY["output"], {"role": "user", "content": "two"}] + assert "gateway_state" not in payload and "previous_response_id" not in payload + assert len(gateway.authority.rows) == 1 + + +def test_parallel_tool_calls_and_results_keep_exact_boundaries(gateway, monkeypatch): + calls = [BODY["output"][0], {**BODY["output"][0], "call_id": "call_2", "id": "fc_2"}] + first, _ = post(gateway, monkeypatch, result={**BODY, "output": calls}) + results = [{"type": "function_call_output", "call_id": "call_2", "output": "second"}, + {"type": "function_call_output", "call_id": "call_1", "output": "first"}] + _, send = post(gateway, monkeypatch, {"previous_response_id": first.json["id"], "input": results}) + assert json.loads(send.call_args.kwargs["data"])["input"] == [{"role": "user", "content": "one"}, *calls, *results] + + +def test_missing_private_backend_is_503_before_provider(gateway, monkeypatch): + gateway.app.extensions["responses_state_store"] = state.ResponsesStateStore() + response, send = post(gateway, monkeypatch) + assert response.status_code == 503 and send.call_count == 0 + + +@pytest.mark.parametrize("failure", ["cancel", "ambiguous", "deadline"]) +def test_lost_or_ambiguous_outcome_is_not_stored(gateway, monkeypatch, failure): + from flask import g + deadline_module = importlib.import_module("services.generation_deadline") + + def provider(*args, **kwargs): + if failure == "ambiguous": + g.usage_context.ambiguous = True + elif failure == "cancel": + g.gateway_cancellation = SimpleNamespace(lost=True, contexts=[]) + else: + g.generation_deadline = deadline_module.Deadline(0, clock=lambda: 1) + return upstream() + + with patch.object(gateway.app_module.ProxyService, "make_request", side_effect=provider) as send: + monkeypatch.setenv("HOSTED_RESPONSES_ENABLED", "true") + response = gateway.client.post("/v1/responses", json={"model": "opencode:grok-4.6", "input": "one", + "gateway_state": True, "store": True}, headers=gateway.headers) + response.get_data() + assert send.call_count == 1 and not gateway.authority.rows + + +def test_unknown_usage_is_recorded_before_success_is_stored(gateway, monkeypatch): + accounting = importlib.import_module("services.request_accounting") + observed = [] + original = accounting._record + + def record(context, status, usage, units): + observed.append(usage) + return original(context, status, usage, units) + + monkeypatch.setattr(accounting, "_record", record) + response, _ = post(gateway, monkeypatch, result={**BODY, "usage": None}) + assert response.status_code == 200 and observed == [None] + assert len(gateway.authority.rows) == 1 + + +def test_storage_ack_after_deadline_is_not_replayable(gateway, monkeypatch): + from flask import g + deadline_module = importlib.import_module("services.generation_deadline") + clock = [0.0] + authority = gateway.authority + + def late(body): + result = authority(body) + if body["operation"] == "put": + clock[0] = 2 + return result + + gateway.app.extensions["responses_state_store"] = state.ResponsesStateStore(late) + monkeypatch.setenv("HOSTED_RESPONSES_ENABLED", "true") + + def provider(*args, **kwargs): + g.generation_deadline = deadline_module.Deadline(1, clock=lambda: clock[0]) + return upstream({"id": "chatcmpl-deadline", "choices": [{"message": {"role": "assistant", "content": "answer"}, + "finish_reason": "stop"}], "usage": BODY["usage"]}) + + with patch.object(gateway.app_module.ProxyService, "make_request", side_effect=provider) as send: + response = gateway.client.post("/v1/responses", json={"model": "opencode:kimi-k2.6", "input": "one", + "gateway_state": True, "store": True}, headers=gateway.headers) + assert response.status_code == 504 and send.call_count == 1 + assert next(iter(authority.rows.values()))["status"] == "failed" + assert [body["operation"] for body in authority.calls] == ["probe", "put", "fail"] + + +def stream_response(chunks): + response = upstream() + response.raw = SimpleNamespace(close=lambda: None) + response.headers["Content-Type"] = "text/event-stream" + response.iter_content = lambda *args, **kwargs: iter(chunks) + return response + + +def sse_event(name, body): + return ("event: " + name + "\ndata: " + json.dumps({"type": name, "response": body}) + "\n\n").encode() + + +@pytest.mark.parametrize("failure", ["disconnect", "after_terminal", "done_in_content", "missing_body", "duplicate_terminal", "storage"]) +def test_interrupted_or_uncommitted_stream_never_exposes_completed_event(gateway, monkeypatch, failure): + monkeypatch.setenv("HOSTED_RESPONSES_ENABLED", "true") + created = sse_event("response.created", {**BODY, "status": "in_progress"}) + completed = sse_event("response.completed", BODY) + + def chunks(): + yield created + yield sse_event("response.completed", None) if failure == "missing_body" else completed + if failure == "disconnect": + raise OSError("private transport detail") + if failure == "after_terminal": + yield created + if failure == "done_in_content": + yield sse_event("response.output_text.delta", {"text": "[DONE]"}) + if failure == "duplicate_terminal": + yield completed + + if failure == "storage": + gateway.authority.fail = "put" + with patch.object(gateway.app_module.ProxyService, "make_request", return_value=stream_response(chunks())) as send: + response = gateway.client.post("/v1/responses", json={"model": "opencode:grok-4.6", "input": "one", + "gateway_state": True, "store": True, "stream": True}, headers=gateway.headers, buffered=False) + data = response.get_data() + assert send.call_count == 1 and not gateway.authority.rows + assert b"event: error" in data and b"event: response.completed" not in data + assert b"private transport detail" not in data + + +def test_client_stream_abort_before_terminal_stores_nothing(gateway, monkeypatch): + monkeypatch.setenv("HOSTED_RESPONSES_ENABLED", "true") + chunks = [sse_event("response.created", {**BODY, "status": "in_progress"}), sse_event("response.completed", BODY)] + with patch.object(gateway.app_module.ProxyService, "make_request", return_value=stream_response(chunks)) as send: + response = gateway.client.post("/v1/responses", json={"model": "opencode:grok-4.6", "input": "one", + "gateway_state": True, "store": True, "stream": True}, headers=gateway.headers, buffered=False) + next(iter(response.response)) + response.close() + assert send.call_count == 1 and not gateway.authority.rows + + +def test_fragmented_crlf_stream_preserves_completed_tool_envelope(gateway, monkeypatch): + monkeypatch.setenv("HOSTED_RESPONSES_ENABLED", "true") + framed = (sse_event("response.created", {**BODY, "status": "in_progress"}) + sse_event("response.completed", BODY)).replace(b"\n", b"\r\n") + chunks = [framed[index:index + 7] for index in range(0, len(framed), 7)] + with patch.object(gateway.app_module.ProxyService, "make_request", return_value=stream_response(chunks)): + response = gateway.client.post("/v1/responses", json={"model": "opencode:grok-4.6", "input": "one", + "gateway_state": True, "store": True, "stream": True}, headers=gateway.headers, buffered=False) + data = response.get_data() + assert data.count(b"event: response.completed") == 1 + assert next(iter(gateway.authority.rows.values()))["document"]["response"]["output"] == BODY["output"] + + +@pytest.mark.parametrize("changes", [{"input": [{"role": "user", "content": float("nan")}]}, {"store": False}]) +def test_invalid_hosted_request_is_400_without_dispatch(gateway, monkeypatch, changes): + response, send = post(gateway, monkeypatch, changes) + assert response.status_code == 400 and response.json["error"]["code"] == "invalid_request" + assert send.call_count == 0 and gateway.authority.calls == [] diff --git a/tests/test_responses_state_worker.mjs b/tests/test_responses_state_worker.mjs new file mode 100644 index 00000000..c51aa6ac --- /dev/null +++ b/tests/test_responses_state_worker.mjs @@ -0,0 +1,216 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { readFileSync } from "node:fs"; +import { DatabaseSync } from "node:sqlite"; +import { handleIntelligenceOutbound } from "../worker/intelligence-outbound.mjs"; +import { ID_PREFIX, MAX_STATE_BYTES, responsesStateEnabled } from "../worker/responses-state-d1.mjs"; + +const owner = "a".repeat(64), id = ID_PREFIX + "b".repeat(32); +const document = { input: [{ role: "user", content: "one" }], response: { id, object: "response", status: "completed", + output: [{ type: "function_call", call_id: "call_1", name: "f", arguments: "{}", status: "completed" }], usage: null } }; +const payload = (operation, changes = {}) => ({ version: 1, operation, owner, id, ...changes }); +const completion = (changes = {}) => payload("put", { provider: "opencode", model: "grok-4.6", parent_id: null, + policy_revision: "c".repeat(64), depth: 1, document, ...changes }); + +function fixture(t, missing = false) { + const sqlite = new DatabaseSync(":memory:"); + t.after(() => sqlite.close()); + sqlite.exec("CREATE TABLE old_rows (n INTEGER); INSERT INTO old_rows VALUES (7)"); + if (!missing) { + const sql = readFileSync(new URL("../intelligence-migrations/0028_responses_state.sql", import.meta.url), "utf8"); + sqlite.exec(sql); sqlite.exec(sql); + } + const wrap = (sql, values = []) => ({ bind: (...args) => wrap(sql, args), + async first() { return sqlite.prepare(sql).get(...values) ?? null; }, + async run() { return { meta: { changes: Number(sqlite.prepare(sql).run(...values).changes) } }; } }); + const objects = new Map(), deleted = []; + const bucket = { async head() { return null; }, async put(key, value) { objects.set(key, value); }, + async get(key) { const value = objects.get(key); return value === undefined ? null : { + size: new TextEncoder().encode(value).length, async text() { return value; } }; }, + async delete(key) { deleted.push(key); objects.delete(key); } }; + const env = { HOSTED_RESPONSES_ENABLED: "true", INTELLIGENCE_DB: { prepare: wrap }, multillm_media: bucket }; + const call = async (body, target = env, url = "http://intelligence.internal/v1/managed-state/responses") => { + const response = await handleIntelligenceOutbound(new Request(url, { method: "POST", + headers: { "content-type": "application/json" }, body: JSON.stringify(body) }), target); + return { status: response.status, data: await response.json() }; + }; + return { call, env, sqlite, objects, deleted }; +} + +test("registered private probe fails closed for missing migration and bindings", async t => { + const { call, env, sqlite } = fixture(t, true); + assert.equal((await call({ version: 1, operation: "probe" })).status, 503); + sqlite.exec("CREATE TABLE hosted_responses (id TEXT)"); + assert.equal((await call({ version: 1, operation: "probe" })).status, 503); + for (const key of ["INTELLIGENCE_DB", "multillm_media"]) + assert.equal((await call({ version: 1, operation: "probe" }, { ...env, [key]: undefined })).status, 503); +}); + +test("disabled and malformed flags never access state", async t => { + const { call } = fixture(t); + for (const flag of [undefined, "", "false", "off", "malformed"]) + assert.equal((await call({ version: 1, operation: "probe" }, { HOSTED_RESPONSES_ENABLED: flag })).status, 404); + assert.equal(responsesStateEnabled({ HOSTED_RESPONSES_ENABLED: "true" }), true); +}); + +test("completed object preserves order, ownership, metadata and old rows", async t => { + const { call, sqlite, objects } = fixture(t); + assert.equal((await call({ version: 1, operation: "probe" })).data.result.ready, true); + assert.equal((await call(completion())).data.result.stored, true); + const found = (await call(payload("get"))).data.result.state; + assert.deepEqual(found.document, document); + assert.equal(found.depth, 1); assert.equal(found.owner, owner); + const row = sqlite.prepare("SELECT * FROM hosted_responses").get(); + assert.equal(row.status, "completed"); assert.equal(row.expires_at - row.created_at, 86400); + assert.match(row.body_key, /^responses-state\//); assert.match(row.body_sha256, /^[0-9a-f]{64}$/); + assert.equal(row.body_bytes, new TextEncoder().encode([...objects.values()][0]).length); + assert.equal(sqlite.prepare("SELECT n FROM old_rows").get().n, 7); +}); + +test("foreign read and delete never reveal or remove owner content", async t => { + const { call, objects } = fixture(t); + await call(completion()); + assert.equal((await call(payload("get", { owner: "d".repeat(64) }))).data.result.state, null); + assert.equal((await call(payload("delete", { owner: "d".repeat(64) }))).data.result.deleted, false); + assert.equal(objects.size, 1); + assert.equal((await call(payload("delete"))).data.result.deleted, true); + assert.equal(objects.size, 0); + assert.equal((await call(payload("get"))).data.result.state, null); +}); + +test("expired content is inaccessible; expiry alone may remove it", async t => { + const { call, sqlite, objects } = fixture(t); + await call(completion()); + sqlite.prepare("UPDATE hosted_responses SET expires_at=0").run(); + assert.equal((await call(payload("get"))).data.result.state, null); + assert.equal(objects.size, 0); + assert.equal(sqlite.prepare("SELECT count(*) AS n FROM hosted_responses").get().n, 0); +}); + +test("incomplete, invalid and oversized state is rejected before storage", async t => { + const { call, objects } = fixture(t); + for (const invalid of [completion({ depth: 17 }), completion({ depth: 0 }), completion({ parent_id: "resp_native" }), + completion({ document: { ...document, response: { ...document.response, status: "incomplete" } } }), + completion({ document: { ...document, response: { ...document.response, error: { message: "failure" } } } }), + completion({ document: { ...document, input: [{ role: "user", content: "x".repeat(MAX_STATE_BYTES) }] } })]) + assert.equal((await call(invalid)).status, 400); + assert.equal(objects.size, 0); +}); + +test("concurrent identical completions are idempotent and conflicting content cannot replace a winner", async t => { + const { call, objects } = fixture(t); + const results = await Promise.all([call(completion()), call(completion())]); + assert.ok(results.every(result => result.data.result.stored)); + assert.equal(objects.size, 1); + assert.equal((await call(completion({ model: "different" }))).status, 409); + assert.deepEqual((await call(payload("get"))).data.result.state.document, document); +}); + +test("corrupt or absent objects fail visibly without automatic deletion or regeneration", async t => { + const { call, objects, deleted } = fixture(t); + await call(completion()); + objects.set([...objects.keys()][0], "corrupt"); + assert.equal((await call(payload("get"))).status, 503); + assert.equal(deleted.length, 0); + objects.clear(); + assert.equal((await call(payload("get"))).status, 503); +}); + +test("R2 write, read and delete failures are sanitized and preserve recoverable metadata", async t => { + const { call, env, sqlite } = fixture(t); + const bucket = env.multillm_media; + assert.equal((await call(completion(), { ...env, multillm_media: { ...bucket, put() { throw Error("private detail"); } } })).status, 503); + assert.equal(sqlite.prepare("SELECT status FROM hosted_responses").get().status, "writing"); + assert.equal((await call(payload("get"))).data.result.state, null); + await call(completion()); + for (const [operation, method] of [["get", "get"], ["delete", "delete"]]) { + const result = await call(payload(operation), { ...env, multillm_media: { ...bucket, [method]() { throw Error("private detail"); } } }); + assert.equal(result.status, 503); assert.ok(!JSON.stringify(result).includes("private detail")); + assert.equal(sqlite.prepare("SELECT count(*) AS n FROM hosted_responses").get().n, 1); + } +}); + +test("only the fixed private path and versioned operations are accepted", async t => { + const { call } = fixture(t); + for (const url of ["https://external.invalid/v1/managed-state/responses", "http://intelligence.internal/v1/managed-state/responses?q=1"]) + assert.equal((await call(payload("get"), undefined, url)).status, 400); + for (const body of [payload("replay"), payload("get", { id: "resp_native" }), payload("get", { extra: true }), { version: 2, operation: "probe" }]) + assert.equal((await call(body)).status, 400); +}); + +test("same owner continuation verifies parent and chain depth", async t => { + const { call } = fixture(t); + await call(completion()); + const child = ID_PREFIX + "e".repeat(32); + const next = completion({ id: child, parent_id: id, depth: 2, + document: { input: [...document.input, ...document.response.output, { type: "function_call_output", call_id: "call_1", output: "result" }], + response: { ...document.response, id: child, previous_response_id: id } } }); + assert.equal((await call({ ...next, depth: 3 })).status, 404); + assert.equal((await call({ ...next, owner: "f".repeat(64) })).status, 404); + assert.equal((await call(next)).status, 200); + const row = (await call(payload("get", { id: child }))).data.result.state; + assert.equal(row.parent_id, id); assert.equal(row.depth, 2); + assert.deepEqual(row.document, next.document); +}); + +test("expiry during a storage write cannot publish completed state", async t => { + const { call, env, sqlite } = fixture(t); + const clock = Date.now; + const instant = clock(); + t.after(() => { Date.now = clock; }); + Date.now = () => instant; + const bucket = env.multillm_media; + const late = { ...bucket, async put(...args) { await bucket.put(...args); Date.now = () => instant + 20; } }; + assert.equal((await call(completion({ deadline_ms: instant - 1 }))).status, 503); + assert.equal(sqlite.prepare("SELECT count(*) AS n FROM hosted_responses").get().n, 0); + assert.equal((await call(completion({ deadline_ms: instant + 10 }), { ...env, multillm_media: late })).status, 503); + assert.equal(sqlite.prepare("SELECT status FROM hosted_responses").get().status, "writing"); + assert.equal((await call(payload("get"))).data.result.state, null); +}); + +test("cancellation failure marker hides completed content without deleting it", async t => { + const { call, objects } = fixture(t); + await call(completion()); + assert.equal((await call(payload("fail", { owner: "d".repeat(64) }))).data.result.failed, false); + assert.equal((await call(payload("fail"))).data.result.failed, true); + assert.equal((await call(payload("get"))).data.result.state, null); + assert.equal(objects.size, 1); + assert.equal((await call(completion())).status, 409); + assert.equal((await call(payload("delete"))).data.result.deleted, true); + assert.equal(objects.size, 0); +}); + +test("explicit deletion racing an unpublished body removes the late object", async t => { + const { call, env, objects, sqlite } = fixture(t); + let started, release; + const began = new Promise(resolve => { started = resolve; }); + const ready = new Promise(resolve => { release = resolve; }); + const bucket = env.multillm_media; + const slow = { ...bucket, async put(...args) { started(); await ready; return bucket.put(...args); } }; + const write = call(completion(), { ...env, multillm_media: slow }); + await began; + assert.equal((await call(payload("delete"))).data.result.deleted, true); + release(); + assert.equal((await write).status, 503); + assert.equal(objects.size, 0); + assert.equal(sqlite.prepare("SELECT count(*) AS n FROM hosted_responses").get().n, 0); +}); + +test("expiry during completion CAS leaves an inaccessible failed state", async t => { + const { call, env, sqlite } = fixture(t); + const originalClock = Date.now, instant = originalClock(); + t.after(() => { Date.now = originalClock; }); + Date.now = () => instant; + const db = env.INTELLIGENCE_DB; + const delayed = { prepare(sql) { + const stmt = db.prepare(sql); + if (!sql.startsWith("UPDATE hosted_responses SET status='completed'")) return stmt; + return { bind(...values) { const bound = stmt.bind(...values); return { async run() { + const result = await bound.run(); Date.now = () => instant + 20; return result; + } }; } }; + } }; + assert.equal((await call(completion({ deadline_ms: instant + 10 }), { ...env, INTELLIGENCE_DB: delayed })).status, 503); + assert.equal(sqlite.prepare("SELECT status FROM hosted_responses").get().status, "failed"); + assert.equal((await call(payload("get"))).data.result.state, null); + assert.equal((await call(completion())).status, 409); +}); diff --git a/tests/test_wave4_registrar.py b/tests/test_wave4_registrar.py index ec904012..9d277124 100644 --- a/tests/test_wave4_registrar.py +++ b/tests/test_wave4_registrar.py @@ -110,7 +110,7 @@ def test_named_hook_order_and_replay_bypass_all_work(harness): app, _, authority, admission, lease, _, provider, reservation, enable = harness enable() assert [hook.__name__ for hook in app.extensions["gateway_after_authentication"]] == [ - "request_policy_hook", "generation_deadline_hook", "idempotency_request_hook", "admit"] + "request_policy_hook", "responses_state_hook", "generation_deadline_hook", "idempotency_request_hook", "admit"] client = app.test_client() headers = {"Idempotency-Key": "one"} first = client.post("/v1/chat/completions", json=BODY, headers=headers) diff --git a/worker/container-env.mjs b/worker/container-env.mjs index c843a825..2263f68a 100644 --- a/worker/container-env.mjs +++ b/worker/container-env.mjs @@ -44,6 +44,7 @@ const DIRECT_ENV_KEYS = [ "CANARY_TRAFFIC_ENABLED", "CONTEXT_PAGING_ENABLED", "GATEWAY_BATCHES_ENABLED", + "HOSTED_RESPONSES_ENABLED", "SEMANTIC_CACHE_ENABLED", "SEMANTIC_CACHE_POLICY_JSON", "GATEWAY_ALERTS_ENABLED", diff --git a/worker/d1-schema.mjs b/worker/d1-schema.mjs index 42b1e92e..28996804 100644 --- a/worker/d1-schema.mjs +++ b/worker/d1-schema.mjs @@ -10,7 +10,7 @@ export const REQUIRED_D1_TABLES = Object.freeze(["intelligence_policy", "intelli "usage_events", "usage_daily", "usage_batches", "media_jobs", "media_job_items", "shadow_eval_samples", "shadow_eval_results", "shadow_eval_config", "shadow_eval_policy_backups", "prompt_templates", - "config_snapshot_revisions", "config_snapshots", "config_snapshot_applications", "control_revisions", "tool_grants", "generation_cache", "usage_reservation_budgets", "usage_reservations", "usage_reservation_transitions", "session_tiers", "gateway_alert_rules", "gateway_alert_events", "managed_idempotency", "context_pages", "canary_traffic", "semantic_generation_cache", "gateway_batch_files", "gateway_batches", "gateway_batch_items"]); + "config_snapshot_revisions", "config_snapshots", "config_snapshot_applications", "control_revisions", "tool_grants", "generation_cache", "usage_reservation_budgets", "usage_reservations", "usage_reservation_transitions", "session_tiers", "gateway_alert_rules", "gateway_alert_events", "managed_idempotency", "context_pages", "canary_traffic", "semantic_generation_cache", "gateway_batch_files", "gateway_batches", "gateway_batch_items", "hosted_responses"]); /** * Readiness for the D1 schema. A deploy that skipped `wrangler d1 migrations apply` reports diff --git a/worker/gateway-extensions.mjs b/worker/gateway-extensions.mjs index ab61764b..8f5d9c9e 100644 --- a/worker/gateway-extensions.mjs +++ b/worker/gateway-extensions.mjs @@ -9,6 +9,7 @@ import { nativeRevisionConsumer, tickNativeRevisionSync } from "./native-config- import { generationDeadlineHook, createGenerationDeadline, forwardedDeadlineHeaders, GenerationDeadlineExceeded, InvalidGenerationDeadline } from "./generation-deadline.mjs"; import { nativeReservationLifecycle, reservationSettings, ReservationError } from "./reservations-d1.mjs"; export { runScheduledMaintenance, scheduledMaintenanceEnabled } from "./scheduled-maintenance.mjs"; +export { handleResponsesStateRequest, responsesStateEnabled } from "./responses-state-d1.mjs"; import { UpstreamCancellation } from "./upstream-cancellation.mjs"; let warned = false; diff --git a/worker/managed-state-dispatch.mjs b/worker/managed-state-dispatch.mjs index 4286437d..05dede98 100644 --- a/worker/managed-state-dispatch.mjs +++ b/worker/managed-state-dispatch.mjs @@ -1,5 +1,6 @@ /** Fixed private managed-state domains; sibling authorities are injected explicitly. */ import { handleIdempotencyRequest } from "./idempotency-d1.mjs"; +import { handleResponsesStateRequest } from "./responses-state-d1.mjs"; export function handleManagedStateRequest(request, env, { reservations, toolGrants } = {}) { const url = new URL(request.url); @@ -8,6 +9,7 @@ export function handleManagedStateRequest(request, env, { reservations, toolGran return Response.json({ version: 1, error: { code: "invalid_store_target", message: "Invalid managed storage target." } }, { status: 400 }); } if (url.pathname === "/v1/managed-state/idempotency") return handleIdempotencyRequest(request, env); + if (url.pathname === "/v1/managed-state/responses") return handleResponsesStateRequest(request, env); if (url.pathname === "/v1/managed-state/reservations" && reservations) return reservations(request, env); if (url.pathname === "/v1/managed-state/tool-grants" && toolGrants) return toolGrants(request, env); return Response.json({ version: 1, error: { code: "not_found", message: "Managed storage operation not found." } }, { status: 404 }); diff --git a/worker/responses-state-d1.mjs b/worker/responses-state-d1.mjs new file mode 100644 index 00000000..c4617a51 --- /dev/null +++ b/worker/responses-state-d1.mjs @@ -0,0 +1,172 @@ +/** Principal-owned metadata and bounded immutable continuation bodies. */ +import { boundedBody } from "./control-users-d1.mjs"; + +export const ID_PREFIX = "gwresp_"; +export const MAX_STATE_BYTES = 1_048_576; +const MAX_DOCUMENT_BYTES = MAX_STATE_BYTES + 16_384; +const TTL_SECONDS = 86_400; +const PATH = "/v1/managed-state/responses"; +const PREFIX = "responses-state/"; +const ID = /^gwresp_[0-9a-f]{32}$/; +const HASH = /^[0-9a-f]{64}$/; +const encoder = new TextEncoder(); +const object = value => value !== null && typeof value === "object" && !Array.isArray(value); +const exact = (body, keys) => object(body) && Object.keys(body).length === keys.length && keys.every(key => Object.hasOwn(body, key)); +const reply = (result, status = 200) => Response.json(status === 200 ? { version: 1, result } + : { version: 1, error: { code: result, message: "Hosted Responses storage could not accept the operation." } }, + { status, headers: { "cache-control": "no-store" } }); +let warned = false; + +export function responsesStateEnabled(env) { + const flag = String(env.HOSTED_RESPONSES_ENABLED ?? "").trim().toLowerCase(); + if (!["", "0", "false", "off", "no", "1", "true", "on", "yes"].includes(flag)) { + if (!warned) { warned = true; console.warn("Invalid HOSTED_RESPONSES_ENABLED; hosted Responses disabled"); } + return false; + } + return ["1", "true", "on", "yes"].includes(flag); +} + +function validDocument(document, id) { + const response = document?.response; + return exact(document, ["input", "response"]) && Array.isArray(document.input) && document.input.every(object) + && object(response) && response.id === id && response.object === "response" && response.status === "completed" + && !response.error && Array.isArray(response.output) + && response.output.every(item => object(item) && (!Object.hasOwn(item, "status") || item.status === "completed")) + && encoder.encode(JSON.stringify(document)).length <= MAX_STATE_BYTES; +} + +function validBody(body) { + if (!object(body) || body.version !== 1) return false; + if (body.operation === "probe") return exact(body, ["version", "operation"]); + if (!HASH.test(body.owner ?? "") || !ID.test(body.id ?? "")) return false; + const keys = ["version", "operation", "owner", "id"]; + if (["get", "delete", "fail"].includes(body.operation)) return exact(body, keys); + if (body.operation !== "put") return false; + keys.push("provider", "model", "parent_id", "policy_revision", "depth", "document"); + if (Object.hasOwn(body, "deadline_ms")) { + keys.push("deadline_ms"); + if (!Number.isSafeInteger(body.deadline_ms) || body.deadline_ms < 1) return false; + } + return exact(body, keys) && typeof body.provider === "string" && /^[a-z][a-z0-9-]{0,31}$/.test(body.provider) + && typeof body.model === "string" && body.model.length >= 1 && body.model.length <= 256 && !/[\r\n\x00]/.test(body.model) + && (body.parent_id === null || typeof body.parent_id === "string" && ID.test(body.parent_id)) + && typeof body.policy_revision === "string" && HASH.test(body.policy_revision) + && Number.isInteger(body.depth) && body.depth >= 1 && body.depth <= 16 + && validDocument(body.document, body.id); +} + +const hash = async text => Array.from(new Uint8Array(await crypto.subtle.digest("SHA-256", encoder.encode(text)))) + .map(byte => byte.toString(16).padStart(2, "0")).join(""); +const bodyKey = row => HASH.test(row.body_sha256 ?? "") && row.body_key === `${PREFIX}${row.owner}/${row.id}/${row.body_sha256}`; +const find = (db, body) => db.prepare("SELECT * FROM hosted_responses WHERE id=? AND owner=?").bind(body.id, body.owner).first(); + +async function remove(db, bucket, row) { + if (!bodyKey(row)) throw Error("Invalid body pointer"); + // A durable tombstone prevents a concurrent completion from restoring deleted content. + await db.prepare("UPDATE hosted_responses SET status='deleting' WHERE id=? AND owner=? AND body_key=?") + .bind(row.id, row.owner, row.body_key).run(); + await bucket.delete(row.body_key); + const result = await db.prepare("DELETE FROM hosted_responses WHERE id=? AND owner=? AND status='deleting' AND body_key=?") + .bind(row.id, row.owner, row.body_key).run(); + return result.meta.changes === 1; +} + +async function get(db, bucket, body, now) { + const row = await find(db, body); + if (!row) return { state: null }; + if (row.expires_at <= now) { await remove(db, bucket, row); return { state: null }; } + if (row.status !== "completed") return { state: null }; + if (!bodyKey(row) || row.body_bytes > MAX_STATE_BYTES) throw Error("Invalid body pointer"); + const stored = await bucket.get(row.body_key); + if (!stored || stored.size !== row.body_bytes || stored.size > MAX_STATE_BYTES) throw Error("Missing body"); + const text = await stored.text(); + if (encoder.encode(text).length !== row.body_bytes || await hash(text) !== row.body_sha256) throw Error("Invalid body"); + const document = JSON.parse(text); + if (!validDocument(document, row.id)) throw Error("Invalid body"); + // DELETE may have taken ownership while R2 was read. + const current = await find(db, body); + if (!current || current.status !== "completed" || current.expires_at <= Math.floor(Date.now() / 1000)) return { state: null }; + return { state: { id: row.id, owner: row.owner, provider: row.provider, model: row.model, + parent_id: row.parent_id, policy_revision: row.policy_revision, depth: row.depth, expires_at: row.expires_at, document } }; +} + +function matches(row, body, digest) { + return row && row.owner === body.owner && row.body_sha256 === digest && row.provider === body.provider + && row.model === body.model && row.parent_id === body.parent_id && row.depth === body.depth + && row.policy_revision === body.policy_revision; +} + +async function put(db, bucket, body, now) { + const expired = () => Object.hasOwn(body, "deadline_ms") && Date.now() >= body.deadline_ms; + if (expired()) return reply("responses_store_unavailable", 503); + const text = JSON.stringify(body.document), digest = await hash(text); + const key = `${PREFIX}${body.owner}/${body.id}/${digest}`; + if (body.parent_id !== null) { + const parent = await find(db, { id: body.parent_id, owner: body.owner }); + if (!parent || parent.status !== "completed" || parent.expires_at <= now || body.depth !== parent.depth + 1) return reply("response_not_found", 404); + } else if (body.depth !== 1) return reply("invalid_responses_state_request", 400); + // Reserve immutable identity first. A failed write exposes no replayable body; + // retries of this storage operation can write only the same content and metadata. + await db.prepare(`INSERT INTO hosted_responses + (id, owner, status, provider, model, parent_id, policy_revision, depth, created_at, expires_at, body_key, body_bytes, body_sha256) + VALUES (?, ?, 'writing', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(id) DO NOTHING`) + .bind(body.id, body.owner, body.provider, body.model, body.parent_id, body.policy_revision, body.depth, + now, now + TTL_SECONDS, key, encoder.encode(text).length, digest).run(); + const row = await find(db, body); + if (!matches(row, body, digest) || row.status === "deleting" || row.expires_at <= now) return reply("response_state_conflict", 409); + if (row.status === "completed") return reply({ stored: true }); + if (row.status !== "writing") return reply("response_state_conflict", 409); + await bucket.put(key, text, { httpMetadata: { contentType: "application/json" }, customMetadata: { expires_at: String(row.expires_at) } }); + if (expired()) return reply("responses_store_unavailable", 503); + const result = await db.prepare("UPDATE hosted_responses SET status='completed' WHERE id=? AND owner=? AND status='writing' AND body_sha256=? AND expires_at>?") + .bind(body.id, body.owner, digest, Math.floor(Date.now() / 1000)).run(); + if (expired()) { + await db.prepare("UPDATE hosted_responses SET status='failed' WHERE id=? AND owner=? AND body_sha256=? AND status='completed'") + .bind(body.id, body.owner, digest).run(); + return reply("responses_store_unavailable", 503); + } + if (result.meta.changes === 1) return reply({ stored: true }); + const current = await find(db, body); + if (matches(current, body, digest) && current.status === "completed" && current.expires_at > now) return reply({ stored: true }); + // Complete the explicit DELETE (or TTL removal) that raced this unpublished write. + if (!current || current.status === "deleting") await bucket.delete(key); + return reply("responses_store_unavailable", 503); +} + +export async function handleResponsesStateRequest(request, env) { + const url = new URL(request.url); + if (request.method !== "POST" || url.origin !== "http://intelligence.internal" || url.pathname !== PATH + || url.search || url.hash || url.username || url.password) return reply("not_found", 404); + if (!responsesStateEnabled(env)) return reply("not_found", 404); + let body; + try { + if (request.headers.get("content-type")?.split(";", 1)[0].trim() !== "application/json") throw Error(); + body = JSON.parse(await boundedBody(request, MAX_DOCUMENT_BYTES)); + if (!validBody(body)) throw Error(); + } catch { return reply("invalid_responses_state_request", 400); } + const db = env.INTELLIGENCE_DB, bucket = env.multillm_media; + if (!db || !bucket) return reply("responses_store_unavailable", 503); + const now = Math.floor(Date.now() / 1000); + try { + if (body.operation === "probe") { + await db.prepare(`SELECT id, owner, status, provider, model, parent_id, policy_revision, + depth, created_at, expires_at, body_key, body_bytes, body_sha256 FROM hosted_responses LIMIT 1`).first(); + await bucket.head(`${PREFIX}.readiness`); + return reply({ ready: true }); + } + if (body.operation === "put") return await put(db, bucket, body, now); + if (body.operation === "get") return reply(await get(db, bucket, body, now)); + if (body.operation === "fail") { + const result = await db.prepare("UPDATE hosted_responses SET status='failed' WHERE id=? AND owner=? AND status='completed'") + .bind(body.id, body.owner).run(); + return reply({ failed: result.meta.changes === 1 }); + } + const row = await find(db, body); + if (!row) return reply({ deleted: false }); + const deleted = await remove(db, bucket, row); + return reply({ deleted: deleted && row.expires_at > now }); + } catch { + console.warn("Hosted Responses storage unavailable"); + return reply("responses_store_unavailable", 503); + } +}