diff --git a/docs/request-cancellation.md b/docs/request-cancellation.md new file mode 100644 index 00000000..ac433c39 --- /dev/null +++ b/docs/request-cancellation.md @@ -0,0 +1,67 @@ +# Disconnect-aware upstream cancellation + +Cleanup is always enabled. There is no environment flag or disabled mode. +Successful bytes, status codes and headers are preserved. No response content, +credentials or request bodies are added to cancellation outcomes or logs. + +For example, closing a streaming client before its first event, after a partial +event, or while waiting for the next event closes the upstream owner once. +Reading the whole response keeps its original bytes. Aborting after a completed +response does not change that completed outcome. + +## Python transport + +`bind_cancellation(response, on_outcome=callback)` attaches a +`CancellationContext` to `response.multillm_cancellation` and shares its +idempotent close operation with existing owners. `iter_stream_content` and +`iter_stream_lines` return closable iterators immediately, before generator +execution. `GeneratorExit`, read exceptions, normal exhaustion and redundant +close calls release the upstream once. A socket error propagates; it does not +restart reads through a different iterator. + +Unified dispatch binds each returned response without changing request bodies +or existing origin-fallback policy. Flask body ownership closes the underlying +iterable even when no body iteration occurred. The stream preflight (`preflight_chat_stream`) +closes that same owner on rejection; validated prefix replay keeps the same +owner. + +WSGI detects disconnects during writes or iteration. It cannot reliably detect +a disconnect while a synchronous provider setup or socket read is blocked. +Existing bounded connect/read timeouts remain in force. No per-request watcher +thread is created, and cancellation cannot promise the provider stopped billing. + +## Worker transport + +`UpstreamCancellation` owns an AbortController, a bounded header timer, the +parent abort listener and an upstream reader. Parent abort propagates to fetch +through its signal. The listener remains active after headers until the body +finishes or closes. Cancelling a downstream body also aborts fetch and cancels +the upstream reader once. Cleanup does not wait for a provider's cancellation +acknowledgement. `readBoundedUpstreamBytes` owns cancellation and reader release +for bounded nonstream reads, including abort and size rejection. + +Roleplay uses this owner for generation and compaction. A client abort, including +one racing an HTTP rejection, stops candidate and origin fallback. Existing +non-cancellation fallback decisions remain unchanged. Terminal HTTP error bodies +retain their owner until consumption or cancellation instead of detaching at +headers. Header timers end at headers; compaction retains its total timer. + +## Outcomes and settlement + +Python callbacks receive `CancellationOutcome`; Worker callbacks receive an +object with `reason`, `ambiguous`, `usageState`, `usage` and `replayPermission`. +Python also reports an [`UpstreamOutcome`](upstream-outcomes.md). After upstream handoff, +cancellation or interruption is ambiguous, usage is unknown (`None`/`null`), +and replay permission is false. Nothing here asserts zero tokens or zero cost. +Callbacks run once and contain no generated content. Cleanup errors are logged +by type only and do not mask the original result. + +These are transient transport outcomes, not durable accounting records. + +## Coverage + +Unified dispatch, the stream preflight and the roleplay transport use these +owners. Other native Worker provider fetches and the provider-specific streaming +paths in `services/proxy_service.py` keep their existing connect and read +timeouts. Whether a provider stops billing after a disconnect depends on that +provider. diff --git a/package.json b/package.json index 42ee5f27..39179f66 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "lint": "ruff check . --select E9,F63,F7,F82", "typecheck": "mypy --ignore-missing-imports --follow-imports=skip --disable-error-code import-untyped --disable-error-code var-annotated --disable-error-code arg-type --disable-error-code assignment --disable-error-code operator app.py routes services providers streaming", "test": "python -m pytest -q", - "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs", + "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_request_cancellation_worker.mjs", "db:migrate": "python scripts/validate_sqlite_schema.py --apply", "db:validate": "python scripts/validate_sqlite_schema.py", "security:secrets": "python scripts/check_static_secrets.py", diff --git a/routes/unified_transport.py b/routes/unified_transport.py index 035cf243..9a253afc 100644 --- a/routes/unified_transport.py +++ b/routes/unified_transport.py @@ -14,6 +14,7 @@ ) from providers.image_relays import image_relay_backup_base_url from route_helpers import copy_raw_provider_response_headers +from services.request_cancellation import bind_cancellation IMAGE_GENERATION_TIMEOUT = (10, 600) @@ -29,15 +30,20 @@ def send_unified_provider_request( request_headers: Mapping[str, Any], ): """Send one unified request with bounded, replay-safe origin fallback.""" + def send_owned_request(kwargs): + response = proxy_service_cls.make_request(**kwargs) + bind_cancellation(response) + return response + secondary_origin = secondary_origin or image_relay_backup_base_url(provider) if secondary_origin is None and provider != "aihubmix": - return proxy_service_cls.make_request(**request_kwargs) + return send_owned_request(request_kwargs) if secondary_origin is None: raise APIError("AIHubMix backup origin is not configured", status_code=500) return request_with_origin_fallback( - lambda origin: proxy_service_cls.make_request( - **{ + lambda origin: send_owned_request( + { **request_kwargs, "url": build_aihubmix_url(origin, upstream_path), } @@ -158,4 +164,4 @@ def normalized_aihubmix_image_response( headers=copy_raw_provider_response_headers(response.headers), ) finally: - response.close() + bind_cancellation(response).complete() diff --git a/services/request_cancellation.py b/services/request_cancellation.py new file mode 100644 index 00000000..f6185726 --- /dev/null +++ b/services/request_cancellation.py @@ -0,0 +1,144 @@ +"""One cleanup owner and content-free outcome for an upstream handoff.""" + +from __future__ import annotations + +import logging +from collections.abc import Callable, Iterator +from dataclasses import dataclass +from typing import Any, Generic, TypeVar + +from flask import Response + +from services.upstream_outcome import UpstreamOutcome, classify_upstream_outcome + +logger = logging.getLogger(__name__) +T = TypeVar("T") + + +@dataclass(frozen=True) +class CancellationOutcome: + reason: str + ambiguous: bool + upstream: UpstreamOutcome + usage_state: str = "unknown" + usage: None = None + + +class CancellationContext: + """Own close without spawning a disconnect watcher or authorizing replay.""" + + def __init__(self, close: Callable[[], Any], *, on_outcome=None): + self._close = close + self._on_outcome = on_outcome + self.handed_off = False + self.closed = False + self.outcome: CancellationOutcome | None = None + + def handoff(self) -> None: + self.handed_off = True + + def _finish(self, reason: str, upstream: UpstreamOutcome) -> None: + if self.closed: + return + self.closed = True + self.outcome = CancellationOutcome( + reason, self.handed_off and reason != "complete", upstream, + ) + try: + self._close() + except Exception as error: + logger.warning("Upstream cleanup failed type=%s", type(error).__name__) + if self._on_outcome is not None: + try: + self._on_outcome(self.outcome) + except Exception as error: + logger.warning("Cancellation observer failed type=%s", type(error).__name__) + + def complete(self) -> None: + self._finish("complete", classify_upstream_outcome(200)) + + def cancel(self) -> None: + self._finish("cancelled", classify_upstream_outcome(cancelled=True)) + + def fail(self) -> None: + self._finish( + "interrupted", classify_upstream_outcome(transport_failure="interrupted"), + ) + + def close(self) -> None: + self.cancel() + + +class CancellationIterator(Iterator[T], Generic[T]): + """A closable lazy iterator, including before its generator has started.""" + + def __init__(self, factory: Callable[[], Iterator[T]], context: CancellationContext, + *, close_source: Callable[[], Any] | None = None): + self._factory = factory + self.context = context + self._source: Iterator[T] | None = None + self._close_source = close_source + self._closed = False + + def __iter__(self) -> CancellationIterator[T]: + return self + + def __next__(self) -> T: + if self._closed or self.context.closed: + raise StopIteration + try: + if self._source is None: + self._source = iter(self._factory()) + return next(self._source) + except StopIteration: + self.context.complete() + self.close() + raise + except GeneratorExit: + self.close() + raise + except BaseException: + self.context.fail() + self.close() + raise + + def throw(self, error): + # GeneratorExit from a downstream owner must never resume the socket. + self.close() + raise error + + def close(self) -> None: + if self._closed: + return + self._closed = True + close = self._close_source or getattr(self._source, "close", None) + try: + if close is not None: + close() + except Exception as error: + logger.warning("Stream cleanup failed type=%s", type(error).__name__) + finally: + self.context.close() + + +def bind_cancellation(response, *, on_outcome=None) -> CancellationContext: + """Share close ownership with existing transport, WSGI and preflight owners. + + Flask bodies must themselves own hidden provider resources. A transport + creating a lazy generator should bind its upstream before building that body. + """ + context = getattr(response, "multillm_cancellation", None) + if isinstance(context, CancellationContext): + return context + # Test doubles and some adapters have no close(); ownership still applies. + close = getattr(response, "close", None) + context = CancellationContext(close if callable(close) else (lambda: None), on_outcome=on_outcome) + context.handoff() + response.multillm_cancellation = context + response.close = context.close + if isinstance(response, Response): + source = response.response + response.response = CancellationIterator( + lambda: iter(source), context, close_source=getattr(source, "close", None), + ) + return context diff --git a/services/upstream_transport.py b/services/upstream_transport.py index a1215320..3669c79e 100644 --- a/services/upstream_transport.py +++ b/services/upstream_transport.py @@ -1,28 +1,23 @@ """Resource-safe helpers for consuming provider HTTP responses.""" -import logging -from typing import Generator +from collections.abc import Iterator import requests from streaming.sse import iter_sse_data - -logger = logging.getLogger(__name__) - +from services.request_cancellation import CancellationIterator, bind_cancellation def close_retry_response(response: requests.Response) -> None: """Release a retryable response without masking the original result.""" - try: - response.close() - except Exception as error: - logger.warning( - "Retry response cleanup failed type=%s", - type(error).__name__, - ) + bind_cancellation(response).close() -def iter_stream_content(response: requests.Response) -> Generator[bytes, None, None]: +def iter_stream_content(response: requests.Response) -> Iterator[bytes]: """Yield decoded upstream bytes as soon as the socket exposes them.""" + return CancellationIterator(lambda: _stream_content(response), bind_cancellation(response)) + + +def _stream_content(response: requests.Response) -> Iterator[bytes]: raw_response = getattr(response, "raw", None) raw_read1 = getattr(raw_response, "read1", None) if callable(raw_read1): @@ -35,21 +30,21 @@ def iter_stream_content(response: requests.Response) -> Generator[bytes, None, N yield from response.iter_content(chunk_size=128) -def iter_stream_lines(response: requests.Response) -> Generator[str, None, None]: +def iter_stream_lines(response: requests.Response) -> Iterator[str]: + """Own the stream before the first read, including preflight rejection.""" + return CancellationIterator(lambda: _stream_lines(response), bind_cancellation(response)) + + +def _stream_lines(response: requests.Response) -> Iterator[str]: """Parse SSE responses incrementally, with a line-based fallback.""" content_type = response.headers.get("content-type", "").lower() if content_type.startswith("text/event-stream") and hasattr( response, "iter_content", ): - try: - for data_payload in iter_sse_data(iter_stream_content(response)): - yield f"data: {data_payload}" - return - except Exception as error: - logger.warning( - "SSE parser failed; falling back to line iteration type=%s", - type(error).__name__, - ) + # A failed socket read is an interrupted handoff, not a fresh read path. + for data_payload in iter_sse_data(_stream_content(response)): + yield f"data: {data_payload}" + return yield from response.iter_lines(decode_unicode=True) diff --git a/tests/fixtures/secret_firewall_coverage.json b/tests/fixtures/secret_firewall_coverage.json index ff0fe87b..07648de7 100644 --- a/tests/fixtures/secret_firewall_coverage.json +++ b/tests/fixtures/secret_firewall_coverage.json @@ -345,6 +345,7 @@ "worker/shadow-eval-schedule.mjs", "worker/sse-heartbeat.mjs", "worker/status-page.mjs", + "worker/upstream-cancellation.mjs", "worker/usage-ledger-d1.mjs" ], "edge_route_literals": { diff --git a/tests/helpers/load_cloudflare_worker.mjs b/tests/helpers/load_cloudflare_worker.mjs index 21205186..14153f3c 100644 --- a/tests/helpers/load_cloudflare_worker.mjs +++ b/tests/helpers/load_cloudflare_worker.mjs @@ -392,6 +392,7 @@ export async function roleplayModuleUrl() { ); const transportDataUrl = dataModuleUrl( transportSource + .replace('from "../upstream-cancellation.mjs";', `from "${new URL("../../worker/upstream-cancellation.mjs", import.meta.url)}";`) .replace('from "../secret-firewall.mjs";', `from "${new URL("../../worker/secret-firewall.mjs", import.meta.url)}";`) .replace('from "./compaction-budget.mjs";', `from "${compactionBudgetUrl}";`) .replace( diff --git a/tests/test_request_cancellation.py b/tests/test_request_cancellation.py new file mode 100644 index 00000000..d3ec92e7 --- /dev/null +++ b/tests/test_request_cancellation.py @@ -0,0 +1,164 @@ +"""Cancellation ownership with synthetic HTTP responses only.""" + +from unittest.mock import Mock + +import pytest +import requests +from flask import Flask, Response + +from routes.unified_transport import send_unified_provider_request +from services.request_cancellation import CancellationContext, bind_cancellation +from services.stream_preflight import preflight_chat_stream +from services.upstream_transport import ( + close_retry_response, iter_stream_content, iter_stream_lines, +) + + +class Upstream: + status_code = 200 + + def __init__(self, chunks=(), error=None): + self.headers = {"content-type": "text/event-stream"} + self.chunks = chunks + self.error = error + self.closes = 0 + self.reads = 0 + + def iter_content(self, chunk_size): + for chunk in self.chunks: + self.reads += 1 + yield chunk + if self.error: + raise self.error + + def iter_lines(self, decode_unicode=True): + raise AssertionError("Socket failures must not restart stream consumption") + + def close(self): + self.closes += 1 + + +@pytest.mark.parametrize("started", [False, True]) +def test_close_before_or_after_first_byte_is_ambiguous_once(started): + upstream = Upstream([b"first", b"second"]) + observed = Mock() + context = bind_cancellation(upstream, on_outcome=observed) + body = iter_stream_content(upstream) + if started: + assert next(body) == b"first" + body.close() + upstream.close() + close_retry_response(upstream) + assert upstream.closes == observed.call_count == 1 + assert context.outcome.ambiguous + assert context.outcome.usage_state == "unknown" + assert context.outcome.usage is None + assert not context.outcome.upstream.replay_permission + assert context.outcome.upstream.credential_health == "unknown" + assert upstream.reads == int(started) + + +def test_generator_exit_closes_once(): + upstream = Upstream([b"first", b"second"]) + body = iter_stream_content(upstream) + next(body) + with pytest.raises(GeneratorExit): + body.throw(GeneratorExit) + body.close() + upstream.close() + assert upstream.closes == 1 + + +def test_socket_timeout_closes_without_line_fallback(): + upstream = Upstream(error=requests.ReadTimeout("synthetic timeout")) + with pytest.raises(requests.ReadTimeout): + list(iter_stream_lines(upstream)) + upstream.close() + assert upstream.closes == 1 + assert upstream.multillm_cancellation.outcome.ambiguous + assert not upstream.multillm_cancellation.outcome.upstream.replay_permission + + +def test_completed_raw_bytes_are_unchanged(): + chunks = [b": ping\r\n\r\n", b"data: raw\n\n", b"\x00\xff"] + upstream = Upstream(chunks) + assert list(iter_stream_content(upstream)) == chunks + upstream.close() + assert upstream.closes == 1 + assert not upstream.multillm_cancellation.outcome.ambiguous + assert upstream.multillm_cancellation.outcome.reason == "complete" + + +@pytest.mark.parametrize("chunks", [[], [b"data: [DONE]\n\n"]]) +def test_preflight_rejection_closes_upstream_once(chunks): + upstream = Upstream(chunks) + response = Response(iter_stream_content(upstream), content_type="text/event-stream") + response.call_on_close(upstream.close) + result = preflight_chat_stream(response) + assert not result.validated + assert result.response.status_code == 502 + result.response.close() + upstream.close() + assert upstream.closes == 1 + + +@pytest.mark.parametrize("started", [False, True]) +def test_unified_flask_handoff_closes_owned_body_before_or_after_iteration(started): + upstream = Upstream([b"first", b"second"]) + body = iter_stream_content(upstream) + response = Response(body, headers={"X-Request-ID": "synthetic"}) + proxy = Mock() + proxy.make_request.return_value = response + result = send_unified_provider_request( + proxy, {"method": "POST", "data": b"raw"}, provider="openai", + primary_origin="https://example.invalid", secondary_origin=None, + upstream_path="v1/chat/completions", request_headers={}, + ) + if started: + assert next(iter(result.response)) == b"first" + result.close() + result.close() + assert upstream.closes == 1 + assert result.headers["X-Request-ID"] == "synthetic" + assert result.multillm_cancellation.outcome.ambiguous + + +def test_registered_wsgi_response_close_owns_unstarted_stream(): + app = Flask(__name__) + upstream = Upstream([b"one", b"two"]) + + @app.route("/synthetic") + def synthetic(): + proxy = Mock() + proxy.make_request.return_value = Response(iter_stream_content(upstream)) + return send_unified_provider_request( + proxy, {"method": "POST"}, provider="openai", + primary_origin="https://example.invalid", secondary_origin=None, + upstream_path="v1/chat/completions", request_headers={}, + ) + + response = app.test_client().get("/synthetic", buffered=False) + response.close() + assert upstream.closes == 1 + + +def test_cleanup_and_observer_failures_do_not_mask_disconnect(caplog): + upstream = Upstream() + upstream.close = Mock(side_effect=RuntimeError("private content")) + original_close = upstream.close + context = bind_cancellation(upstream, on_outcome=Mock(side_effect=ValueError("private content"))) + context.close() + context.close() + assert original_close.call_count == 1 + assert "private content" not in caplog.text + + +def test_cancellation_before_handoff_has_no_usage_claim_or_replay(): + close = Mock() + context = CancellationContext(close) + context.cancel() + context.cancel() + assert close.call_count == 1 + assert not context.outcome.ambiguous + assert context.outcome.usage is None + assert not context.outcome.upstream.replay_permission diff --git a/tests/test_request_cancellation_worker.mjs b/tests/test_request_cancellation_worker.mjs new file mode 100644 index 00000000..b110ba35 --- /dev/null +++ b/tests/test_request_cancellation_worker.mjs @@ -0,0 +1,202 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { UpstreamCancellation } from "../worker/upstream-cancellation.mjs"; +import { attemptRoleplayCandidates, readBoundedBytes } from "../worker/roleplay/transport.mjs"; + +async function withGlobalFetch(fetchImpl, callback) { + const previous = globalThis.fetch; + globalThis.fetch = fetchImpl; + try { return await callback(); } finally { globalThis.fetch = previous; } +} + +const candidate = { + key: "synthetic", provider: "nanogpt", model: "synthetic", + token: "synthetic", endpoint: "https://example.invalid", +}; +const settings = { upstreamHeaderTimeoutMs: 100, preResponseFallbackEnabled: true }; + +function pendingBody() { + let cancelled = 0; + let streamController; + const body = new ReadableStream({ + start(controller) { streamController = controller; }, + cancel() { cancelled += 1; }, + }, { highWaterMark: 0 }); + return { body, get cancelled() { return cancelled; }, get controller() { return streamController; } }; +} + +for (const started of [false, true]) { + test(`abort after headers closes once, reader started=${started}`, async () => { + const client = new AbortController(); + const outcomes = []; + const owner = new UpstreamCancellation({ signal: client.signal, onOutcome: outcome => outcomes.push(outcome) }); + const upstream = pendingBody(); + owner.handoff(); + const response = owner.wrapResponse(new Response(upstream.body)); + const reader = response.body.getReader(); + const read = started ? reader.read() : null; + client.abort("synthetic disconnect"); + if (read) await assert.rejects(read, { name: "AbortError" }); + await assert.rejects(reader.read(), { name: "AbortError" }); + owner.cleanup(); + await owner.close(); + assert.equal(upstream.cancelled, 1); + assert.equal(owner.controller.signal.aborted, true); + assert.equal(outcomes.length, 1); + assert.equal(outcomes[0].ambiguous, true); + assert.equal(outcomes[0].usage, null); + assert.equal(outcomes[0].usageState, "unknown"); + assert.equal(outcomes[0].replayPermission, false); + }); +} + +test("downstream body cancellation aborts fetch even without Request.signal", async () => { + const owner = new UpstreamCancellation(); + owner.handoff(); + const upstream = pendingBody(); + const response = owner.wrapResponse(new Response(upstream.body)); + await response.body.cancel(); + await owner.close(); + assert.equal(upstream.cancelled, 1); + assert.equal(owner.controller.signal.aborted, true); + assert.equal(owner.outcome.ambiguous, true); +}); + +test("successful bytes and headers survive and late abort is inert", async () => { + const client = new AbortController(); + const owner = new UpstreamCancellation({ signal: client.signal }); + const bytes = new Uint8Array([0, 255, 13, 10, 37]); + owner.handoff(); + const response = owner.wrapResponse(new Response(bytes, { status: 201, headers: { "X-Request-ID": "synthetic" } })); + assert.deepEqual(new Uint8Array(await response.arrayBuffer()), bytes); + assert.equal(response.status, 201); + assert.equal(response.headers.get("X-Request-ID"), "synthetic"); + owner.cleanup(); + client.abort(); + assert.equal(owner.controller.signal.aborted, false); + assert.equal(owner.outcome.reason, "complete"); + assert.equal(owner.outcome.ambiguous, false); +}); + +test("bounded reader abort and size rejection close pending reads once", async () => { + const client = new AbortController(); + const upstream = pendingBody(); + const reading = readBoundedBytes(upstream.body, 10, client.signal); + client.abort(); + await assert.rejects(reading, { name: "AbortError" }); + assert.equal(upstream.cancelled, 1); + assert.equal(upstream.body.locked, false); + const oversized = pendingBody(); + oversized.controller.enqueue(new Uint8Array(11)); + await assert.rejects(readBoundedBytes(oversized.body, 10), /response limit/); + assert.equal(oversized.cancelled, 1); +}); + +test("already aborted candidate dispatch does not call provider", async () => { + const controller = new AbortController(); + controller.abort(); + const candidate = { key: "synthetic", provider: "nanogpt", model: "synthetic", endpoint: "https://example.invalid" }; + let calls = 0; + const result = await withGlobalFetch(async () => { calls += 1; throw new Error("unexpected fetch"); }, () => + attemptRoleplayCandidates({ stats: {} }, [candidate], () => ({ model: "synthetic" }), {}, + { upstreamHeaderTimeoutMs: 100, preResponseFallbackEnabled: true }, controller.signal, "")); + assert.equal(calls, 0); + assert.equal(result.terminalResponse.status, 499); +}); + +test("roleplay abort racing HTTP rejection never falls back", async () => { + const client = new AbortController(); + let calls = 0; + const response = await withGlobalFetch(async () => { + calls += 1; + client.abort(); + return Response.json({ error: { code: "synthetic" } }, { status: 429 }); + }, () => attemptRoleplayCandidates({ stats: {} }, [candidate, candidate], + () => ({ model: "synthetic" }), {}, settings, client.signal, "")); + assert.equal(calls, 1); + assert.equal(response.terminalResponse.status, 499); +}); + +test("header timeout aborts and is bounded without a body", async () => { + const owner = new UpstreamCancellation({ headerTimeoutMs: 1 }); + owner.handoff(); + await new Promise(resolve => owner.controller.signal.addEventListener("abort", resolve, { once: true })); + assert.equal(owner.controller.signal.reason, "upstream_header_timeout"); + assert.equal(owner.outcome.ambiguous, true); + assert.equal(owner.outcome.replayPermission, false); +}); + +test("in-flight provider fetch sees client abort and never advances", async () => { + const client = new AbortController(); + const outcomes = []; + let calls = 0; + let started; + const connected = new Promise(resolve => { started = resolve; }); + const requesting = withGlobalFetch(async (_input, init) => { + calls += 1; + started(); + return new Promise((resolve, reject) => init.signal.addEventListener("abort", () => { + assert.equal(init.signal.reason, "synthetic disconnect"); + reject(new DOMException("Aborted", "AbortError")); + }, { once: true })); + }, () => attemptRoleplayCandidates({ stats: {} }, [candidate, candidate], + () => ({ model: "synthetic" }), {}, { ...settings, onCancellationOutcome: value => outcomes.push(value) }, client.signal, "")); + await connected; + client.abort("synthetic disconnect"); + const result = await requesting; + assert.equal(result.terminalResponse.status, 499); + assert.equal(calls, 1); + assert.equal(outcomes.length, 1); + assert.equal(outcomes[0].ambiguous, true); + assert.equal(outcomes[0].usage, null); + assert.equal(outcomes[0].replayPermission, false); +}); + +test("roleplay disconnect after first byte closes and preserves unknown usage", async () => { + const client = new AbortController(); + const upstream = pendingBody(); + upstream.controller.enqueue(new Uint8Array([1, 2])); + const outcomes = []; + let calls = 0; + const result = await withGlobalFetch(async () => { + calls += 1; + return new Response(upstream.body); + }, () => attemptRoleplayCandidates({ stats: {} }, [candidate, candidate], + () => ({ model: "synthetic" }), {}, { ...settings, onCancellationOutcome: value => outcomes.push(value) }, client.signal, "")); + const reader = result.response.body.getReader(); + assert.deepEqual((await reader.read()).value, new Uint8Array([1, 2])); + client.abort(); + await assert.rejects(reader.read(), { name: "AbortError" }); + result.cleanup(); + assert.equal(calls, 1); + assert.equal(upstream.cancelled, 1); + assert.equal(outcomes.length, 1); + assert.equal(outcomes[0].usage, null); + assert.equal(outcomes[0].ambiguous, true); +}); + +test("cancellation does not await a provider acknowledgement", async () => { + const owner = new UpstreamCancellation(); + owner.handoff(); + let cancelled = 0; + const response = owner.wrapResponse(new Response(new ReadableStream({ + cancel() { cancelled += 1; return new Promise(() => {}); }, + }, { highWaterMark: 0 }))); + await response.body.cancel(); + await owner.close(); + assert.equal(cancelled, 1); +}); + +test("terminal provider rejection keeps bytes and abort ownership", async () => { + const client = new AbortController(); + const body = '{"error":{"code":"synthetic"}}'; + const result = await withGlobalFetch(async () => new Response(body, { + status: 429, headers: { "Retry-After": "17" }, + }), () => attemptRoleplayCandidates({ stats: {} }, [candidate], + () => ({ model: "synthetic" }), {}, settings, client.signal, "")); + assert.equal(result.terminalResponse.status, 429); + assert.equal(await result.terminalResponse.text(), body); + assert.equal(result.terminalResponse.headers.get("Retry-After"), "17"); + client.abort(); +}); diff --git a/worker/roleplay/transport.mjs b/worker/roleplay/transport.mjs index 817c950b..6ebbb863 100644 --- a/worker/roleplay/transport.mjs +++ b/worker/roleplay/transport.mjs @@ -1,3 +1,4 @@ +import { UpstreamCancellation, readBoundedUpstreamBytes } from "../upstream-cancellation.mjs"; import { firewallFetch, isSecretScanBlock } from "../secret-firewall.mjs"; import { buildProviderHeaders, @@ -121,62 +122,10 @@ export async function unwrapClineEnvelope(response) { } export async function readBoundedBytes(stream, maximumBytes, signal) { - if (!stream) { - return { bytes: new Uint8Array(), firstByteMs: 0 }; - } - - const reader = stream.getReader(); - const chunks = []; - let size = 0; - let firstByteAt = 0; - const startedAt = performance.now(); - const abort = () => { void reader.cancel().catch(() => {}); }; - signal?.addEventListener("abort", abort, { once: true }); - - try { - while (true) { - if (signal?.aborted) { - throw new DOMException("Request aborted", "AbortError"); - } - const { value, done } = await reader.read(); - if (signal?.aborted) { - throw new DOMException("Request aborted", "AbortError"); - } - if (done) { - break; - } - if (!value) { - continue; - } - if (!firstByteAt) { - firstByteAt = performance.now(); - } - const chunk = value instanceof Uint8Array ? value : new Uint8Array(value); - size += chunk.byteLength; - if (size > maximumBytes) { - await reader.cancel("Response exceeded configured limit"); - throw new RoleplayRequestError( - "Upstream response exceeded the roleplay response limit", - 502, - ); - } - chunks.push(chunk); - } - } finally { - signal?.removeEventListener("abort", abort); - reader.releaseLock(); - } - - const bytes = new Uint8Array(size); - let offset = 0; - for (const chunk of chunks) { - bytes.set(chunk, offset); - offset += chunk.byteLength; - } - return { - bytes, - firstByteMs: firstByteAt ? firstByteAt - startedAt : 0, - }; + return readBoundedUpstreamBytes(stream, maximumBytes, signal, () => + new RoleplayRequestError( + "Upstream response exceeded the roleplay response limit", 502, + )); } export function copyUpstreamResponseHeaders(headers) { @@ -223,7 +172,13 @@ async function handleCandidateHttpFailure( const explicitProviderError = settings.providerErrorFallbackEnabled ? await classifyExplicitProviderError(attempted.response) : null; - attempted.cleanup(); + if (attempted.cancellation?.parentSignal?.aborted) { + attempted.cleanup(); + return handleCandidatePreResponseFailure( + state, candidate, new RoleplayPreResponseFailure("client_abort"), + settings, false, fallbackCount, + ); + } const safeFallback = isSafeFallbackStatus(attempted.response.status) || @@ -241,7 +196,7 @@ async function handleCandidateHttpFailure( ); if (safeFallback && hasFallbackCandidate) { - await attempted.response.body?.cancel(); + attempted.cleanup(); return { state: nextState, fallbackCount: fallbackCount + 1, @@ -596,21 +551,15 @@ async function fetchCandidateWithPaygoFallback( } async function fetchCandidate(candidate, payload, env, settings, signal, key) { - const controller = new AbortController(); - const abort = () => controller.abort(signal?.reason); - let cleaned = false; - if (signal?.aborted) { - abort(); - } else { - signal?.addEventListener("abort", abort, { once: true }); - } - const timeout = setTimeout( - () => controller.abort("upstream_header_timeout"), - settings.upstreamHeaderTimeoutMs, - ); + const cancellation = new UpstreamCancellation({ + signal, headerTimeoutMs: settings.upstreamHeaderTimeoutMs, + onOutcome: settings.onCancellationOutcome, + }); + const { controller } = cancellation; const startedAt = performance.now(); try { + cancellation.throwIfAborted(); const headers = buildProviderHeaders(candidate, env, key, settings.clientHeaders); const containerNamespace = env.MULTILLM_PROXY_CONTAINER; const useOpenCodeContainer = @@ -628,6 +577,7 @@ async function fetchCandidate(candidate, payload, env, settings, signal, key) { redirect: "manual", signal: controller.signal, }; + cancellation.handoff(); const response = useOpenCodeContainer ? await containerNamespace.getByName("primary").fetch( new Request( @@ -640,28 +590,23 @@ async function fetchCandidate(candidate, payload, env, settings, signal, key) { onDecision: settings.onSecretScan, principal: { id: env.ADMIN_USERNAME || "admin" }, }, async checked => fetch(checked.url, { ...requestInit, headers: checked.headers, body: await checked.text() })); - clearTimeout(timeout); + cancellation.headersReceived(); + const ownedResponse = isSecretScanBlock(response) + ? response : cancellation.wrapResponse(response); + if (isSecretScanBlock(response)) cancellation.complete(); + cancellation.throwIfAborted(); const headerMs = performance.now() - startedAt; return { - response: - candidate.provider === "cline-pass" - ? await unwrapClineEnvelope(response) - : response, + response: candidate.provider === "cline-pass" + ? await unwrapClineEnvelope(ownedResponse) : ownedResponse, + cancellation, controller, startedAt, headerMs, - cleanup() { - if (cleaned) { - return; - } - cleaned = true; - clearTimeout(timeout); - signal?.removeEventListener("abort", abort); - }, + cleanup: () => cancellation.cleanup(), }; } catch (error) { - clearTimeout(timeout); - signal?.removeEventListener("abort", abort); + cancellation.cleanup(); const failureKind = signal?.aborted ? "client_abort" : controller.signal.reason === "compaction_timeout" @@ -777,20 +722,13 @@ export async function requestCompaction( settings, signal, ) { - const compactionController = new AbortController(); - const forwardAbort = () => - compactionController.abort(signal?.reason); - if (signal?.aborted) { - forwardAbort(); - } else { - signal?.addEventListener("abort", forwardAbort, { once: true }); - } + const compaction = new UpstreamCancellation({ + signal, headerTimeoutMs: settings.compactionTimeoutMs, + timeoutReason: "compaction_timeout", + }); + const compactionController = compaction.controller; const startedAt = performance.now(); const deadlineAt = startedAt + settings.compactionTimeoutMs; - const deadline = setTimeout( - () => compactionController.abort("compaction_timeout"), - settings.compactionTimeoutMs, - ); let fallbackCount = 0; try { const compactionCandidates = prepareCompactionCandidates( @@ -857,10 +795,8 @@ export async function requestCompaction( ); if (isSafeFallbackStatus(response.status)) { fallbackCount += 1; - await response.body?.cancel(); continue; } - await response.body?.cancel(); throw new Error( "Compaction provider returned an ambiguous failure", ); @@ -893,8 +829,7 @@ export async function requestCompaction( } throw new Error("No configured model accepted memory compaction"); } finally { - clearTimeout(deadline); - signal?.removeEventListener("abort", forwardAbort); + compaction.cleanup(); } } diff --git a/worker/upstream-cancellation.mjs b/worker/upstream-cancellation.mjs new file mode 100644 index 00000000..7ecbedc4 --- /dev/null +++ b/worker/upstream-cancellation.mjs @@ -0,0 +1,160 @@ +// One owner for request abort, header timeout and the upstream response body. +export class UpstreamCancellation { + constructor({ signal, headerTimeoutMs, timeoutReason = "upstream_header_timeout", onOutcome } = {}) { + this.controller = new AbortController(); + this.parentSignal = signal; + this.onOutcome = onOutcome; + this.handedOff = false; + this.settled = false; + this.reader = null; + this.bodyController = null; + this.outcome = null; + this.cancelPromise = null; + this.forwardAbort = () => this.controller.abort(signal.reason); + this.handleAbort = () => { void this.close(this.controller.signal.reason); }; + this.controller.signal.addEventListener("abort", this.handleAbort, { once: true }); + signal?.addEventListener("abort", this.forwardAbort, { once: true }); + if (signal?.aborted) this.forwardAbort(); + if (!this.settled && Number.isFinite(headerTimeoutMs) && headerTimeoutMs > 0) { + this.timeout = setTimeout(() => this.controller.abort(timeoutReason), headerTimeoutMs); + } + } + + handoff() { this.handedOff = true; } + + headersReceived() { clearTimeout(this.timeout); } + + throwIfAborted() { + if (this.controller.signal.aborted) throw new DOMException("Request aborted", "AbortError"); + } + + finish(reason) { + if (this.settled) return; + this.settled = true; + clearTimeout(this.timeout); + this.parentSignal?.removeEventListener("abort", this.forwardAbort); + this.controller.signal.removeEventListener("abort", this.handleAbort); + this.outcome = Object.freeze({ + reason, ambiguous: this.handedOff && reason !== "complete", + usageState: "unknown", usage: null, replayPermission: false, + }); + try { this.onOutcome?.(this.outcome); } catch { + console.warn("Upstream cancellation observer failed"); + } + } + + releaseReader() { + if (this.reader) { + this.reader.releaseLock(); + this.reader = null; + } + } + + complete() { + this.finish("complete"); + this.releaseReader(); + } + + close(reason = "cancelled") { + if (this.settled) return Promise.resolve(); + this.finish(reason === "upstream_header_timeout" ? reason : "cancelled"); + if (!this.controller.signal.aborted) this.controller.abort(reason); + // Settle the downstream read without waiting for a remote acknowledgement. + this.bodyController?.error(new DOMException("Request aborted", "AbortError")); + if (this.reader) { + try { this.cancelPromise = Promise.resolve(this.reader.cancel(reason)).catch(() => {}); } + catch { this.cancelPromise = Promise.resolve(); } + this.releaseReader(); + } + return Promise.resolve(); + } + + cleanup() { void this.close(); } + + wrapResponse(response) { + this.headersReceived(); + if (!response.body) { + this.complete(); + return response; + } + const reader = response.body.getReader(); + this.reader = reader; + const owner = this; + const body = new ReadableStream({ + start(controller) { owner.bodyController = controller; }, + async pull(controller) { + try { + owner.throwIfAborted(); + const { value, done } = await reader.read(); + if (owner.settled) return; + owner.throwIfAborted(); + if (done) { + controller.close(); + owner.complete(); + } else controller.enqueue(value); + } catch (error) { + if (!owner.settled) { + controller.error(error); + owner.bodyController = null; + void owner.close("interrupted"); + } + } + }, + cancel(reason) { return owner.close(reason); }, + }, { highWaterMark: 0 }); + const wrapped = new Response(body, { + status: response.status, statusText: response.statusText, headers: response.headers, + }); + if (owner.settled) { + // An abort can race a fetch implementation returning its response. + owner.bodyController.error(new DOMException("Request aborted", "AbortError")); + owner.cancelPromise = Promise.resolve(reader.cancel()).catch(() => {}); + owner.releaseReader(); + } + return wrapped; + } +} + +export async function readBoundedUpstreamBytes(stream, maximumBytes, signal, limitError) { + if (!stream) return { bytes: new Uint8Array(), firstByteMs: 0 }; + const reader = stream.getReader(); + const chunks = []; + let size = 0; + let firstByteAt = 0; + let cancelled = false; + let complete = false; + const startedAt = performance.now(); + const cancel = (reason) => { + if (cancelled || complete) return; + cancelled = true; + // A provider may never acknowledge cancellation. Do not wait on it. + void reader.cancel(reason).catch(() => {}); + }; + const abort = () => cancel(signal.reason); + const checkAbort = () => { + if (signal?.aborted) throw new DOMException("Request aborted", "AbortError"); + }; + signal?.addEventListener("abort", abort, { once: true }); + try { + while (true) { + checkAbort(); + const { value, done } = await reader.read(); + checkAbort(); + if (done) { complete = true; break; } + if (!value) continue; + if (!firstByteAt) firstByteAt = performance.now(); + const chunk = value instanceof Uint8Array ? value : new Uint8Array(value); + size += chunk.byteLength; + if (size > maximumBytes) throw limitError(); + chunks.push(chunk); + } + } finally { + signal?.removeEventListener("abort", abort); + if (!complete) cancel("incomplete_read"); + reader.releaseLock(); + } + const bytes = new Uint8Array(size); + let offset = 0; + for (const chunk of chunks) { bytes.set(chunk, offset); offset += chunk.byteLength; } + return { bytes, firstByteMs: firstByteAt ? firstByteAt - startedAt : 0 }; +}