From cf05eaa1a24d2078cc50f5dbd2c0a33391aafb3f Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 20 Nov 2026 04:59:39 +0000 Subject: [PATCH 1/2] Cut worker and result response bytes: capabilities digest, history-free result (#352) Worker-plane requests opt in to the server capabilities digest (durable-workflow/server#1262): they send X-Durable-Workflow-Server-Capabilities-Digest with the digest held when the request is sent (or "none"), and the answer, success or error body and negotiated poll envelopes included, is resolved against that captured pair up to three levels deep. A tagged full copy is learned for later requests and its digest member stripped; an unknown digest raises ServerCapabilitiesDigestError. A request in flight across a rolling deploy restores the object it was sent with. get_result returns a completed run's output from the status read's output_envelope when it names the same run, skipping the history read; non-success outcomes and envelope-less answers keep the history path. gzip needs no change: httpx advertises and decodes it (test added). Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 11 + src/durable_workflow/_capabilities_digest.py | 84 +++++++ src/durable_workflow/client.py | 31 ++- src/durable_workflow/errors.py | 8 + tests/test_capabilities_digest.py | 230 +++++++++++++++++++ 5 files changed, 361 insertions(+), 3 deletions(-) create mode 100644 src/durable_workflow/_capabilities_digest.py create mode 100644 tests/test_capabilities_digest.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e9027b1..39b8412 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Replay safety:** every worker of this release follows a marker a run already recorded, whether or not recording is on, for every workflow task, query and update. A marked run needs no per-group decision, because the first failure a task sees in append-only history is the first every later replay sees. A run whose first such group was recorded without the marker keeps the earlier rule for its whole life. Its settled groups raise the lowest-position failure, and a failure raised while siblings are open still records the #338 decision. This holds even when a task recorded no command after catching the failure, or another worker answered an update in between. A run with a cancellation request when it reaches its first group records no marker. - **Regression evidence:** fixtures recorded by published 2.19.1 on a Server built from main replay under the earlier rule. They cover a settled group in flight, a catch followed by a task that recorded no command, an update answered by 2.19.1 before or between later tasks, and a select() cancellation. Marked runs recorded on native and PHP Server replay under the new rule without the setting. New fixtures cover the first task with recording on and off. Existing fixtures, including `parallel-group-metadata-emission`, the completion-limit fixtures and `business-clock-parallel-failure-selection-gap`, are unchanged. +### Changed + +- Workers send far fewer response bytes on a Server that supports the capabilities digest (#352, durable-workflow/server#1262). + - **The header:** worker-plane requests now send `X-Durable-Workflow-Server-Capabilities-Digest` (`none` until the first full copy arrives). + - **The Server's answer:** the Server replaces each repeated `server_capabilities` object (about 17 KB on native 3.15/3.16) with its digest, or sends a full copy tagged with a `digest` member. + - **What callers see:** the SDK restores elided copies before any caller sees the answer, in success bodies, error bodies and negotiated poll envelopes, up to three levels deep. It strips the `digest` member, so callers still see complete capabilities. + - **Overlapping requests:** each answer is resolved against the digest and object its own request was sent with. A request in flight during a rolling deploy that changes the capabilities restores the object it was sent with, while later requests use the newer digest. + - **Errors:** a digest the worker does not hold raises the new `ServerCapabilitiesDigestError`, a protocol failure. + - **Older Servers** ignore the header and answer in full as before. +- `WorkflowHandle.result()` and `Client.get_result()` no longer read the run's history for a completed run when the status read already carries that run's `output_envelope`. The status read is `GET /workflows/{id}`, which the result poll makes anyway. Failed, cancelled, terminated and timed-out runs still read the history for their typed failure, and so does a completed run whose status read has no `output_envelope` or names a different run. + ## [2.20.0] - 2026-11-19 Both additions need **native Server 3.15.0 or later**, the first Server release that advertises `capabilities.signal_with_start` and the `signal_external_workflow` worker command. Each is discovered from the Server's capabilities and refused before anything is sent on an older Server, so the SDK's supported Server baselines are unchanged and every other feature keeps working on older Servers. Worker protocol (1.20) and control-plane (2) versions are unchanged. diff --git a/src/durable_workflow/_capabilities_digest.py b/src/durable_workflow/_capabilities_digest.py new file mode 100644 index 0000000..9c07b78 --- /dev/null +++ b/src/durable_workflow/_capabilities_digest.py @@ -0,0 +1,84 @@ +"""Worker answers without repeated ``server_capabilities`` (server#1262). + +A worker request sends the digest of the capabilities this client holds, or +``none``. A Server that supports the digest answers with each copy of its +current capabilities either tagged with a ``digest`` member, which is kept for +later requests, or replaced by the digest string, which is restored here before +any caller sees the answer. Older Servers ignore the header and answer in full, +so callers always observe complete capabilities. + +The Server only replaces copies with the digest that request sent, so an answer +is resolved against the pair captured when its request was sent, not the pair +held when it arrives. Overlapping requests across a change of capabilities (a +rolling deploy) each restore their own object. Contract: +``docs/contracts/server-capabilities-digest.md`` in the Server repository. +""" + +from __future__ import annotations + +import copy +from typing import Any + +from .errors import ServerCapabilitiesDigestError + +HEADER = "X-Durable-Workflow-Server-Capabilities-Digest" +_KEY = "server_capabilities" +_PREFIX = "sha256:" +# The Server elides copies at the top level and up to two objects deep. +_MAX_DEPTH = 3 + +Captured = tuple[str | None, dict[str, Any] | None] + + +class ServerCapabilitiesDigest: + """The newest (digest, capabilities) pair this client has learned.""" + + def __init__(self) -> None: + self._digest: str | None = None + self._capabilities: dict[str, Any] | None = None + + def capture(self) -> Captured: + """The pair a request is sent with; pass it to :meth:`restore` for its answer.""" + return self._digest, self._capabilities + + @staticmethod + def header(captured: Captured) -> str: + return captured[0] if captured[0] is not None and captured[1] is not None else "none" + + def restore(self, answer: Any, captured: Captured) -> None: + """Strip and learn tagged copies, then restore elided ones, in place.""" + if not isinstance(answer, dict): + return + learned: dict[str, dict[str, Any]] = {} + self._learn(answer, learned, 1) + self._resolve(answer, captured, learned, 1) + + def _learn(self, answer: dict[str, Any], learned: dict[str, dict[str, Any]], depth: int) -> None: + for key, value in answer.items(): + if key == _KEY: + if isinstance(value, dict) and isinstance(value.get("digest"), str) \ + and value["digest"].startswith(_PREFIX): + digest = value.pop("digest") + learned[digest] = value + # Later requests send the newest digest this client has seen. + # Callers own the answer, so the held pair is a private copy. + self._digest, self._capabilities = digest, copy.deepcopy(value) + elif isinstance(value, dict) and depth < _MAX_DEPTH: + self._learn(value, learned, depth + 1) + + def _resolve(self, answer: dict[str, Any], captured: Captured, learned: dict[str, dict[str, Any]], + depth: int) -> None: + for key, value in answer.items(): + if key == _KEY: + if isinstance(value, str) and value.startswith(_PREFIX): + if value == captured[0] and captured[1] is not None: + answer[key] = copy.deepcopy(captured[1]) + elif value in learned: + answer[key] = copy.deepcopy(learned[value]) + else: + raise ServerCapabilitiesDigestError( + "the Server elided server_capabilities this worker does not hold " + f"({value}; sent {self.header(captured)})" + ) + elif isinstance(value, dict) and depth < _MAX_DEPTH: + self._resolve(value, captured, learned, depth + 1) diff --git a/src/durable_workflow/client.py b/src/durable_workflow/client.py index 5652bc5..3ba5f27 100644 --- a/src/durable_workflow/client.py +++ b/src/durable_workflow/client.py @@ -33,6 +33,7 @@ import warnings from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass +from dataclasses import field as dataclass_field from importlib.metadata import PackageNotFoundError from importlib.metadata import version as _pkg_version from typing import Any, cast @@ -49,6 +50,8 @@ CancellationScopeDeliveryReceipt, scope_timestamp, ) +from ._capabilities_digest import HEADER as CAPABILITIES_DIGEST_HEADER +from ._capabilities_digest import ServerCapabilitiesDigest from ._cooperative_cancellation import CancellationDelivery, CancellationRequest from ._failure_reporting import ( DETAIL_BYTES, @@ -486,6 +489,8 @@ class WorkflowExecution: payload_codec: str | None = None memo: dict[str, Any] | None = None search_attributes: dict[str, Any] | None = None + # Whether ``output`` was decoded from the status read's ``output_envelope``. + _output_from_envelope: bool = dataclass_field(default=False, init=False, repr=False, compare=False) @dataclass @@ -1988,6 +1993,7 @@ def __init__( ) self._cluster_info: dict[str, Any] | None = None self._cluster_info_lock = asyncio.Lock() + self._capabilities_digest = ServerCapabilitiesDigest() self._runtime_external_payload_transport_cache: ( _RuntimeExternalPayloadTransport | None ) = None @@ -2242,6 +2248,8 @@ async def _request( status_code = "none" outcome = "pending" request_started = time.monotonic() + # The digest pair this request is sent with; its answer resolves against it. + capabilities_digest = self._capabilities_digest.capture() if worker else None async def _do_request() -> httpx.Response: nonlocal request_started @@ -2250,6 +2258,8 @@ async def _do_request() -> httpx.Response: if worker and activity_authority is not None and activity_authority.governs(self, self.namespace, path): request_timeout = min(request_timeout, activity_authority.remaining()) headers = self._discovery_headers() if discovery else self._headers(worker=worker) + if capabilities_digest is not None: + headers[CAPABILITIES_DIGEST_HEADER] = ServerCapabilitiesDigest.header(capabilities_digest) if progress_poll: headers["Accept"] = _POLL_ACCEPT pending = self._http.request( @@ -2290,6 +2300,8 @@ async def _do_request() -> httpx.Response: body = exc.response.json() except ValueError: body = exc.response.text + if capabilities_digest is not None: + self._capabilities_digest.restore(body, capabilities_digest) # The typed completion-size refusal carries the limit; the # size is the encoded request body this client sent. if raise_for_status is _raise_for_status: @@ -2305,6 +2317,8 @@ async def _do_request() -> httpx.Response: outcome = "ok" return None result = resp.json() + if capabilities_digest is not None: + self._capabilities_digest.restore(result, capabilities_digest) if (worker and activity_authority is not None and activity_authority.governs(self, self.namespace, path) and path.endswith("/heartbeat") and isinstance(result, dict) @@ -4131,6 +4145,7 @@ async def describe_workflow(self, workflow_id: str) -> WorkflowExecution: data = await self._request("GET", f"/workflows/{workflow_id}", context=workflow_id) input_val = data.get("input") output_val = data.get("output") + output_from_envelope = False envelope_jobs: list[tuple[str, Any]] = [] if data.get("input_envelope"): envelope_jobs.append(("input", data["input_envelope"])) @@ -4142,12 +4157,13 @@ async def describe_workflow(self, workflow_id: str) -> WorkflowExecution: external_storage=self.external_storage, external_storage_cache=self.external_storage_cache, ) - for (field, _), value in zip(envelope_jobs, decoded, strict=True): - if field == "input": + for (field_name, _), value in zip(envelope_jobs, decoded, strict=True): + if field_name == "input": input_val = value else: output_val = value - return WorkflowExecution( + output_from_envelope = True + execution = WorkflowExecution( workflow_id=data.get("workflow_id", workflow_id), run_id=data.get("run_id"), workflow_type=data.get("workflow_type", ""), @@ -4162,6 +4178,8 @@ async def describe_workflow(self, workflow_id: str) -> WorkflowExecution: data.get("search_attributes") if isinstance(data.get("search_attributes"), dict) else None ), ) + execution._output_from_envelope = output_from_envelope + return execution async def list_workflows( self, @@ -5507,6 +5525,13 @@ async def get_result( status = desc.status if status in ("completed", "failed", "terminated", "canceled", "cancelled"): run_id = handle.run_id or desc.run_id + if (status == "completed" and desc._output_from_envelope and desc.run_id is not None + and desc.run_id == run_id): + # The status read already carries this run's decoded output; + # the history is only needed for a non-success outcome. + if result_type is None: + return desc.output + return decode_payload_value(desc.output, result_type, "workflow result") if run_id is None: raise WorkflowFailed("no run_id available to fetch history") page_token: str | None = None diff --git a/src/durable_workflow/errors.py b/src/durable_workflow/errors.py index 308859c..cebdfa0 100644 --- a/src/durable_workflow/errors.py +++ b/src/durable_workflow/errors.py @@ -64,6 +64,14 @@ def __init__(self, query_task_id: str, reason: str) -> None: self.reason = reason +class ServerCapabilitiesDigestError(DurableWorkflowError): + """A worker answer elided ``server_capabilities`` with a digest this worker does not hold. + + The Server only elides with the digest a request sent (server#1262), so + this is a protocol failure, never a transient condition. + """ + + class ActivityLeaseAuthorityLost(DurableWorkflowError): """An activity delivery lost its original live execution authority.""" diff --git a/tests/test_capabilities_digest.py b/tests/test_capabilities_digest.py new file mode 100644 index 0000000..62bd4db --- /dev/null +++ b/tests/test_capabilities_digest.py @@ -0,0 +1,230 @@ +"""Server capabilities digest (server#1262) and the history-free completed result.""" +from __future__ import annotations + +import asyncio +import copy +import json +from typing import Any + +import httpx +import pytest + +from durable_workflow import Client, serializer +from durable_workflow._capabilities_digest import HEADER, ServerCapabilitiesDigest +from durable_workflow.errors import ServerCapabilitiesDigestError, ServerError, WorkflowFailed + +D1, D2 = "sha256:" + "1" * 64, "sha256:" + "2" * 64 +CAPS1 = {"activity_lease_renewal": True, "supported_workflow_task_commands": ["complete_workflow"], "v": 1} +CAPS2 = {"activity_lease_renewal": True, "supported_workflow_task_commands": ["complete_workflow"], "v": 2} + + +def tagged(capabilities: dict[str, Any], digest: str) -> dict[str, Any]: + return {**copy.deepcopy(capabilities), "digest": digest} + + +def test_header_is_none_until_a_tagged_copy_is_learned() -> None: + held = ServerCapabilitiesDigest() + assert held.header(held.capture()) == "none" + answer = {"task": None, "server_capabilities": tagged(CAPS1, D1)} + held.restore(answer, held.capture()) + assert answer["server_capabilities"] == CAPS1, "the digest member is stripped" + assert held.capture() == (D1, CAPS1) + assert held.header(held.capture()) == D1 + + +def test_elided_copies_are_restored_up_to_three_levels_deep() -> None: + held = ServerCapabilitiesDigest() + held.restore({"server_capabilities": tagged(CAPS1, D1)}, held.capture()) + captured = held.capture() + answer = { + "server_capabilities": D1, + "activity_poll": {"answer": {"server_capabilities": D1, "task": {"server_capabilities": D1}}}, + } + held.restore(answer, captured) + assert answer["server_capabilities"] == CAPS1 + assert answer["activity_poll"]["answer"]["server_capabilities"] == CAPS1 + # Deeper than the Server ever elides: left untouched. + assert answer["activity_poll"]["answer"]["task"]["server_capabilities"] == D1 + answer["server_capabilities"]["v"] = 99 + assert held.capture()[1] == CAPS1, "callers cannot mutate the held pair" + + +def test_a_digest_string_resolves_against_a_tagged_copy_in_the_same_answer() -> None: + held = ServerCapabilitiesDigest() + answer = {"server_capabilities": D2, "workflow_poll": {"answer": {"server_capabilities": tagged(CAPS2, D2)}}} + held.restore(answer, (None, None)) + assert answer["server_capabilities"] == CAPS2 + assert answer["workflow_poll"]["answer"]["server_capabilities"] == CAPS2 + + +@pytest.mark.parametrize("captured", [(None, None), (D1, CAPS1)]) +def test_unknown_digest_is_a_protocol_failure(captured: tuple[Any, Any]) -> None: + with pytest.raises(ServerCapabilitiesDigestError): + ServerCapabilitiesDigest().restore({"server_capabilities": D2}, captured) + + +def test_a_differing_full_object_is_kept_as_is() -> None: + held = ServerCapabilitiesDigest() + held.restore({"server_capabilities": tagged(CAPS1, D1)}, held.capture()) + answer = {"server_capabilities": CAPS2} + held.restore(answer, held.capture()) + assert answer["server_capabilities"] == CAPS2 + assert held.capture() == (D1, CAPS1) + + +class DigestServer: + """Elides with the digest each request sent, as server#1262 does.""" + + def __init__(self) -> None: + self.current: tuple[str, dict[str, Any]] = (D1, CAPS1) + self.sent: list[tuple[str, str | None]] = [] + self.gates: dict[str, asyncio.Event] = {} + self.status = 200 + + def answer_capabilities(self, sent: str | None) -> Any: + digest, capabilities = self.current + if sent is None: + return copy.deepcopy(capabilities) + return digest if sent == digest else tagged(capabilities, digest) + + async def respond(self, request: httpx.Request) -> httpx.Response: + sent = request.headers.get(HEADER) + self.sent.append((request.url.path, sent)) + # Elision is decided when the request is received, before any wait. + body = {"ok": True, "server_capabilities": self.answer_capabilities(sent)} + gate = self.gates.get(request.url.path) + if gate is not None: + await gate.wait() + if request.url.path.endswith("/refused"): + return httpx.Response(409, json={**body, "reason": "stale_attempt", "message": "refused"}) + return httpx.Response(self.status, json=body) + + +def client_for(server: DigestServer) -> Client: + client = Client("http://digest.invalid", namespace="17", worker_token="worker-token", token="control-token") + client._http = httpx.AsyncClient(base_url="http://digest.invalid", transport=httpx.MockTransport(server.respond)) + return client + + +async def test_worker_requests_opt_in_and_callers_always_see_full_capabilities() -> None: + server = DigestServer() + async with client_for(server) as client: + first = await client._request("POST", "/worker/a", worker=True, json={}) + second = await client._request("POST", "/worker/b", worker=True, json={}) + await client._request("GET", "/workflows/w", json=None) + assert server.sent == [("/api/worker/a", "none"), ("/api/worker/b", D1), ("/api/workflows/w", None)] + assert first["server_capabilities"] == CAPS1 + assert second["server_capabilities"] == CAPS1 + + +async def test_error_bodies_are_restored() -> None: + server = DigestServer() + async with client_for(server) as client: + await client._request("POST", "/worker/a", worker=True, json={}) + with pytest.raises(ServerError) as refusal: + await client._request("POST", "/worker/refused", worker=True, json={}) + assert server.sent[-1] == ("/api/worker/refused", D1) + assert refusal.value.body["server_capabilities"] == CAPS1 + + +async def test_rolling_deploy_restores_each_answer_against_the_pair_its_request_sent() -> None: + server = DigestServer() + async with client_for(server) as client: + await client._request("POST", "/worker/learn", worker=True, json={}) + server.gates["/api/worker/slow"] = asyncio.Event() + # In flight under D1, elided with D1 by the Server, answered late. + slow = asyncio.create_task(client._request("POST", "/worker/slow", worker=True, json={})) + while len(server.sent) < 2: + await asyncio.sleep(0) + server.current = (D2, CAPS2) # the deploy changes the capabilities + moved = await client._request("POST", "/worker/fast", worker=True, json={}) + assert moved["server_capabilities"] == CAPS2 + assert client._capabilities_digest.capture() == (D2, CAPS2) + server.gates["/api/worker/slow"].set() + late = await slow + after = await client._request("POST", "/worker/next", worker=True, json={}) + assert late["server_capabilities"] == CAPS1, "the late D1 answer restores the D1 object" + assert after["server_capabilities"] == CAPS2 + assert [sent for _, sent in server.sent] == ["none", D1, D1, D2] + + +async def test_poll_transport_envelope_is_restored() -> None: + async def respond(request: httpx.Request) -> httpx.Response: + body = {"task": None, "server_capabilities": tagged(CAPS1, D1)} + envelope = {"schema": "durable-workflow.poll-transport.v1", "status": 200, "headers": {}, "body": body} + return httpx.Response(200, headers={"content-type": "application/vnd.durable-workflow.poll.v1+json"}, + content=json.dumps(envelope).encode()) + + async with Client("http://digest.invalid", namespace="17", worker_token="worker-token") as client: + client._http = httpx.AsyncClient(base_url="http://digest.invalid", transport=httpx.MockTransport(respond)) + result = await client._request("POST", "/worker/query-tasks/poll", worker=True, json={"worker_id": "w"}) + assert result["server_capabilities"] == CAPS1 + assert client._capabilities_digest.capture() == (D1, CAPS1) + + +class ResultServer: + def __init__(self, status: str, *, run_id: str = "run-2", envelope: bool = True) -> None: + self.status, self.run_id, self.envelope = status, run_id, envelope + self.paths: list[str] = [] + + async def respond(self, request: httpx.Request) -> httpx.Response: + self.paths.append(request.url.path) + if request.url.path.endswith("/history"): + if self.status == "failed": + event = {"event_type": "WorkflowFailed", "payload": {"message": "boom", "exception_class": "E"}} + else: + event = {"event_type": "WorkflowCompleted", + "payload": {"output": serializer.envelope({"from": "history"})}} + return httpx.Response(200, json={"events": [event], "next_page_token": None}) + body: dict[str, Any] = {"workflow_id": "wf", "run_id": self.run_id, "workflow_type": "t", "status": self.status} + if self.envelope and self.status == "completed": + body["output_envelope"] = serializer.envelope({"from": "status"}) + return httpx.Response(200, json=body) + + +async def run_result(server: ResultServer, run_id: str | None = None) -> Any: + async with Client("http://result.invalid", namespace="17", token="control-token") as client: + transport = httpx.MockTransport(server.respond) + client._http = httpx.AsyncClient(base_url="http://result.invalid", transport=transport) + return await client.get_workflow_handle("wf", run_id=run_id).result(timeout=1) + + +@pytest.mark.parametrize("run_id", [None, "run-2"]) +async def test_completed_result_comes_from_the_status_read(run_id: str | None) -> None: + server = ResultServer("completed") + assert await run_result(server, run_id) == {"from": "status"} + assert server.paths == ["/api/workflows/wf"], "no history read" + + +@pytest.mark.parametrize(("server", "run_id"), [ + (ResultServer("completed", envelope=False), None), + (ResultServer("completed"), "run-1"), # the status read names a newer run +]) +async def test_completed_result_falls_back_to_history(server: ResultServer, run_id: str | None) -> None: + assert await run_result(server, run_id) == {"from": "history"} + assert any(path.endswith("/history") for path in server.paths) + + +async def test_failed_result_still_reads_history() -> None: + server = ResultServer("failed") + with pytest.raises(WorkflowFailed, match="boom"): + await run_result(server) + assert server.paths[-1] == "/api/workflows/wf/runs/run-2/history" + + +async def test_requests_advertise_gzip() -> None: + # Native Server compresses JSON answers when asked (server#1261); httpx + # decodes them transparently. + seen: list[str] = [] + + async def respond(request: httpx.Request) -> httpx.Response: + seen.append(request.headers.get("accept-encoding", "")) + return httpx.Response(200, json={"ok": True}) + + async with Client("http://gzip.invalid", namespace="17", token="control-token") as client: + default = client._http + client._http = httpx.AsyncClient(base_url="http://gzip.invalid", transport=httpx.MockTransport(respond), + headers=default.headers) + await default.aclose() + await client._request("GET", "/workflows/w") + assert "gzip" in seen[0] From 8caad2a04f72c1c180628c94986937b3e8fcf433 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 20 Nov 2026 06:31:27 +0000 Subject: [PATCH 2/2] Bound decoded response bodies at 128 MiB by default (#352) httpx decodes gzip transparently, so a small compressed answer could inflate without limit (a reviewer expanded 1.5 MB of gzip to 3.2 GB in other SDKs). A response hook wraps each body stream and counts the bytes it decodes to, inflating at most 1 MiB at a time, and raises the new ResponseBodyTooLarge (not retried) before httpx decodes past the bound. Configurable with Client(max_response_body_bytes=...), also on the sync client. The client now asks only for gzip; another response Content-Encoding is refused. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 + src/durable_workflow/_response_limit.py | 74 ++++++++++++++++++++ src/durable_workflow/client.py | 7 ++ src/durable_workflow/errors.py | 14 ++++ src/durable_workflow/sync.py | 3 + tests/test_response_body_limit.py | 93 +++++++++++++++++++++++++ 6 files changed, 194 insertions(+) create mode 100644 src/durable_workflow/_response_limit.py create mode 100644 tests/test_response_body_limit.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 39b8412..c5bc0ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Overlapping requests:** each answer is resolved against the digest and object its own request was sent with. A request in flight during a rolling deploy that changes the capabilities restores the object it was sent with, while later requests use the newer digest. - **Errors:** a digest the worker does not hold raises the new `ServerCapabilitiesDigestError`, a protocol failure. - **Older Servers** ignore the header and answer in full as before. +- Response bodies are bounded after decoding: 128 MiB by default, configurable with `Client(max_response_body_bytes=...)` (and the sync client). httpx decodes `Content-Encoding: gzip` transparently, so before this a small compressed answer could expand without limit in memory. + - **How:** each response stream is counted as it decodes, inflating at most 1 MiB at a time. It is refused with the new `ResponseBodyTooLarge` before httpx decodes past the limit, and that error is not retried. The bound also applies to runtime external-payload fetches. + - **Encodings:** the client now asks only for `gzip`, which is what native Server sends. Any other response `Content-Encoding` is refused with `httpx.DecodingError`. - `WorkflowHandle.result()` and `Client.get_result()` no longer read the run's history for a completed run when the status read already carries that run's `output_envelope`. The status read is `GET /workflows/{id}`, which the result poll makes anyway. Failed, cancelled, terminated and timed-out runs still read the history for their typed failure, and so does a completed run whose status read has no `output_envelope` or names a different run. ## [2.20.0] - 2026-11-19 diff --git a/src/durable_workflow/_response_limit.py b/src/durable_workflow/_response_limit.py new file mode 100644 index 0000000..ba045f8 --- /dev/null +++ b/src/durable_workflow/_response_limit.py @@ -0,0 +1,74 @@ +"""Bound the decoded size of every HTTP response body. + +httpx decodes ``Content-Encoding: gzip`` transparently, so a small compressed +answer could expand without limit in memory (a 1.5 MB gzip body can inflate to +gigabytes). Each response stream is wrapped before httpx reads it: the wrapper +counts the bytes the body decodes to, inflating at most 1 MiB at a time, and +raises :class:`~durable_workflow.errors.ResponseBodyTooLarge` before httpx +decodes a chunk that would cross the limit. +""" + +from __future__ import annotations + +import zlib +from collections.abc import AsyncIterator, Awaitable, Callable + +import httpx + +from .errors import ResponseBodyTooLarge + +#: Default bound on a decoded response body: 128 MiB. +DEFAULT_MAX_RESPONSE_BODY_BYTES = 128 * 1024 * 1024 +#: Encodings this client asks for; httpx is not offered others. +ACCEPT_ENCODING = "gzip" +_INFLATE_STEP = 1 << 20 + + +class _BoundedBody(httpx.AsyncByteStream): + def __init__(self, inner: httpx.AsyncByteStream, *, gzip: bool, limit: int, url: str) -> None: + self._inner = inner + self._gzip = gzip + self._limit = limit + self._url = url + + async def __aiter__(self) -> AsyncIterator[bytes]: + inflater = zlib.decompressobj(16 + zlib.MAX_WBITS) if self._gzip else None + decoded = 0 + async for chunk in self._inner: + if inflater is None: + decoded += len(chunk) + else: + pending = chunk + while pending and not inflater.eof: + try: + decoded += len(inflater.decompress(pending, _INFLATE_STEP)) + except zlib.error: + # Not ours to judge: httpx reports the invalid encoding. + inflater = None + break + if decoded > self._limit: + break + pending = inflater.unconsumed_tail + if decoded > self._limit: + raise ResponseBodyTooLarge(self._limit, self._url) + yield chunk + + async def aclose(self) -> None: + await self._inner.aclose() + + +def response_body_limit(limit: int) -> Callable[[httpx.Response], Awaitable[None]]: + """An httpx response hook that bounds the decoded body at ``limit`` bytes.""" + + async def bound(response: httpx.Response) -> None: + encoding = response.headers.get("content-encoding", "").strip().lower() + if encoding not in ("", "identity", "gzip"): + raise httpx.DecodingError( + f"unrequested response content-encoding {encoding!r}", request=response.request, + ) + if isinstance(response.stream, httpx.AsyncByteStream): + response.stream = _BoundedBody( + response.stream, gzip=encoding == "gzip", limit=limit, url=str(response.request.url.path), + ) + + return bound diff --git a/src/durable_workflow/client.py b/src/durable_workflow/client.py index 3ba5f27..0ff8ca9 100644 --- a/src/durable_workflow/client.py +++ b/src/durable_workflow/client.py @@ -67,6 +67,7 @@ from ._poll_transport import unwrap_response as _unwrap_poll_response from ._poll_transport import warn_early_cutoff as _warn_poll_cutoff from ._query_task_authority import QueryTaskAuthority, current_query_authority +from ._response_limit import ACCEPT_ENCODING, DEFAULT_MAX_RESPONSE_BODY_BYTES, response_body_limit from ._typed_values import decode_payload_value, to_payload_values, validate_annotation from .errors import ( REQUEST_ID_AMBIGUOUS, @@ -1948,6 +1949,7 @@ def __init__( external_storage_threshold_bytes: int | None = None, external_storage_cache: ExternalPayloadCache | None = None, telemetry: ExecutionTelemetry | None = None, + max_response_body_bytes: int = DEFAULT_MAX_RESPONSE_BODY_BYTES, ) -> None: _validate_static_credential("token", token) _validate_static_credential("control_token", control_token) @@ -1988,8 +1990,13 @@ def __init__( if external_storage_cache is not None else ExternalPayloadCache() ) + if type(max_response_body_bytes) is not int or max_response_body_bytes < 1: + raise ValueError("max_response_body_bytes must be a positive integer") + self.max_response_body_bytes = max_response_body_bytes self._http = httpx.AsyncClient( base_url=self.base_url, timeout=timeout, verify=tls_context if tls_context is not None else True, + headers={"Accept-Encoding": ACCEPT_ENCODING}, + event_hooks={"response": [response_body_limit(max_response_body_bytes)]}, ) self._cluster_info: dict[str, Any] | None = None self._cluster_info_lock = asyncio.Lock() diff --git a/src/durable_workflow/errors.py b/src/durable_workflow/errors.py index cebdfa0..1565bec 100644 --- a/src/durable_workflow/errors.py +++ b/src/durable_workflow/errors.py @@ -64,6 +64,20 @@ def __init__(self, query_task_id: str, reason: str) -> None: self.reason = reason +class ResponseBodyTooLarge(DurableWorkflowError): + """A response body decoded to more bytes than the client allows. + + The bound (``Client(max_response_body_bytes=...)``, 128 MiB by default) + applies after ``Content-Encoding`` decoding, so a small compressed answer + cannot expand without limit in memory. Not retried. + """ + + def __init__(self, limit: int, path: str = "") -> None: + super().__init__(f"response body{f' from {path}' if path else ''} exceeds {limit} decoded bytes") + self.limit = limit + self.path = path + + class ServerCapabilitiesDigestError(DurableWorkflowError): """A worker answer elided ``server_capabilities`` with a digest this worker does not hold. diff --git a/src/durable_workflow/sync.py b/src/durable_workflow/sync.py index 26520dd..0f28d21 100644 --- a/src/durable_workflow/sync.py +++ b/src/durable_workflow/sync.py @@ -7,6 +7,7 @@ from collections.abc import Coroutine from typing import Any, TypeVar +from ._response_limit import DEFAULT_MAX_RESPONSE_BODY_BYTES from .client import Client as AsyncClient from .client import ( NamespaceDescription, @@ -320,6 +321,7 @@ def __init__( external_storage: ExternalStorageDriver | None = None, external_storage_threshold_bytes: int | None = None, external_storage_cache: ExternalPayloadCache | None = None, + max_response_body_bytes: int = DEFAULT_MAX_RESPONSE_BODY_BYTES, ) -> None: self._async = AsyncClient( base_url, @@ -334,6 +336,7 @@ def __init__( external_storage=external_storage, external_storage_threshold_bytes=external_storage_threshold_bytes, external_storage_cache=external_storage_cache, + max_response_body_bytes=max_response_body_bytes, ) self._runner = _LoopRunner() self._closed = False diff --git a/tests/test_response_body_limit.py b/tests/test_response_body_limit.py new file mode 100644 index 0000000..e43d855 --- /dev/null +++ b/tests/test_response_body_limit.py @@ -0,0 +1,93 @@ +"""Decoded response bodies are bounded, so a small gzip answer cannot expand without limit.""" +from __future__ import annotations + +import gzip +import json +from collections.abc import AsyncIterator + +import httpx +import pytest + +from durable_workflow import Client +from durable_workflow.errors import ResponseBodyTooLarge +from durable_workflow.retry_policy import TransportRetryPolicy + +LIMIT = 1024 * 1024 + + +class Chunks(httpx.AsyncByteStream): + def __init__(self, data: bytes, size: int = 16 * 1024) -> None: + self.data, self.size, self.sent, self.closed = data, size, 0, False + + async def __aiter__(self) -> AsyncIterator[bytes]: + for start in range(0, len(self.data), self.size): + self.sent += 1 + yield self.data[start:start + self.size] + + async def aclose(self) -> None: + self.closed = True + + +def client_answering(body: bytes, *, encoding: str | None, limit: int = LIMIT) -> tuple[Client, list[Chunks]]: + streams: list[Chunks] = [] + + async def respond(request: httpx.Request) -> httpx.Response: + assert request.headers["accept-encoding"] == "gzip" + stream = Chunks(body) + streams.append(stream) + headers = {"content-type": "application/json", **({"content-encoding": encoding} if encoding else {})} + return httpx.Response(200, headers=headers, stream=stream) + + client = Client("http://limit.invalid", token="t", max_response_body_bytes=limit, + retry_policy=TransportRetryPolicy(max_attempts=3)) + # Keep the client's own headers and response hook; replace only the network. + default = client._http + client._http = httpx.AsyncClient(base_url="http://limit.invalid", transport=httpx.MockTransport(respond), + headers=default.headers, event_hooks=default.event_hooks) + return client, streams + + +def json_bytes(size: int) -> bytes: + return json.dumps({"pad": "0" * size}).encode() + + +async def test_a_gzip_bomb_is_refused_before_it_is_inflated() -> None: + bomb = gzip.compress(json_bytes(64 * LIMIT)) # 64 MiB of JSON in about 64 KiB of gzip + assert len(bomb) < 128 * 1024 + client, streams = client_answering(bomb, encoding="gzip") + async with client: + with pytest.raises(ResponseBodyTooLarge) as refusal: + await client._request("GET", "/workflows/w") + assert refusal.value.limit == LIMIT + assert len(streams) == 1, "a body over the limit is not retried" + assert streams[0].sent < 4, "refused within the first chunks, not after inflating everything" + assert streams[0].closed + + +async def test_an_identity_body_over_the_limit_is_refused() -> None: + client, _ = client_answering(json_bytes(LIMIT + 1), encoding=None) + async with client: + with pytest.raises(ResponseBodyTooLarge): + await client._request("GET", "/workflows/w") + + +@pytest.mark.parametrize("encoding", ["gzip", None]) +async def test_bodies_within_the_limit_decode_normally(encoding: str | None) -> None: + body = json_bytes(LIMIT // 2) + client, _ = client_answering(gzip.compress(body) if encoding else body, encoding=encoding) + async with client: + assert await client._request("GET", "/workflows/w") == json.loads(body) + + +async def test_an_unrequested_encoding_is_refused() -> None: + client, _ = client_answering(b"{}", encoding="br") + async with client: + with pytest.raises(httpx.DecodingError, match="unrequested"): + await client._request("GET", "/workflows/w") + + +def test_the_default_bound_is_128_mib_and_validated() -> None: + assert Client("http://limit.invalid").max_response_body_bytes == 128 * 1024 * 1024 + for invalid in (0, -1, 1.5, True): + with pytest.raises(ValueError, match="max_response_body_bytes"): + Client("http://limit.invalid", max_response_body_bytes=invalid) # type: ignore[arg-type]