From 8e71778a79e003554a1509fe6af6eea7421fb037 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 20 Nov 2026 11:09:56 +0000 Subject: [PATCH 1/5] Rediscover a stale payload policy on 413 and resend the completion once The worker resolves the runtime external payload policy once. When the Server lowered its limit mid-run, a result under the old threshold was sent inline and refused 413 payload_too_large; the worker logged "failed to complete" and the result was lost (the activity later failed as a lost lease). On that 413 for an activity completion, failure report or workflow-task completion, rediscover the policy (cluster info, retried through outages with the shared backoff while the lease lasts) and, if it changed, resend once re-encoded, externalising what the new threshold requires. The 413 is refused before processing, so the resend is safe. If the policy did not change or the resend is still too large, today's handling stands. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 + src/durable_workflow/client.py | 18 +++ src/durable_workflow/worker.py | 77 +++++++++++- tests/test_payload_policy_refresh.py | 181 +++++++++++++++++++++++++++ 4 files changed, 279 insertions(+), 1 deletion(-) create mode 100644 tests/test_payload_policy_refresh.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a4ccd80..9c34faf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- A completion refused as `413 payload_too_large` because the worker's cached payload policy is stale no longer loses the computed result. + - **The bug:** the worker resolves the runtime external payload policy (`namespace.external_payload_storage`, including `threshold_bytes`) once and keeps it. When the Server lowered its limit mid-run, a result under the old threshold was sent inline and refused. Python did not exit, unlike PHP, but logged `failed to complete`. The result was lost, and the activity failed later as a lost lease. + - **The fix:** on such a 413 for an activity completion, failure report or workflow-task completion, the worker rediscovers the policy (`GET /api/cluster/info`, retried through outages with the shared backoff while the lease lasts). If the policy changed, it re-encodes the request, externalising payloads the new threshold requires, and resends once. The 413 is refused before the request is processed, so the resend is safe. The worker never exits and never reruns the activity. + - **Unchanged:** if the policy did not change, or the resend is still too large, today's handling stands: a warning for activity outcomes, and the structural-limit run failure for an oversized workflow-task completion. - A cluster-info read failure at worker startup no longer stops the worker. - **Retried:** `GET /api/cluster/info` has no side effects, so `Worker.run()` and `run_until()` now treat any failed connection and any 5xx or 429 answer to it as transient, unless the Server says `retryable: false`. Before, only the narrower transient set did (for example 503 `storage_unavailable`), so a plain `500` stopped the worker. The read is retried with the shared backoff `uniform(0, min(4, 0.25 * 2**(n - 1)))` seconds until the runtime answers. - **Logging:** startup retries now escalate like poll outages, to ERROR after 60 s and then every 5 minutes. diff --git a/src/durable_workflow/client.py b/src/durable_workflow/client.py index c2a16da..cae65ef 100644 --- a/src/durable_workflow/client.py +++ b/src/durable_workflow/client.py @@ -2432,6 +2432,24 @@ async def _resolve_activity_claim_payloads(self, value: Any, *, worker: bool) -> raise ExternalPayloadUnsupported("activity payload reference has no authenticated runtime transport") return await self._resolve_runtime_payloads(value, worker=worker, transport=transport, resolved={}) + def _payload_policy_snapshot(self) -> tuple[int, int] | None: + """The cached runtime payload policy as (threshold, max payload) bytes, if any.""" + transport = self._runtime_external_payload_transport_cache + return (transport.threshold_bytes, transport.max_payload_bytes) if transport is not None else None + + def _forget_payload_policy(self) -> None: + """Drop the cached payload policy so the next payload re-resolves it from discovery.""" + self._runtime_external_payload_transport_cache = None + self._runtime_external_payload_transport_resolved = False + + async def _payload_policy_resolved_snapshot(self) -> tuple[int, int] | None: + """Resolve the payload policy from the held discovery and snapshot it.""" + try: + await self._runtime_external_payload_transport() + except Exception: + return None + return self._payload_policy_snapshot() + async def _runtime_external_payload_transport( self, ) -> _RuntimeExternalPayloadTransport | None: diff --git a/src/durable_workflow/worker.py b/src/durable_workflow/worker.py index 123c392..e5aeb41 100644 --- a/src/durable_workflow/worker.py +++ b/src/durable_workflow/worker.py @@ -358,6 +358,12 @@ def _positive_seconds(value: object) -> bool: ) +def _payload_too_large(error: BaseException) -> bool: + """A ``413 payload_too_large`` request-size refusal (refused before processing).""" + return (isinstance(error, ServerError) and error.status == 413 and isinstance(error.body, Mapping) + and error.body.get("reason") == "payload_too_large") + + def _auth_refusal(error: BaseException) -> bool: return isinstance(error, Unauthorized) or isinstance(error, ServerError) and error.status in (401, 403) @@ -3319,8 +3325,22 @@ async def _complete_workflow_task_with_retry( sticky_cache=sticky_cache, ) except WorkflowTaskCompletionTooLarge as error: + if _payload_too_large(error) and await self._refresh_payload_policy(f"workflow task {task_id} completion"): + # The policy was stale: resend once, re-encoded under it. + try: + await self._send_workflow_task_completion( + task_id=task_id, + attempt=attempt, + commands=commands, + message_stream_cursors=message_stream_cursors, + message_stream_waits=message_stream_waits, + sticky_cache=sticky_cache, + ) + return + except WorkflowTaskCompletionTooLarge as still_too_large: + error = still_too_large await self._fail_run_for_oversized_completion(task_id, attempt, error) - raise + raise error async def _fail_run_for_oversized_completion( self, @@ -3838,11 +3858,22 @@ async def _deliver_activity_outcome(self, task_id: str, attempt_id: str, outcome """ failures = 0 started = time.monotonic() + policy_refreshed = False while True: try: await publish() return except Exception as error: + if _payload_too_large(error) and not policy_refreshed: + # The request was refused before it was processed. A + # cached payload policy may be stale (the Server lowered + # its limit): rediscover it and resend once, re-encoded. + policy_refreshed = True + if await self._refresh_payload_policy( + f"activity task {task_id} {outcome}", lambda: self._activity_outcome_window(task_id), + ): + continue + raise # The client's own transport retry can resend too, so this # holds on the first round as well. if outcome == "completed" and _committed_own_activity_completion( @@ -3873,6 +3904,50 @@ async def _deliver_activity_outcome(self, task_id: str, attempt_id: str, outcome task_id, outcome, summary, delay) await asyncio.sleep(delay) + async def _refresh_payload_policy(self, context: str, + window: Callable[[], float | None] | None = None) -> bool: + """Rediscover the payload policy after a ``413 payload_too_large``. + + Returns whether the policy changed, so a re-encoded resend could fit. + Discovery outages are retried with the shared backoff while ``window`` + (seconds left, ``None`` when unknown) allows; the held discovery is + replaced only by a successful read. + """ + client = self.client + before = getattr(client, "_payload_policy_snapshot", lambda: None)() + failures = 0 + started = time.monotonic() + while True: + try: + await client.get_cluster_info() + break + except Exception as error: + if not _transient_discovery_error(error) or self._stop.is_set(): + log.warning("%s: payload policy rediscovery failed: %s", context, _poll_error_summary(error)) + return False + failures += 1 + delay = _transient_retry_delay(failures) + left = window() if window is not None else None + if left is None: + left = _UNKNOWN_LEASE_OUTCOME_RETRY_SECONDS - (time.monotonic() - started) + if left <= delay: + log.error("%s: payload policy rediscovery failed %d times (%s) and no time is left", + context, failures, _poll_error_summary(error)) + return False + log.warning("%s: payload policy rediscovery failed transiently (%s); retrying in %.2f s", + context, _poll_error_summary(error), delay) + await asyncio.sleep(delay) + forget = getattr(client, "_forget_payload_policy", None) + if forget is not None: + forget() + after = await client._payload_policy_resolved_snapshot() if hasattr( + client, "_payload_policy_resolved_snapshot") else None + changed = after != before + log.warning("%s refused as payload_too_large; payload policy %s (%s -> %s)%s", + context, "changed" if changed else "unchanged", before, after, + "; resending once re-encoded" if changed else "") + return changed + async def _report_activity_task_failure(self, task_id: str, attempt_id: str, context: str, **failure: Any) -> None: try: diff --git a/tests/test_payload_policy_refresh.py b/tests/test_payload_policy_refresh.py new file mode 100644 index 0000000..69159fd --- /dev/null +++ b/tests/test_payload_policy_refresh.py @@ -0,0 +1,181 @@ +"""A completion refused as 413 payload_too_large rediscovers a stale payload policy and resends once.""" +from __future__ import annotations + +import logging +from typing import Any + +import httpx +import pytest + +from durable_workflow import Worker, activity, serializer +from durable_workflow.client import Client +from durable_workflow.errors import WorkflowTaskCompletionTooLarge +from durable_workflow.retry_policy import TransportRetryPolicy +from durable_workflow.worker import _current_activity_claim + +from .test_runtime_external_payload_transport import FakeRuntimePayloadServer + +MIB = 1024 * 1024 +RESULT = "r" * (300 * 1024) +ran: list[str] = [] + + +@activity.defn(name="policy.big") +def big_result() -> str: + ran.append("ran") + return RESULT + + +class LimitServer(FakeRuntimePayloadServer): + """Refuses requests over its request limit; the limit (and threshold) can change mid-run.""" + + def __init__(self) -> None: + super().__init__(threshold_bytes=2 * MIB, max_payload_bytes=8 * MIB) + self.request_limit = 2 * MIB + self.completions: list[int] = [] + self.failures = 0 + self.discovery_failures = 0 + self.discovery_reads = 0 + + def lower_limit(self, limit: int, *, threshold: int | None = None) -> None: + # Native: an absent namespace threshold is the payload ceiling. + self.request_limit = limit + self.threshold_bytes = threshold if threshold is not None else limit + + def handler(self, request: httpx.Request) -> httpx.Response: + path = request.url.path + if path == "/api/cluster/info": + self.discovery_reads += 1 + if self.discovery_failures: + self.discovery_failures -= 1 + return httpx.Response(503, json={"reason": "storage_unavailable"}) + if path.startswith("/api/worker/") and len(request.content) > self.request_limit: + if path.endswith("/complete"): + self.completions.append(len(request.content)) + return httpx.Response(413, json={ + "message": f"Request payload exceeds the maximum allowed size of {self.request_limit} bytes.", + "reason": "payload_too_large", "limit": self.request_limit, + }) + if path.endswith("/activity-tasks/task-1/complete"): + self.completions.append(len(request.content)) + return httpx.Response(200, json={"outcome": "completed", "recorded": True}) + if path.endswith("/activity-tasks/task-1/fail"): + self.failures += 1 + return httpx.Response(200, json={"outcome": "failed", "recorded": True}) + return super().handler(request) + + +def task() -> dict[str, Any]: + return { + "task_id": "task-1", "activity_attempt_id": "attempt-1", "attempt_number": 1, + "activity_type": "policy.big", "workflow_id": "wf-1", "run_id": "run-1", + "lease_owner": "worker-1", "task_queue": "q", + "arguments": serializer.envelope([], codec="avro"), "payload_codec": "avro", + } + + +async def run_activity(server: LimitServer, *, lower: dict[str, Any] | None, + outage: int = 0) -> tuple[str, Client]: + client = Client("https://runtime.example", token="runtime-role-token", namespace="billing", + retry_policy=TransportRetryPolicy(max_attempts=1)) + client._http = httpx.AsyncClient(base_url=client.base_url, transport=httpx.MockTransport(server.handler)) + # The worker discovered the 2 MiB policy earlier in its run. + await client.get_cluster_info() + assert await client._payload_policy_resolved_snapshot() == (2 * MIB, 8 * MIB) + if lower is not None: + server.lower_limit(**lower) # the Server's limit changes mid-run + server.discovery_failures = outage + worker = Worker(client, task_queue="q", workflows=[], activities=[big_result], worker_id="worker-1") + claim = _current_activity_claim.set(task()) + try: + return await worker._run_activity_task(task()), client + finally: + _current_activity_claim.reset(claim) + if worker._remote_activity_executor is not None: + worker._remote_activity_executor.shutdown(wait=True) + + +@pytest.fixture(autouse=True) +def fresh(monkeypatch: pytest.MonkeyPatch) -> None: + ran.clear() + monkeypatch.setattr("durable_workflow.worker._TRANSIENT_RETRY_BASE_SECONDS", 0.001) + + +async def test_a_lowered_limit_is_rediscovered_and_the_result_resent_externalised() -> None: + server = LimitServer() + outcome, client = await run_activity(server, lower={"limit": 100 * 1024}) + try: + assert outcome == "completed" + assert ran == ["ran"], "the activity ran once; its computed result was kept" + assert len(server.completions) == 2, "refused once inline, then resent once" + assert server.completions[0] > 100 * 1024 >= server.completions[1] + assert server.upload_count == 1, "the resend externalised the result under the new policy" + assert server.failures == 0 + assert client._payload_policy_snapshot() == (100 * 1024, 8 * MIB) + finally: + await client.aclose() + + +async def test_rediscovery_rides_out_a_discovery_outage() -> None: + server = LimitServer() + outcome, client = await run_activity(server, lower={"limit": 100 * 1024}, outage=3) + try: + assert outcome == "completed" + assert server.upload_count == 1 and len(server.completions) == 2 + assert server.discovery_reads == 1 + 4, "three failed rediscovery reads, then one that answered" + finally: + await client.aclose() + + +async def test_a_genuinely_too_large_result_keeps_todays_handling(caplog: pytest.LogCaptureFixture) -> None: + # The limit drops but the namespace threshold stays at 2 MiB: rediscovery + # finds nothing to change, so the refusal stands after a single send. + server = LimitServer() + with caplog.at_level(logging.WARNING, logger="durable_workflow.worker"): + outcome, client = await run_activity(server, lower={"limit": 100 * 1024, "threshold": 2 * MIB}) + try: + assert outcome == "complete_error" + assert ran == ["ran"] + assert len(server.completions) == 1, "no resend when the policy did not change" + assert server.failures == 0, "a size refusal never becomes a failure report" + assert "payload policy unchanged" in caplog.text + finally: + await client.aclose() + + +def oversized() -> WorkflowTaskCompletionTooLarge: + return WorkflowTaskCompletionTooLarge(413, { + "reason": "payload_too_large", "limit_kind": "workflow_task_completion_bytes", "limit": 32 * MIB, + }, size=40 * MIB) + + +@pytest.mark.parametrize("changed", [True, False]) +async def test_a_workflow_task_completion_refusal_resends_once_after_a_policy_change( + monkeypatch: pytest.MonkeyPatch, changed: bool, +) -> None: + from unittest.mock import AsyncMock + + worker = Worker(AsyncMock(spec=Client), task_queue="q", workflows=[], activities=[big_result]) + sends: list[int] = [] + + async def send(**_: Any) -> None: + sends.append(1) + if len(sends) == 1: + raise oversized() + + async def refresh(context: str, window: Any = None) -> bool: + return changed + + failed = AsyncMock() + monkeypatch.setattr(worker, "_send_workflow_task_completion", send) + monkeypatch.setattr(worker, "_refresh_payload_policy", refresh) + monkeypatch.setattr(worker, "_fail_run_for_oversized_completion", failed) + if changed: + await worker._complete_workflow_task_with_retry(task_id="wt-1", attempt=1, commands=[]) + assert len(sends) == 2 + failed.assert_not_awaited() + else: + with pytest.raises(WorkflowTaskCompletionTooLarge): + await worker._complete_workflow_task_with_retry(task_id="wt-1", attempt=1, commands=[]) + assert len(sends) == 1 + failed.assert_awaited_once() From d90e3f217afa5da40baad5b503945bc83c6aee9f Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 20 Nov 2026 11:37:37 +0000 Subject: [PATCH 2/5] Fail an undeliverable oversized activity result as ActivityResultTooLarge When the result is still too large after rediscovery, there is no external storage to offload to, or it is over the upload limit, the attempt used to expire silently. Report it as a non-retryable failure with failure.type and failure.class "ActivityResultTooLarge" (the same wire shape as PHP and Rust) and a message naming the Server's limit; the worker keeps running. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 +- src/durable_workflow/worker.py | 33 ++++++++++++++++++ tests/test_payload_policy_refresh.py | 51 ++++++++++++++++++++++++---- 3 files changed, 79 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c34faf..5ddd4bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - A completion refused as `413 payload_too_large` because the worker's cached payload policy is stale no longer loses the computed result. - **The bug:** the worker resolves the runtime external payload policy (`namespace.external_payload_storage`, including `threshold_bytes`) once and keeps it. When the Server lowered its limit mid-run, a result under the old threshold was sent inline and refused. Python did not exit, unlike PHP, but logged `failed to complete`. The result was lost, and the activity failed later as a lost lease. - **The fix:** on such a 413 for an activity completion, failure report or workflow-task completion, the worker rediscovers the policy (`GET /api/cluster/info`, retried through outages with the shared backoff while the lease lasts). If the policy changed, it re-encodes the request, externalising payloads the new threshold requires, and resends once. The 413 is refused before the request is processed, so the resend is safe. The worker never exits and never reruns the activity. - - **Unchanged:** if the policy did not change, or the resend is still too large, today's handling stands: a warning for activity outcomes, and the structural-limit run failure for an oversized workflow-task completion. + - **Undeliverable results:** some activity results still cannot be delivered. That covers a result that is still too large after rediscovery, one with no external storage to offload to, and one over the upload limit. Instead of expiring silently, the attempt is now reported as a non-retryable failure with `failure.type` and `failure.class` set to `ActivityResultTooLarge`, the same as PHP and Rust, and a message naming the Server's limit. The worker keeps running. + - **Unchanged:** an oversized workflow-task completion still fails the run with the structural-limit failure. - A cluster-info read failure at worker startup no longer stops the worker. - **Retried:** `GET /api/cluster/info` has no side effects, so `Worker.run()` and `run_until()` now treat any failed connection and any 5xx or 429 answer to it as transient, unless the Server says `retryable: false`. Before, only the narrower transient set did (for example 503 `storage_unavailable`), so a plain `500` stopped the worker. The read is retried with the shared backoff `uniform(0, min(4, 0.25 * 2**(n - 1)))` seconds until the runtime answers. - **Logging:** startup retries now escalate like poll outages, to ERROR after 60 s and then every 5 minutes. diff --git a/src/durable_workflow/worker.py b/src/durable_workflow/worker.py index e5aeb41..5cfa982 100644 --- a/src/durable_workflow/worker.py +++ b/src/durable_workflow/worker.py @@ -87,6 +87,7 @@ DuplicateRegistrationError, DurableWorkflowError, ExternalPayloadError, + ExternalPayloadOversized, FailureReportTooLarge, InvalidArgument, NonDeterministicReplayError, @@ -358,6 +359,24 @@ def _positive_seconds(value: object) -> bool: ) +#: Failure type and class of an activity attempt whose result cannot be +#: delivered under the Server's payload limits (the same in PHP and Rust). +ACTIVITY_RESULT_TOO_LARGE = "ActivityResultTooLarge" + + +def _activity_result_too_large_message(error: BaseException) -> str | None: + """The failure message for an undeliverable oversized result, or ``None``.""" + if _payload_too_large(error): + body = error.body if isinstance(error, ServerError) and isinstance(error.body, Mapping) else {} + limit = body.get("limit") + limit_text = (f"the Server's payload limit of {limit} bytes" if type(limit) is int + else "the Server's payload limit") + return f"activity result exceeds {limit_text} and cannot be stored externally under the current policy" + if isinstance(error, ExternalPayloadOversized): + return f"activity result exceeds the Server's external payload limit: {error}" + return None + + def _payload_too_large(error: BaseException) -> bool: """A ``413 payload_too_large`` request-size refusal (refused before processing).""" return (isinstance(error, ServerError) and error.status == 413 and isinstance(error.body, Mapping) @@ -3780,6 +3799,20 @@ async def _run_activity_task_body(self, task: dict[str, Any]) -> str: task_id, ) return "claim_aborted" + too_large = _activity_result_too_large_message(e) + if too_large is not None: + # The result cannot be delivered under the Server's current + # policy (still too large after rediscovery, no external + # storage, or over the upload limit). Report it as a + # non-retryable failure, as PHP and Rust do, instead of + # letting the attempt expire silently. + log.error("activity task %s result is too large to deliver: %s", task_id, too_large) + await self._report_activity_task_failure( + task_id, attempt_id, "oversized activity result", message=too_large, + failure_type=ACTIVITY_RESULT_TOO_LARGE, failure_class=ACTIVITY_RESULT_TOO_LARGE, + non_retryable=True, + ) + return "failed_non_retryable" log.warning("failed to complete activity task %s: %s", task_id, e) return "complete_error" return "completed" diff --git a/tests/test_payload_policy_refresh.py b/tests/test_payload_policy_refresh.py index 69159fd..40cbdca 100644 --- a/tests/test_payload_policy_refresh.py +++ b/tests/test_payload_policy_refresh.py @@ -1,6 +1,7 @@ """A completion refused as 413 payload_too_large rediscovers a stale payload policy and resends once.""" from __future__ import annotations +import json import logging from typing import Any @@ -29,14 +30,22 @@ def big_result() -> str: class LimitServer(FakeRuntimePayloadServer): """Refuses requests over its request limit; the limit (and threshold) can change mid-run.""" - def __init__(self) -> None: + def __init__(self, *, storage: bool = True) -> None: super().__init__(threshold_bytes=2 * MIB, max_payload_bytes=8 * MIB) + self.storage = storage self.request_limit = 2 * MIB + self.fail_bodies: list[dict[str, Any]] = [] self.completions: list[int] = [] self.failures = 0 self.discovery_failures = 0 self.discovery_reads = 0 + def cluster_info(self) -> dict[str, Any]: + info = super().cluster_info() + if not self.storage: + info["namespace"].pop("external_payload_storage") + return info + def lower_limit(self, limit: int, *, threshold: int | None = None) -> None: # Native: an absent namespace threshold is the payload ceiling. self.request_limit = limit @@ -61,6 +70,7 @@ def handler(self, request: httpx.Request) -> httpx.Response: return httpx.Response(200, json={"outcome": "completed", "recorded": True}) if path.endswith("/activity-tasks/task-1/fail"): self.failures += 1 + self.fail_bodies.append(json.loads(request.content)) return httpx.Response(200, json={"outcome": "failed", "recorded": True}) return super().handler(request) @@ -81,7 +91,8 @@ async def run_activity(server: LimitServer, *, lower: dict[str, Any] | None, client._http = httpx.AsyncClient(base_url=client.base_url, transport=httpx.MockTransport(server.handler)) # The worker discovered the 2 MiB policy earlier in its run. await client.get_cluster_info() - assert await client._payload_policy_resolved_snapshot() == (2 * MIB, 8 * MIB) + expected = (2 * MIB, 8 * MIB) if server.storage else None + assert await client._payload_policy_resolved_snapshot() == expected if lower is not None: server.lower_limit(**lower) # the Server's limit changes mid-run server.discovery_failures = outage @@ -127,18 +138,44 @@ async def test_rediscovery_rides_out_a_discovery_outage() -> None: await client.aclose() -async def test_a_genuinely_too_large_result_keeps_todays_handling(caplog: pytest.LogCaptureFixture) -> None: +def assert_failed_as_too_large(server: LimitServer, limit: int) -> None: + assert server.failures == 1, "reported once" + failure = server.fail_bodies[0]["failure"] + assert failure["type"] == "ActivityResultTooLarge" + assert failure["class"] == "ActivityResultTooLarge" + assert failure["non_retryable"] is True + assert f"payload limit of {limit} bytes" in failure["message"] + + +async def test_a_result_still_too_large_after_rediscovery_fails_non_retryably( + caplog: pytest.LogCaptureFixture, +) -> None: # The limit drops but the namespace threshold stays at 2 MiB: rediscovery - # finds nothing to change, so the refusal stands after a single send. + # finds nothing to change, so the result cannot be delivered. server = LimitServer() with caplog.at_level(logging.WARNING, logger="durable_workflow.worker"): outcome, client = await run_activity(server, lower={"limit": 100 * 1024, "threshold": 2 * MIB}) try: - assert outcome == "complete_error" - assert ran == ["ran"] + assert outcome == "failed_non_retryable" + assert ran == ["ran"], "the activity ran once" assert len(server.completions) == 1, "no resend when the policy did not change" - assert server.failures == 0, "a size refusal never becomes a failure report" + assert_failed_as_too_large(server, 100 * 1024) assert "payload policy unchanged" in caplog.text + # The worker carries on: the same client keeps serving requests. + assert await client._request("POST", "/worker/heartbeat", worker=True, json={}) == {"ok": True} + finally: + await client.aclose() + + +async def test_a_result_that_cannot_be_externalised_fails_non_retryably() -> None: + server = LimitServer(storage=False) + outcome, client = await run_activity(server, lower={"limit": 100 * 1024}) + try: + assert outcome == "failed_non_retryable" + assert ran == ["ran"] + assert len(server.completions) == 1 + assert server.upload_count == 0, "no external storage to offload to" + assert_failed_as_too_large(server, 100 * 1024) finally: await client.aclose() From 8dfa2469fa33b034ed42eefcc2cc533d15408829 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 20 Nov 2026 11:47:00 +0000 Subject: [PATCH 3/5] Address SonarCloud findings on the payload-policy refresh No behaviour change: read the refusal's limit without a redundant type check, move the rediscovery loop into _rediscover_cluster_info (cognitive complexity), log its final failure with log.exception, and split a composite test assertion. Co-Authored-By: Claude Opus 5.5 --- src/durable_workflow/worker.py | 39 ++++++++++++++++------------ tests/test_payload_policy_refresh.py | 3 ++- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/src/durable_workflow/worker.py b/src/durable_workflow/worker.py index 5cfa982..92410a4 100644 --- a/src/durable_workflow/worker.py +++ b/src/durable_workflow/worker.py @@ -367,8 +367,8 @@ def _positive_seconds(value: object) -> bool: def _activity_result_too_large_message(error: BaseException) -> str | None: """The failure message for an undeliverable oversized result, or ``None``.""" if _payload_too_large(error): - body = error.body if isinstance(error, ServerError) and isinstance(error.body, Mapping) else {} - limit = body.get("limit") + # _payload_too_large established a ServerError with a mapping body. + limit = getattr(error, "body", {}).get("limit") limit_text = (f"the Server's payload limit of {limit} bytes" if type(limit) is int else "the Server's payload limit") return f"activity result exceeds {limit_text} and cannot be stored externally under the current policy" @@ -3937,23 +3937,14 @@ async def _deliver_activity_outcome(self, task_id: str, attempt_id: str, outcome task_id, outcome, summary, delay) await asyncio.sleep(delay) - async def _refresh_payload_policy(self, context: str, - window: Callable[[], float | None] | None = None) -> bool: - """Rediscover the payload policy after a ``413 payload_too_large``. - - Returns whether the policy changed, so a re-encoded resend could fit. - Discovery outages are retried with the shared backoff while ``window`` - (seconds left, ``None`` when unknown) allows; the held discovery is - replaced only by a successful read. - """ - client = self.client - before = getattr(client, "_payload_policy_snapshot", lambda: None)() + async def _rediscover_cluster_info(self, context: str, window: Callable[[], float | None] | None) -> bool: + """Read cluster info again, through transient outages while ``window`` allows.""" failures = 0 started = time.monotonic() while True: try: - await client.get_cluster_info() - break + await self.client.get_cluster_info() + return True except Exception as error: if not _transient_discovery_error(error) or self._stop.is_set(): log.warning("%s: payload policy rediscovery failed: %s", context, _poll_error_summary(error)) @@ -3964,12 +3955,26 @@ async def _refresh_payload_policy(self, context: str, if left is None: left = _UNKNOWN_LEASE_OUTCOME_RETRY_SECONDS - (time.monotonic() - started) if left <= delay: - log.error("%s: payload policy rediscovery failed %d times (%s) and no time is left", - context, failures, _poll_error_summary(error)) + log.exception("%s: payload policy rediscovery failed %d times (%s) and no time is left", + context, failures, _poll_error_summary(error)) return False log.warning("%s: payload policy rediscovery failed transiently (%s); retrying in %.2f s", context, _poll_error_summary(error), delay) await asyncio.sleep(delay) + + async def _refresh_payload_policy(self, context: str, + window: Callable[[], float | None] | None = None) -> bool: + """Rediscover the payload policy after a ``413 payload_too_large``. + + Returns whether the policy changed, so a re-encoded resend could fit. + Discovery outages are retried with the shared backoff while ``window`` + (seconds left, ``None`` when unknown) allows; the held discovery is + replaced only by a successful read. + """ + client = self.client + before = getattr(client, "_payload_policy_snapshot", lambda: None)() + if not await self._rediscover_cluster_info(context, window): + return False forget = getattr(client, "_forget_payload_policy", None) if forget is not None: forget() diff --git a/tests/test_payload_policy_refresh.py b/tests/test_payload_policy_refresh.py index 40cbdca..b6a0f3a 100644 --- a/tests/test_payload_policy_refresh.py +++ b/tests/test_payload_policy_refresh.py @@ -132,7 +132,8 @@ async def test_rediscovery_rides_out_a_discovery_outage() -> None: outcome, client = await run_activity(server, lower={"limit": 100 * 1024}, outage=3) try: assert outcome == "completed" - assert server.upload_count == 1 and len(server.completions) == 2 + assert server.upload_count == 1 + assert len(server.completions) == 2 assert server.discovery_reads == 1 + 4, "three failed rediscovery reads, then one that answered" finally: await client.aclose() From 68cca187a3bceb95bf32131a357754ff33a4bc11 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sat, 21 Nov 2026 03:42:57 +0000 Subject: [PATCH 4/5] Keep oversized results recoverable when payload discovery fails --- src/durable_workflow/client.py | 5 +--- src/durable_workflow/errors.py | 6 ++--- src/durable_workflow/worker.py | 20 +++++++-------- tests/test_payload_policy_refresh.py | 38 +++++++++++++++++++++++++++- 4 files changed, 51 insertions(+), 18 deletions(-) diff --git a/src/durable_workflow/client.py b/src/durable_workflow/client.py index cae65ef..90bda2d 100644 --- a/src/durable_workflow/client.py +++ b/src/durable_workflow/client.py @@ -2444,10 +2444,7 @@ def _forget_payload_policy(self) -> None: async def _payload_policy_resolved_snapshot(self) -> tuple[int, int] | None: """Resolve the payload policy from the held discovery and snapshot it.""" - try: - await self._runtime_external_payload_transport() - except Exception: - return None + await self._runtime_external_payload_transport() return self._payload_policy_snapshot() async def _runtime_external_payload_transport( diff --git a/src/durable_workflow/errors.py b/src/durable_workflow/errors.py index 7135d26..c5b2aa2 100644 --- a/src/durable_workflow/errors.py +++ b/src/durable_workflow/errors.py @@ -359,9 +359,9 @@ class WorkflowTaskCompletionTooLarge(ServerError): Server 3.6.0 and later accept a workflow-task completion of up to 16 payload limits (32 MiB by default) and refuse a larger one with ``413 payload_too_large`` and ``limit_kind: workflow_task_completion_bytes``. - Replaying the task produces the same commands, so resending it cannot - succeed: this error is not retryable. The worker fails the run with a - ``structural_limit`` failure instead of resending the completion. + The worker may re-encode the same commands once under a refreshed payload + policy. If the current policy still cannot fit the completion, the worker + fails the run with a ``structural_limit`` failure. :attr:`limit` is the Server's aggregate limit in bytes, when it reported one. :attr:`size` is the encoded request body the SDK sent, in bytes. diff --git a/src/durable_workflow/worker.py b/src/durable_workflow/worker.py index 92410a4..d94fa77 100644 --- a/src/durable_workflow/worker.py +++ b/src/durable_workflow/worker.py @@ -3369,8 +3369,8 @@ async def _fail_run_for_oversized_completion( ) -> None: """Fail the run once with a structural-limit failure (.github#228). - The refused completion is never resent: a replay produces the same - commands. The same leased task completes with one small + Replaying the task would produce the same oversized commands under the + current policy. The same leased task completes with one small ``fail_workflow`` command instead, which the Server records with the ``structural_limit`` failure category. If that completion also fails, the typed error is raised again with the follow-up's error as its @@ -3937,18 +3937,18 @@ async def _deliver_activity_outcome(self, task_id: str, attempt_id: str, outcome task_id, outcome, summary, delay) await asyncio.sleep(delay) - async def _rediscover_cluster_info(self, context: str, window: Callable[[], float | None] | None) -> bool: - """Read cluster info again, through transient outages while ``window`` allows.""" + async def _rediscover_cluster_info(self, context: str, window: Callable[[], float | None] | None) -> None: + """Read cluster info again, raising the discovery error if retries end.""" failures = 0 started = time.monotonic() while True: try: await self.client.get_cluster_info() - return True + return except Exception as error: if not _transient_discovery_error(error) or self._stop.is_set(): log.warning("%s: payload policy rediscovery failed: %s", context, _poll_error_summary(error)) - return False + raise failures += 1 delay = _transient_retry_delay(failures) left = window() if window is not None else None @@ -3957,7 +3957,7 @@ async def _rediscover_cluster_info(self, context: str, window: Callable[[], floa if left <= delay: log.exception("%s: payload policy rediscovery failed %d times (%s) and no time is left", context, failures, _poll_error_summary(error)) - return False + raise log.warning("%s: payload policy rediscovery failed transiently (%s); retrying in %.2f s", context, _poll_error_summary(error), delay) await asyncio.sleep(delay) @@ -3969,12 +3969,12 @@ async def _refresh_payload_policy(self, context: str, Returns whether the policy changed, so a re-encoded resend could fit. Discovery outages are retried with the shared backoff while ``window`` (seconds left, ``None`` when unknown) allows; the held discovery is - replaced only by a successful read. + replaced only by a successful read. A discovery error propagates so + callers do not record a permanent size failure on an unknown policy. """ client = self.client before = getattr(client, "_payload_policy_snapshot", lambda: None)() - if not await self._rediscover_cluster_info(context, window): - return False + await self._rediscover_cluster_info(context, window) forget = getattr(client, "_forget_payload_policy", None) if forget is not None: forget() diff --git a/tests/test_payload_policy_refresh.py b/tests/test_payload_policy_refresh.py index b6a0f3a..ce426ff 100644 --- a/tests/test_payload_policy_refresh.py +++ b/tests/test_payload_policy_refresh.py @@ -39,6 +39,7 @@ def __init__(self, *, storage: bool = True) -> None: self.failures = 0 self.discovery_failures = 0 self.discovery_reads = 0 + self.refuse_offloaded_completion = False def cluster_info(self) -> dict[str, Any]: info = super().cluster_info() @@ -58,7 +59,10 @@ def handler(self, request: httpx.Request) -> httpx.Response: if self.discovery_failures: self.discovery_failures -= 1 return httpx.Response(503, json={"reason": "storage_unavailable"}) - if path.startswith("/api/worker/") and len(request.content) > self.request_limit: + over_limit = len(request.content) > self.request_limit + if path.endswith("/activity-tasks/task-1/complete") and self.refuse_offloaded_completion: + over_limit |= b'"external_payload"' in request.content + if path.startswith("/api/worker/") and over_limit: if path.endswith("/complete"): self.completions.append(len(request.content)) return httpx.Response(413, json={ @@ -139,6 +143,23 @@ async def test_rediscovery_rides_out_a_discovery_outage() -> None: await client.aclose() +async def test_rediscovery_outage_leaves_the_activity_attempt_for_lease_recovery( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(Worker, "_activity_outcome_window", lambda *_: 0.0) + server = LimitServer() + outcome, client = await run_activity(server, lower={"limit": 100 * 1024}, outage=1) + try: + assert outcome == "complete_error" + assert ran == ["ran"] + assert len(server.completions) == 1 + assert server.discovery_reads == 2 + assert server.failures == 0 + assert client._payload_policy_snapshot() == (2 * MIB, 8 * MIB) + finally: + await client.aclose() + + def assert_failed_as_too_large(server: LimitServer, limit: int) -> None: assert server.failures == 1, "reported once" failure = server.fail_bodies[0]["failure"] @@ -181,6 +202,21 @@ async def test_a_result_that_cannot_be_externalised_fails_non_retryably() -> Non await client.aclose() +async def test_a_second_size_refusal_reports_the_failure_without_resending_again() -> None: + server = LimitServer() + server.refuse_offloaded_completion = True + outcome, client = await run_activity(server, lower={"limit": 100 * 1024}) + try: + assert outcome == "failed_non_retryable" + assert ran == ["ran"] + assert len(server.completions) == 2 + assert server.discovery_reads == 2 + assert server.upload_count == 1 + assert_failed_as_too_large(server, 100 * 1024) + finally: + await client.aclose() + + def oversized() -> WorkflowTaskCompletionTooLarge: return WorkflowTaskCompletionTooLarge(413, { "reason": "payload_too_large", "limit_kind": "workflow_task_completion_bytes", "limit": 32 * MIB, From 93bb9c6fba0c855fcf030edfe5c64572cb0e83fa Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sat, 21 Nov 2026 03:44:40 +0000 Subject: [PATCH 5/5] Keep the replay corpus guard focused on behavior --- src/durable_workflow/worker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/durable_workflow/worker.py b/src/durable_workflow/worker.py index d94fa77..cd11ec9 100644 --- a/src/durable_workflow/worker.py +++ b/src/durable_workflow/worker.py @@ -3369,8 +3369,8 @@ async def _fail_run_for_oversized_completion( ) -> None: """Fail the run once with a structural-limit failure (.github#228). - Replaying the task would produce the same oversized commands under the - current policy. The same leased task completes with one small + The refused completion is never resent: a replay produces the same + commands. The same leased task completes with one small ``fail_workflow`` command instead, which the Server records with the ``structural_limit`` failure category. If that completion also fails, the typed error is raised again with the follow-up's error as its