diff --git a/CHANGELOG.md b/CHANGELOG.md index a4ccd80..5ddd4bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- A completion refused as `413 payload_too_large` because the worker's cached payload policy is stale no longer loses the computed result. + - **The bug:** the worker resolves the runtime external payload policy (`namespace.external_payload_storage`, including `threshold_bytes`) once and keeps it. When the Server lowered its limit mid-run, a result under the old threshold was sent inline and refused. Python did not exit, unlike PHP, but logged `failed to complete`. The result was lost, and the activity failed later as a lost lease. + - **The fix:** on such a 413 for an activity completion, failure report or workflow-task completion, the worker rediscovers the policy (`GET /api/cluster/info`, retried through outages with the shared backoff while the lease lasts). If the policy changed, it re-encodes the request, externalising payloads the new threshold requires, and resends once. The 413 is refused before the request is processed, so the resend is safe. The worker never exits and never reruns the activity. + - **Undeliverable results:** some activity results still cannot be delivered. That covers a result that is still too large after rediscovery, one with no external storage to offload to, and one over the upload limit. Instead of expiring silently, the attempt is now reported as a non-retryable failure with `failure.type` and `failure.class` set to `ActivityResultTooLarge`, the same as PHP and Rust, and a message naming the Server's limit. The worker keeps running. + - **Unchanged:** an oversized workflow-task completion still fails the run with the structural-limit failure. - A cluster-info read failure at worker startup no longer stops the worker. - **Retried:** `GET /api/cluster/info` has no side effects, so `Worker.run()` and `run_until()` now treat any failed connection and any 5xx or 429 answer to it as transient, unless the Server says `retryable: false`. Before, only the narrower transient set did (for example 503 `storage_unavailable`), so a plain `500` stopped the worker. The read is retried with the shared backoff `uniform(0, min(4, 0.25 * 2**(n - 1)))` seconds until the runtime answers. - **Logging:** startup retries now escalate like poll outages, to ERROR after 60 s and then every 5 minutes. diff --git a/src/durable_workflow/client.py b/src/durable_workflow/client.py index c2a16da..90bda2d 100644 --- a/src/durable_workflow/client.py +++ b/src/durable_workflow/client.py @@ -2432,6 +2432,21 @@ async def _resolve_activity_claim_payloads(self, value: Any, *, worker: bool) -> raise ExternalPayloadUnsupported("activity payload reference has no authenticated runtime transport") return await self._resolve_runtime_payloads(value, worker=worker, transport=transport, resolved={}) + def _payload_policy_snapshot(self) -> tuple[int, int] | None: + """The cached runtime payload policy as (threshold, max payload) bytes, if any.""" + transport = self._runtime_external_payload_transport_cache + return (transport.threshold_bytes, transport.max_payload_bytes) if transport is not None else None + + def _forget_payload_policy(self) -> None: + """Drop the cached payload policy so the next payload re-resolves it from discovery.""" + self._runtime_external_payload_transport_cache = None + self._runtime_external_payload_transport_resolved = False + + async def _payload_policy_resolved_snapshot(self) -> tuple[int, int] | None: + """Resolve the payload policy from the held discovery and snapshot it.""" + await self._runtime_external_payload_transport() + return self._payload_policy_snapshot() + async def _runtime_external_payload_transport( self, ) -> _RuntimeExternalPayloadTransport | None: diff --git a/src/durable_workflow/errors.py b/src/durable_workflow/errors.py index 7135d26..c5b2aa2 100644 --- a/src/durable_workflow/errors.py +++ b/src/durable_workflow/errors.py @@ -359,9 +359,9 @@ class WorkflowTaskCompletionTooLarge(ServerError): Server 3.6.0 and later accept a workflow-task completion of up to 16 payload limits (32 MiB by default) and refuse a larger one with ``413 payload_too_large`` and ``limit_kind: workflow_task_completion_bytes``. - Replaying the task produces the same commands, so resending it cannot - succeed: this error is not retryable. The worker fails the run with a - ``structural_limit`` failure instead of resending the completion. + The worker may re-encode the same commands once under a refreshed payload + policy. If the current policy still cannot fit the completion, the worker + fails the run with a ``structural_limit`` failure. :attr:`limit` is the Server's aggregate limit in bytes, when it reported one. :attr:`size` is the encoded request body the SDK sent, in bytes. diff --git a/src/durable_workflow/worker.py b/src/durable_workflow/worker.py index 123c392..cd11ec9 100644 --- a/src/durable_workflow/worker.py +++ b/src/durable_workflow/worker.py @@ -87,6 +87,7 @@ DuplicateRegistrationError, DurableWorkflowError, ExternalPayloadError, + ExternalPayloadOversized, FailureReportTooLarge, InvalidArgument, NonDeterministicReplayError, @@ -358,6 +359,30 @@ def _positive_seconds(value: object) -> bool: ) +#: Failure type and class of an activity attempt whose result cannot be +#: delivered under the Server's payload limits (the same in PHP and Rust). +ACTIVITY_RESULT_TOO_LARGE = "ActivityResultTooLarge" + + +def _activity_result_too_large_message(error: BaseException) -> str | None: + """The failure message for an undeliverable oversized result, or ``None``.""" + if _payload_too_large(error): + # _payload_too_large established a ServerError with a mapping body. + limit = getattr(error, "body", {}).get("limit") + limit_text = (f"the Server's payload limit of {limit} bytes" if type(limit) is int + else "the Server's payload limit") + return f"activity result exceeds {limit_text} and cannot be stored externally under the current policy" + if isinstance(error, ExternalPayloadOversized): + return f"activity result exceeds the Server's external payload limit: {error}" + return None + + +def _payload_too_large(error: BaseException) -> bool: + """A ``413 payload_too_large`` request-size refusal (refused before processing).""" + return (isinstance(error, ServerError) and error.status == 413 and isinstance(error.body, Mapping) + and error.body.get("reason") == "payload_too_large") + + def _auth_refusal(error: BaseException) -> bool: return isinstance(error, Unauthorized) or isinstance(error, ServerError) and error.status in (401, 403) @@ -3319,8 +3344,22 @@ async def _complete_workflow_task_with_retry( sticky_cache=sticky_cache, ) except WorkflowTaskCompletionTooLarge as error: + if _payload_too_large(error) and await self._refresh_payload_policy(f"workflow task {task_id} completion"): + # The policy was stale: resend once, re-encoded under it. + try: + await self._send_workflow_task_completion( + task_id=task_id, + attempt=attempt, + commands=commands, + message_stream_cursors=message_stream_cursors, + message_stream_waits=message_stream_waits, + sticky_cache=sticky_cache, + ) + return + except WorkflowTaskCompletionTooLarge as still_too_large: + error = still_too_large await self._fail_run_for_oversized_completion(task_id, attempt, error) - raise + raise error async def _fail_run_for_oversized_completion( self, @@ -3760,6 +3799,20 @@ async def _run_activity_task_body(self, task: dict[str, Any]) -> str: task_id, ) return "claim_aborted" + too_large = _activity_result_too_large_message(e) + if too_large is not None: + # The result cannot be delivered under the Server's current + # policy (still too large after rediscovery, no external + # storage, or over the upload limit). Report it as a + # non-retryable failure, as PHP and Rust do, instead of + # letting the attempt expire silently. + log.error("activity task %s result is too large to deliver: %s", task_id, too_large) + await self._report_activity_task_failure( + task_id, attempt_id, "oversized activity result", message=too_large, + failure_type=ACTIVITY_RESULT_TOO_LARGE, failure_class=ACTIVITY_RESULT_TOO_LARGE, + non_retryable=True, + ) + return "failed_non_retryable" log.warning("failed to complete activity task %s: %s", task_id, e) return "complete_error" return "completed" @@ -3838,11 +3891,22 @@ async def _deliver_activity_outcome(self, task_id: str, attempt_id: str, outcome """ failures = 0 started = time.monotonic() + policy_refreshed = False while True: try: await publish() return except Exception as error: + if _payload_too_large(error) and not policy_refreshed: + # The request was refused before it was processed. A + # cached payload policy may be stale (the Server lowered + # its limit): rediscover it and resend once, re-encoded. + policy_refreshed = True + if await self._refresh_payload_policy( + f"activity task {task_id} {outcome}", lambda: self._activity_outcome_window(task_id), + ): + continue + raise # The client's own transport retry can resend too, so this # holds on the first round as well. if outcome == "completed" and _committed_own_activity_completion( @@ -3873,6 +3937,55 @@ async def _deliver_activity_outcome(self, task_id: str, attempt_id: str, outcome task_id, outcome, summary, delay) await asyncio.sleep(delay) + async def _rediscover_cluster_info(self, context: str, window: Callable[[], float | None] | None) -> None: + """Read cluster info again, raising the discovery error if retries end.""" + failures = 0 + started = time.monotonic() + while True: + try: + await self.client.get_cluster_info() + return + except Exception as error: + if not _transient_discovery_error(error) or self._stop.is_set(): + log.warning("%s: payload policy rediscovery failed: %s", context, _poll_error_summary(error)) + raise + failures += 1 + delay = _transient_retry_delay(failures) + left = window() if window is not None else None + if left is None: + left = _UNKNOWN_LEASE_OUTCOME_RETRY_SECONDS - (time.monotonic() - started) + if left <= delay: + log.exception("%s: payload policy rediscovery failed %d times (%s) and no time is left", + context, failures, _poll_error_summary(error)) + raise + log.warning("%s: payload policy rediscovery failed transiently (%s); retrying in %.2f s", + context, _poll_error_summary(error), delay) + await asyncio.sleep(delay) + + async def _refresh_payload_policy(self, context: str, + window: Callable[[], float | None] | None = None) -> bool: + """Rediscover the payload policy after a ``413 payload_too_large``. + + Returns whether the policy changed, so a re-encoded resend could fit. + Discovery outages are retried with the shared backoff while ``window`` + (seconds left, ``None`` when unknown) allows; the held discovery is + replaced only by a successful read. A discovery error propagates so + callers do not record a permanent size failure on an unknown policy. + """ + client = self.client + before = getattr(client, "_payload_policy_snapshot", lambda: None)() + await self._rediscover_cluster_info(context, window) + forget = getattr(client, "_forget_payload_policy", None) + if forget is not None: + forget() + after = await client._payload_policy_resolved_snapshot() if hasattr( + client, "_payload_policy_resolved_snapshot") else None + changed = after != before + log.warning("%s refused as payload_too_large; payload policy %s (%s -> %s)%s", + context, "changed" if changed else "unchanged", before, after, + "; resending once re-encoded" if changed else "") + return changed + async def _report_activity_task_failure(self, task_id: str, attempt_id: str, context: str, **failure: Any) -> None: try: diff --git a/tests/test_payload_policy_refresh.py b/tests/test_payload_policy_refresh.py new file mode 100644 index 0000000..ce426ff --- /dev/null +++ b/tests/test_payload_policy_refresh.py @@ -0,0 +1,255 @@ +"""A completion refused as 413 payload_too_large rediscovers a stale payload policy and resends once.""" +from __future__ import annotations + +import json +import logging +from typing import Any + +import httpx +import pytest + +from durable_workflow import Worker, activity, serializer +from durable_workflow.client import Client +from durable_workflow.errors import WorkflowTaskCompletionTooLarge +from durable_workflow.retry_policy import TransportRetryPolicy +from durable_workflow.worker import _current_activity_claim + +from .test_runtime_external_payload_transport import FakeRuntimePayloadServer + +MIB = 1024 * 1024 +RESULT = "r" * (300 * 1024) +ran: list[str] = [] + + +@activity.defn(name="policy.big") +def big_result() -> str: + ran.append("ran") + return RESULT + + +class LimitServer(FakeRuntimePayloadServer): + """Refuses requests over its request limit; the limit (and threshold) can change mid-run.""" + + def __init__(self, *, storage: bool = True) -> None: + super().__init__(threshold_bytes=2 * MIB, max_payload_bytes=8 * MIB) + self.storage = storage + self.request_limit = 2 * MIB + self.fail_bodies: list[dict[str, Any]] = [] + self.completions: list[int] = [] + self.failures = 0 + self.discovery_failures = 0 + self.discovery_reads = 0 + self.refuse_offloaded_completion = False + + def cluster_info(self) -> dict[str, Any]: + info = super().cluster_info() + if not self.storage: + info["namespace"].pop("external_payload_storage") + return info + + def lower_limit(self, limit: int, *, threshold: int | None = None) -> None: + # Native: an absent namespace threshold is the payload ceiling. + self.request_limit = limit + self.threshold_bytes = threshold if threshold is not None else limit + + def handler(self, request: httpx.Request) -> httpx.Response: + path = request.url.path + if path == "/api/cluster/info": + self.discovery_reads += 1 + if self.discovery_failures: + self.discovery_failures -= 1 + return httpx.Response(503, json={"reason": "storage_unavailable"}) + over_limit = len(request.content) > self.request_limit + if path.endswith("/activity-tasks/task-1/complete") and self.refuse_offloaded_completion: + over_limit |= b'"external_payload"' in request.content + if path.startswith("/api/worker/") and over_limit: + if path.endswith("/complete"): + self.completions.append(len(request.content)) + return httpx.Response(413, json={ + "message": f"Request payload exceeds the maximum allowed size of {self.request_limit} bytes.", + "reason": "payload_too_large", "limit": self.request_limit, + }) + if path.endswith("/activity-tasks/task-1/complete"): + self.completions.append(len(request.content)) + return httpx.Response(200, json={"outcome": "completed", "recorded": True}) + if path.endswith("/activity-tasks/task-1/fail"): + self.failures += 1 + self.fail_bodies.append(json.loads(request.content)) + return httpx.Response(200, json={"outcome": "failed", "recorded": True}) + return super().handler(request) + + +def task() -> dict[str, Any]: + return { + "task_id": "task-1", "activity_attempt_id": "attempt-1", "attempt_number": 1, + "activity_type": "policy.big", "workflow_id": "wf-1", "run_id": "run-1", + "lease_owner": "worker-1", "task_queue": "q", + "arguments": serializer.envelope([], codec="avro"), "payload_codec": "avro", + } + + +async def run_activity(server: LimitServer, *, lower: dict[str, Any] | None, + outage: int = 0) -> tuple[str, Client]: + client = Client("https://runtime.example", token="runtime-role-token", namespace="billing", + retry_policy=TransportRetryPolicy(max_attempts=1)) + client._http = httpx.AsyncClient(base_url=client.base_url, transport=httpx.MockTransport(server.handler)) + # The worker discovered the 2 MiB policy earlier in its run. + await client.get_cluster_info() + expected = (2 * MIB, 8 * MIB) if server.storage else None + assert await client._payload_policy_resolved_snapshot() == expected + if lower is not None: + server.lower_limit(**lower) # the Server's limit changes mid-run + server.discovery_failures = outage + worker = Worker(client, task_queue="q", workflows=[], activities=[big_result], worker_id="worker-1") + claim = _current_activity_claim.set(task()) + try: + return await worker._run_activity_task(task()), client + finally: + _current_activity_claim.reset(claim) + if worker._remote_activity_executor is not None: + worker._remote_activity_executor.shutdown(wait=True) + + +@pytest.fixture(autouse=True) +def fresh(monkeypatch: pytest.MonkeyPatch) -> None: + ran.clear() + monkeypatch.setattr("durable_workflow.worker._TRANSIENT_RETRY_BASE_SECONDS", 0.001) + + +async def test_a_lowered_limit_is_rediscovered_and_the_result_resent_externalised() -> None: + server = LimitServer() + outcome, client = await run_activity(server, lower={"limit": 100 * 1024}) + try: + assert outcome == "completed" + assert ran == ["ran"], "the activity ran once; its computed result was kept" + assert len(server.completions) == 2, "refused once inline, then resent once" + assert server.completions[0] > 100 * 1024 >= server.completions[1] + assert server.upload_count == 1, "the resend externalised the result under the new policy" + assert server.failures == 0 + assert client._payload_policy_snapshot() == (100 * 1024, 8 * MIB) + finally: + await client.aclose() + + +async def test_rediscovery_rides_out_a_discovery_outage() -> None: + server = LimitServer() + outcome, client = await run_activity(server, lower={"limit": 100 * 1024}, outage=3) + try: + assert outcome == "completed" + assert server.upload_count == 1 + assert len(server.completions) == 2 + assert server.discovery_reads == 1 + 4, "three failed rediscovery reads, then one that answered" + finally: + await client.aclose() + + +async def test_rediscovery_outage_leaves_the_activity_attempt_for_lease_recovery( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(Worker, "_activity_outcome_window", lambda *_: 0.0) + server = LimitServer() + outcome, client = await run_activity(server, lower={"limit": 100 * 1024}, outage=1) + try: + assert outcome == "complete_error" + assert ran == ["ran"] + assert len(server.completions) == 1 + assert server.discovery_reads == 2 + assert server.failures == 0 + assert client._payload_policy_snapshot() == (2 * MIB, 8 * MIB) + finally: + await client.aclose() + + +def assert_failed_as_too_large(server: LimitServer, limit: int) -> None: + assert server.failures == 1, "reported once" + failure = server.fail_bodies[0]["failure"] + assert failure["type"] == "ActivityResultTooLarge" + assert failure["class"] == "ActivityResultTooLarge" + assert failure["non_retryable"] is True + assert f"payload limit of {limit} bytes" in failure["message"] + + +async def test_a_result_still_too_large_after_rediscovery_fails_non_retryably( + caplog: pytest.LogCaptureFixture, +) -> None: + # The limit drops but the namespace threshold stays at 2 MiB: rediscovery + # finds nothing to change, so the result cannot be delivered. + server = LimitServer() + with caplog.at_level(logging.WARNING, logger="durable_workflow.worker"): + outcome, client = await run_activity(server, lower={"limit": 100 * 1024, "threshold": 2 * MIB}) + try: + assert outcome == "failed_non_retryable" + assert ran == ["ran"], "the activity ran once" + assert len(server.completions) == 1, "no resend when the policy did not change" + assert_failed_as_too_large(server, 100 * 1024) + assert "payload policy unchanged" in caplog.text + # The worker carries on: the same client keeps serving requests. + assert await client._request("POST", "/worker/heartbeat", worker=True, json={}) == {"ok": True} + finally: + await client.aclose() + + +async def test_a_result_that_cannot_be_externalised_fails_non_retryably() -> None: + server = LimitServer(storage=False) + outcome, client = await run_activity(server, lower={"limit": 100 * 1024}) + try: + assert outcome == "failed_non_retryable" + assert ran == ["ran"] + assert len(server.completions) == 1 + assert server.upload_count == 0, "no external storage to offload to" + assert_failed_as_too_large(server, 100 * 1024) + finally: + await client.aclose() + + +async def test_a_second_size_refusal_reports_the_failure_without_resending_again() -> None: + server = LimitServer() + server.refuse_offloaded_completion = True + outcome, client = await run_activity(server, lower={"limit": 100 * 1024}) + try: + assert outcome == "failed_non_retryable" + assert ran == ["ran"] + assert len(server.completions) == 2 + assert server.discovery_reads == 2 + assert server.upload_count == 1 + assert_failed_as_too_large(server, 100 * 1024) + finally: + await client.aclose() + + +def oversized() -> WorkflowTaskCompletionTooLarge: + return WorkflowTaskCompletionTooLarge(413, { + "reason": "payload_too_large", "limit_kind": "workflow_task_completion_bytes", "limit": 32 * MIB, + }, size=40 * MIB) + + +@pytest.mark.parametrize("changed", [True, False]) +async def test_a_workflow_task_completion_refusal_resends_once_after_a_policy_change( + monkeypatch: pytest.MonkeyPatch, changed: bool, +) -> None: + from unittest.mock import AsyncMock + + worker = Worker(AsyncMock(spec=Client), task_queue="q", workflows=[], activities=[big_result]) + sends: list[int] = [] + + async def send(**_: Any) -> None: + sends.append(1) + if len(sends) == 1: + raise oversized() + + async def refresh(context: str, window: Any = None) -> bool: + return changed + + failed = AsyncMock() + monkeypatch.setattr(worker, "_send_workflow_task_completion", send) + monkeypatch.setattr(worker, "_refresh_payload_policy", refresh) + monkeypatch.setattr(worker, "_fail_run_for_oversized_completion", failed) + if changed: + await worker._complete_workflow_task_with_retry(task_id="wt-1", attempt=1, commands=[]) + assert len(sends) == 2 + failed.assert_not_awaited() + else: + with pytest.raises(WorkflowTaskCompletionTooLarge): + await worker._complete_workflow_task_with_retry(task_id="wt-1", attempt=1, commands=[]) + assert len(sends) == 1 + failed.assert_awaited_once()