Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

### Fixed

- A completion refused as `413 payload_too_large` because the worker's cached payload policy is stale no longer loses the computed result.
- **The bug:** the worker resolves the runtime external payload policy (`namespace.external_payload_storage`, including `threshold_bytes`) once and keeps it. When the Server lowered its limit mid-run, a result under the old threshold was sent inline and refused. Python did not exit, unlike PHP, but logged `failed to complete`. The result was lost, and the activity failed later as a lost lease.
- **The fix:** on such a 413 for an activity completion, failure report or workflow-task completion, the worker rediscovers the policy (`GET /api/cluster/info`, retried through outages with the shared backoff while the lease lasts). If the policy changed, it re-encodes the request, externalising payloads the new threshold requires, and resends once. The 413 is refused before the request is processed, so the resend is safe. The worker never exits and never reruns the activity.
- **Undeliverable results:** some activity results still cannot be delivered. That covers a result that is still too large after rediscovery, one with no external storage to offload to, and one over the upload limit. Instead of expiring silently, the attempt is now reported as a non-retryable failure with `failure.type` and `failure.class` set to `ActivityResultTooLarge`, the same as PHP and Rust, and a message naming the Server's limit. The worker keeps running.
- **Unchanged:** an oversized workflow-task completion still fails the run with the structural-limit failure.
- A cluster-info read failure at worker startup no longer stops the worker.
- **Retried:** `GET /api/cluster/info` has no side effects, so `Worker.run()` and `run_until()` now treat any failed connection and any 5xx or 429 answer to it as transient, unless the Server says `retryable: false`. Before, only the narrower transient set did (for example 503 `storage_unavailable`), so a plain `500` stopped the worker. The read is retried with the shared backoff `uniform(0, min(4, 0.25 * 2**(n - 1)))` seconds until the runtime answers.
- **Logging:** startup retries now escalate like poll outages, to ERROR after 60 s and then every 5 minutes.
Expand Down
15 changes: 15 additions & 0 deletions src/durable_workflow/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -2432,6 +2432,21 @@ async def _resolve_activity_claim_payloads(self, value: Any, *, worker: bool) ->
raise ExternalPayloadUnsupported("activity payload reference has no authenticated runtime transport")
return await self._resolve_runtime_payloads(value, worker=worker, transport=transport, resolved={})

def _payload_policy_snapshot(self) -> tuple[int, int] | None:
"""The cached runtime payload policy as (threshold, max payload) bytes, if any."""
transport = self._runtime_external_payload_transport_cache
return (transport.threshold_bytes, transport.max_payload_bytes) if transport is not None else None

def _forget_payload_policy(self) -> None:
"""Drop the cached payload policy so the next payload re-resolves it from discovery."""
self._runtime_external_payload_transport_cache = None
self._runtime_external_payload_transport_resolved = False

async def _payload_policy_resolved_snapshot(self) -> tuple[int, int] | None:
"""Resolve the payload policy from the held discovery and snapshot it."""
await self._runtime_external_payload_transport()
return self._payload_policy_snapshot()

async def _runtime_external_payload_transport(
self,
) -> _RuntimeExternalPayloadTransport | None:
Expand Down
6 changes: 3 additions & 3 deletions src/durable_workflow/errors.py
Original file line number Diff line number Diff line change
Expand Up @@ -359,9 +359,9 @@ class WorkflowTaskCompletionTooLarge(ServerError):
Server 3.6.0 and later accept a workflow-task completion of up to 16
payload limits (32 MiB by default) and refuse a larger one with ``413
payload_too_large`` and ``limit_kind: workflow_task_completion_bytes``.
Replaying the task produces the same commands, so resending it cannot
succeed: this error is not retryable. The worker fails the run with a
``structural_limit`` failure instead of resending the completion.
The worker may re-encode the same commands once under a refreshed payload
policy. If the current policy still cannot fit the completion, the worker
fails the run with a ``structural_limit`` failure.

:attr:`limit` is the Server's aggregate limit in bytes, when it reported
one. :attr:`size` is the encoded request body the SDK sent, in bytes.
Expand Down
115 changes: 114 additions & 1 deletion src/durable_workflow/worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@
DuplicateRegistrationError,
DurableWorkflowError,
ExternalPayloadError,
ExternalPayloadOversized,
FailureReportTooLarge,
InvalidArgument,
NonDeterministicReplayError,
Expand Down Expand Up @@ -358,6 +359,30 @@
)


#: Failure type and class of an activity attempt whose result cannot be
#: delivered under the Server's payload limits (the same in PHP and Rust).
ACTIVITY_RESULT_TOO_LARGE = "ActivityResultTooLarge"


def _activity_result_too_large_message(error: BaseException) -> str | None:
"""The failure message for an undeliverable oversized result, or ``None``."""
if _payload_too_large(error):
# _payload_too_large established a ServerError with a mapping body.
limit = getattr(error, "body", {}).get("limit")
limit_text = (f"the Server's payload limit of {limit} bytes" if type(limit) is int
else "the Server's payload limit")
return f"activity result exceeds {limit_text} and cannot be stored externally under the current policy"
if isinstance(error, ExternalPayloadOversized):
return f"activity result exceeds the Server's external payload limit: {error}"
return None


def _payload_too_large(error: BaseException) -> bool:
"""A ``413 payload_too_large`` request-size refusal (refused before processing)."""
return (isinstance(error, ServerError) and error.status == 413 and isinstance(error.body, Mapping)
and error.body.get("reason") == "payload_too_large")


def _auth_refusal(error: BaseException) -> bool:
return isinstance(error, Unauthorized) or isinstance(error, ServerError) and error.status in (401, 403)

Expand Down Expand Up @@ -3319,8 +3344,22 @@
sticky_cache=sticky_cache,
)
except WorkflowTaskCompletionTooLarge as error:
if _payload_too_large(error) and await self._refresh_payload_policy(f"workflow task {task_id} completion"):
# The policy was stale: resend once, re-encoded under it.
try:
await self._send_workflow_task_completion(
task_id=task_id,
attempt=attempt,
commands=commands,
message_stream_cursors=message_stream_cursors,
message_stream_waits=message_stream_waits,
sticky_cache=sticky_cache,
)
return
except WorkflowTaskCompletionTooLarge as still_too_large:
error = still_too_large
await self._fail_run_for_oversized_completion(task_id, attempt, error)
raise
raise error

async def _fail_run_for_oversized_completion(
self,
Expand Down Expand Up @@ -3760,6 +3799,20 @@
task_id,
)
return "claim_aborted"
too_large = _activity_result_too_large_message(e)
if too_large is not None:
# The result cannot be delivered under the Server's current
# policy (still too large after rediscovery, no external
# storage, or over the upload limit). Report it as a
# non-retryable failure, as PHP and Rust do, instead of
# letting the attempt expire silently.
log.error("activity task %s result is too large to deliver: %s", task_id, too_large)
await self._report_activity_task_failure(
task_id, attempt_id, "oversized activity result", message=too_large,
failure_type=ACTIVITY_RESULT_TOO_LARGE, failure_class=ACTIVITY_RESULT_TOO_LARGE,
non_retryable=True,
)
return "failed_non_retryable"
log.warning("failed to complete activity task %s: %s", task_id, e)
return "complete_error"
return "completed"
Expand Down Expand Up @@ -3838,11 +3891,22 @@
"""
failures = 0
started = time.monotonic()
policy_refreshed = False
while True:
try:
await publish()
return
except Exception as error:
if _payload_too_large(error) and not policy_refreshed:
# The request was refused before it was processed. A
# cached payload policy may be stale (the Server lowered
# its limit): rediscover it and resend once, re-encoded.
policy_refreshed = True
if await self._refresh_payload_policy(
f"activity task {task_id} {outcome}", lambda: self._activity_outcome_window(task_id),
):
continue
raise
# The client's own transport retry can resend too, so this
# holds on the first round as well.
if outcome == "completed" and _committed_own_activity_completion(
Expand Down Expand Up @@ -3873,6 +3937,55 @@
task_id, outcome, summary, delay)
await asyncio.sleep(delay)

async def _rediscover_cluster_info(self, context: str, window: Callable[[], float | None] | None) -> None:

Check failure on line 3940 in src/durable_workflow/worker.py

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Refactor this function to reduce its Cognitive Complexity from 16 to the 15 allowed.

See more on https://sonarcloud.io/project/issues?id=durable-workflow_sdk-python&issues=AaH4yTfH1GRH8L0w986t&open=AaH4yTfH1GRH8L0w986t&pullRequest=360
"""Read cluster info again, raising the discovery error if retries end."""
failures = 0
started = time.monotonic()
while True:
try:
await self.client.get_cluster_info()
return
except Exception as error:
if not _transient_discovery_error(error) or self._stop.is_set():
log.warning("%s: payload policy rediscovery failed: %s", context, _poll_error_summary(error))
raise
failures += 1
delay = _transient_retry_delay(failures)
left = window() if window is not None else None
if left is None:
left = _UNKNOWN_LEASE_OUTCOME_RETRY_SECONDS - (time.monotonic() - started)
if left <= delay:
log.exception("%s: payload policy rediscovery failed %d times (%s) and no time is left",
context, failures, _poll_error_summary(error))
raise
log.warning("%s: payload policy rediscovery failed transiently (%s); retrying in %.2f s",
context, _poll_error_summary(error), delay)
await asyncio.sleep(delay)

async def _refresh_payload_policy(self, context: str,
window: Callable[[], float | None] | None = None) -> bool:
"""Rediscover the payload policy after a ``413 payload_too_large``.

Returns whether the policy changed, so a re-encoded resend could fit.
Discovery outages are retried with the shared backoff while ``window``
(seconds left, ``None`` when unknown) allows; the held discovery is
replaced only by a successful read. A discovery error propagates so
callers do not record a permanent size failure on an unknown policy.
"""
client = self.client
before = getattr(client, "_payload_policy_snapshot", lambda: None)()
await self._rediscover_cluster_info(context, window)
forget = getattr(client, "_forget_payload_policy", None)
if forget is not None:
forget()
after = await client._payload_policy_resolved_snapshot() if hasattr(
client, "_payload_policy_resolved_snapshot") else None
changed = after != before
log.warning("%s refused as payload_too_large; payload policy %s (%s -> %s)%s",
context, "changed" if changed else "unchanged", before, after,
"; resending once re-encoded" if changed else "")
return changed

async def _report_activity_task_failure(self, task_id: str, attempt_id: str,
context: str, **failure: Any) -> None:
try:
Expand Down
Loading