diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f78726..4afa472 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- A worker answer that names a capabilities digest the worker does not hold no longer strands the request (PHP and Rust parity). Before, it raised `ServerCapabilitiesDigestError`, and a leased task sat until its lease expired. In a lab run with garbled digests, 0 of 10 workflows finished in 150 s. + - **Non-idempotent requests are never resent.** For a completion, failure report or heartbeat, the client keeps the answer as received, since everything but the capabilities object is valid. It reads the capabilities with a read-only `GET /api/cluster/info` (`worker_protocol.server_capabilities`, the same object) and forgets its held digest, so later requests send `none` and learn a fresh copy. If discovery is unavailable, the field is dropped; no caller reads capabilities from those answers. + - **Task polls** are safe to repeat, so a poll is instead resent once with `none` to get the full object. An unknown digest on that resend, or in an answer to a request that already sent `none`, is a protocol error. + - **Error bodies:** an unknown digest in an error body is dropped. The real HTTP error surfaces unchanged and is not resent. +- A worker no longer exits when the runtime is warming up or briefly unavailable while it registers (same issue as durable-workflow/sdk-php#448). Before, `Worker.run()` and `Worker.run_until()` raised once the client's three transport attempts had failed on `GET /api/cluster/info` or `POST /api/worker/register`. That covered native Server's `503 {"reason":"storage_unavailable"}` during warm-up, a refused connection and an upstream 502/503/504. + - **Retried:** discovery and registration are now repeated with the backoff shared across SDKs, `uniform(0, min(4, 0.25 * 2**(n - 1)))` seconds, logging a warning each time. The transient failures are: refused, reset or timed-out connections; upstream 502/503/504 without a protocol body; 503 `storage_unavailable` or `backend_unavailable`; and any 5xx or 429 with `retryable: true`. + - **Still fatal:** `retryable: false`, HTTP 401/403, other 4xx refusals and an incompatible Server still stop the worker. It now logs `worker stopping: registration failed with a non-retryable error: ...` at error level first. + - **Activity outcomes:** an activity completion or failure report that hits one of those transients is now sent again with the same backoff while the attempt's lease covers the next wait. Before, it was dropped after the client's three quick attempts, with only a warning. A transient never turns a completion into a failure report. When the lease runs out, the worker stops resending and logs an error, and the Server settles the attempt at lease expiry. + - **Duplicate completions:** a resend answered `409 stale_attempt` with `recorded: false`, the same task, attempt and lease owner, and activity, attempt and task status all `completed` is this worker's committed completion and counts as delivered. That is what native Server 3.17.0 answers after a transient hid the committed first completion, and it is the same rule as PHP. Any other `stale_attempt` is a closed attempt and is never treated as success. + - **Poll-time auth refusals:** pollers still retry 401/403 so credentials can rotate under a running worker. PHP and Rust stop the worker instead. Every such refusal is now logged at ERROR level, not rate-limited as a warning, including refusals before a later recovery. + - **Poll backoff:** poll retry waits now follow `uniform(0, min(10, 0.25 * 2**(n - 1)))` seconds, the shared full jitter with a 10 s cap, while still honouring a bounded Server retry delay. Before, the ceiling started at 1 s and doubled to 30 s, which slowed recovery. + - **Long outages:** a poller failing continuously for 60 s now logs the outage at ERROR level, then every 5 minutes while it lasts. Before, it logged only a warning about once a minute. A successful poll resets this. + - **Heartbeats:** worker heartbeats already rode out these failures. - `WorkflowHandle.result()` and `Client.get_result()` on a handle bound to a run that continued as new no longer silently return `None` (#356). They now follow the continue-as-new chain to the run that actually ended, as the PHP and Rust SDKs do. That run's result is returned, or its typed failure is raised. The chain is followed through each `WorkflowContinuedAsNew` event's `continued_to_run_id`; when no successor is recorded, the newest run named by the status read is used. A chain that cannot be followed raises `WorkflowFailed`. - An activity worker no longer silently drops a claimed task when lease renewal fails transiently (#352, same bug as durable-workflow/sdk-php#445). Before, if the `renew-lease` confirmation sent right after an activity poll timed out, hit a connection reset or got a 5xx/429, the poll raised `ActivityLeaseAuthorityLost("activity_initial_lease_unavailable")` and the worker discarded the task. The Server then held the lease (300 s by default) until it expired and, with the default single attempt, failed the workflow. - **Keeps working:** the activity now runs under the lease it already holds, and renewal is retried inside that lease. The backoff is the same in every SDK: the *n*-th retry waits `uniform(0, min(4, 0.25 * 2**n))` seconds, so 0.25 s doubling to a 4 s cap, with full jitter. Each renewal exchange is bounded to leave a reserve of the lease (at most 10 s, at most a fifth of the grant). An unconfirmed lease is charged from the poll's start, because the Server stamps it only after receiving the poll. diff --git a/docs/reference/worker.md b/docs/reference/worker.md index 568a38d..fcda5eb 100644 --- a/docs/reference/worker.md +++ b/docs/reference/worker.md @@ -90,17 +90,31 @@ This backoff requires Python SDK 2.10.0 or later. Earlier packages use a fixed one-second poll-error wait. Workflow, activity and query pollers back off independently after failed polls, -including HTTP errors, rejected credentials and unreachable runtimes. The -retry ceiling starts at 1 second, doubles on each consecutive failure and stops -at 30 seconds. Each delay is chosen randomly between half and all of that -ceiling, so workers do not retry in lockstep. Any successful poll, including an -empty response, resets the ceiling to 1 second. Shutdown interrupts the wait. - -The first failure is logged immediately. An unchanged failure is logged at most -once per minute per poller until polling succeeds. HTTP 401 identifies rejected -authentication; HTTP 403 identifies denied authorization. Check the worker -credentials and their permissions. Once requests succeed again, polling -recovers without a worker restart. Poll-error metrics +including HTTP errors, rejected credentials and unreachable runtimes. The wait +before the *n*-th consecutive retry is `uniform(0, min(10, 0.25 * 2**(n - 1)))` +seconds. That is the shared full-jitter schedule with a 10-second cap, so a +poller recovers quickly once the runtime is back, and workers do not retry in +lockstep. A bounded retry delay advertised by the Server is still honoured. Any +successful poll, including an empty response, resets the schedule. Shutdown +interrupts the wait. + +The first failure is logged as a warning immediately. An unchanged failure is +logged at most once per minute per poller. Once a poller has failed +continuously for 60 seconds, the outage is logged at ERROR level, then again +every 5 minutes while it lasts. A successful poll ends the outage, and the next +failure starts again with a warning. + +Authentication and authorization refusals are the exception to that limit: +**every** poll answered with HTTP 401 (rejected authentication) or HTTP 403 +(denied authorization) is logged at ERROR level, including refusals that come +before a later recovery. Check the worker credentials and their permissions. +The poller keeps retrying with the same backoff, so rotated credentials +recover without a worker restart. + +This differs from the PHP and Rust SDKs, which log an error and stop the +worker on a poll-time 401 or 403. Python retries so that credentials can be +rotated under a running worker. Registration at startup treats 401 and 403 as +fatal in every SDK (see below). Poll-error metrics still count every failed poll, including failures whose warning is suppressed. Starting with Python SDK 2.10.0, warnings name the exception class and static @@ -114,6 +128,61 @@ Typed capacity backpressure follows the Server's advertised retry delay. Storage-admission failures retain their separate identity-preserving retry and worker-stop behavior. Neither uses this generic poll-failure backoff. +### Startup while the runtime is unavailable + +`Worker.run()` and `Worker.run_until()` also ride out a runtime that is warming +up or briefly unavailable while they register. Cluster discovery and +registration are repeated whole after a transient failure: + +- a refused, reset or timed-out connection +- an upstream 502, 503 or 504 without a protocol body +- a 503 whose reason is `storage_unavailable` or `backend_unavailable` + (native Server answers every storage failure with this body) +- any 5xx or 429 that says `retryable: true` + +The wait before the *n*-th retry is `uniform(0, min(4, 0.25 * 2**(n - 1)))` +seconds. This is the backoff schedule shared by every Durable Workflow SDK. Each +retry logs a warning, and shutdown interrupts the wait. + +Some failures are never retried: + +- `retryable: false` +- HTTP 401 or 403 +- any other 4xx protocol refusal +- an incompatible Server + +For these, the worker logs `worker stopping: registration failed with a +non-retryable error: ...` at error level, and `run()` raises the error. + +### Activity outcomes through transient failures + +An activity completion or failure report that hits one of the transient +failures listed above is sent again with the same backoff, as long as the +attempt's lease still covers the next wait. The lease is the renewed lease when +the Server supports activity lease renewal, and the claimed `lease_expires_at` +otherwise. A transient never turns a completion into a failure report. + +When the lease cannot cover another wait, the worker stops resending and logs +`activity task ... outcome not delivered after N transient failures ...: +its lease ran out` at error level. The Server then settles the attempt when its +lease expires. + +A transient can hide an outcome the Server already committed. Native Server +answers the resend with `409` and this body: + +```json +{"reason": "stale_attempt", "recorded": false, "outcome": "completed", + "lease_owner": "", "activity_status": "completed", + "attempt_status": "completed", "task_status": "completed"} +``` + +The body also carries the same task and attempt IDs. That exact shape counts +as this worker's committed completion, so the activity is not run again and +no second outcome is recorded. Any other `stale_attempt` means the attempt is +closed. It is reported as such and never treated as success. After +registration, pollers keep the poll-failure backoff described above. Worker +heartbeats log a failure and try again at the next interval. + ## Lifecycle and shutdown `Worker.run()` and `Worker.run_until()` register through the worker protocol diff --git a/src/durable_workflow/_capabilities_digest.py b/src/durable_workflow/_capabilities_digest.py index 9c07b78..e5538a9 100644 --- a/src/durable_workflow/_capabilities_digest.py +++ b/src/durable_workflow/_capabilities_digest.py @@ -45,13 +45,25 @@ def capture(self) -> Captured: def header(captured: Captured) -> str: return captured[0] if captured[0] is not None and captured[1] is not None else "none" - def restore(self, answer: Any, captured: Captured) -> None: - """Strip and learn tagged copies, then restore elided ones, in place.""" + def forget(self) -> None: + """Drop the held pair, so the next request sends ``none`` and gets full capabilities.""" + self._digest = None + self._capabilities = None + + def restore(self, answer: Any, captured: Captured, *, drop_unknown: bool = False, + substitute: dict[str, Any] | None = None) -> bool: + """Strip and learn tagged copies, then restore elided ones, in place. + + An elided copy naming a digest neither captured for the request nor + tagged in this answer raises :class:`ServerCapabilitiesDigestError`. + With ``substitute`` it is replaced by a copy of that object; with + ``drop_unknown`` it is removed. Either way ``True`` is returned. + """ if not isinstance(answer, dict): - return + return False learned: dict[str, dict[str, Any]] = {} self._learn(answer, learned, 1) - self._resolve(answer, captured, learned, 1) + return self._resolve(answer, captured, learned, 1, drop_unknown, substitute) def _learn(self, answer: dict[str, Any], learned: dict[str, dict[str, Any]], depth: int) -> None: for key, value in answer.items(): @@ -67,18 +79,26 @@ def _learn(self, answer: dict[str, Any], learned: dict[str, dict[str, Any]], dep self._learn(value, learned, depth + 1) def _resolve(self, answer: dict[str, Any], captured: Captured, learned: dict[str, dict[str, Any]], - depth: int) -> None: - for key, value in answer.items(): + depth: int, drop_unknown: bool, substitute: dict[str, Any] | None) -> bool: + dropped = False + for key, value in list(answer.items()): if key == _KEY: if isinstance(value, str) and value.startswith(_PREFIX): if value == captured[0] and captured[1] is not None: answer[key] = copy.deepcopy(captured[1]) elif value in learned: answer[key] = copy.deepcopy(learned[value]) + elif substitute is not None: + answer[key] = copy.deepcopy(substitute) + dropped = True + elif drop_unknown: + del answer[key] + dropped = True else: raise ServerCapabilitiesDigestError( "the Server elided server_capabilities this worker does not hold " f"({value}; sent {self.header(captured)})" ) elif isinstance(value, dict) and depth < _MAX_DEPTH: - self._resolve(value, captured, learned, depth + 1) + dropped = self._resolve(value, captured, learned, depth + 1, drop_unknown, substitute) or dropped + return dropped diff --git a/src/durable_workflow/client.py b/src/durable_workflow/client.py index 7bb8636..c2a16da 100644 --- a/src/durable_workflow/client.py +++ b/src/durable_workflow/client.py @@ -24,6 +24,7 @@ import hashlib import json import json as json_module +import logging import math import os import re @@ -79,6 +80,7 @@ NonDeterministicReplayError, RuntimeCapabilityUnsupported, RuntimeDiscoveryUnavailable, + ServerCapabilitiesDigestError, ServerError, SignalFailed, UnrecognizedUpdateOutcome, @@ -130,6 +132,8 @@ _MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL = (1, 15) CONTROL_PLANE_REQUEST_CONTRACT_SCHEMA = "durable-workflow.v2.control-plane-request.contract" CONTROL_PLANE_REQUEST_CONTRACT_VERSION = 1 +log = logging.getLogger("durable_workflow.client") + _QUERY_TASKS_DISCOVERY_PATH = "worker_protocol.server_capabilities.query_tasks" _COOPERATIVE_CANCELLATION_DISCOVERY_PATH = "worker_protocol.server_capabilities.cooperative_cancellation" _UPDATE_OUTCOME_READS_DISCOVERY_PATH = "capabilities.update_outcome_reads" @@ -2296,9 +2300,10 @@ async def _do_request() -> httpx.Response: raise return resp - try: + async def _answer() -> httpx.Response: + nonlocal status_code, outcome try: - resp = await self.retry_policy.execute(_do_request) if retry else await _do_request() + return await self.retry_policy.execute(_do_request) if retry else await _do_request() except httpx.HTTPStatusError as exc: status_code = str(exc.response.status_code) outcome = "http_error" @@ -2307,8 +2312,12 @@ async def _do_request() -> httpx.Response: body = exc.response.json() except ValueError: body = exc.response.text - if capabilities_digest is not None: - self._capabilities_digest.restore(body, capabilities_digest) + if capabilities_digest is not None and self._capabilities_digest.restore( + body, capabilities_digest, drop_unknown=True, + ): + # The HTTP error is the answer; an unusable digest in it + # never replaces it. Ask for full capabilities next time. + self._capabilities_digest.forget() # The typed completion-size refusal carries the limit; the # size is the encoded request body this client sent. if raise_for_status is _raise_for_status: @@ -2319,13 +2328,41 @@ async def _do_request() -> httpx.Response: raise_for_status(exc.response.status_code, body, context=context) raise # unreachable, but keeps type checker happy + try: + resp = await _answer() status_code = str(resp.status_code) if resp.status_code == 204 or not resp.content: outcome = "ok" return None result = resp.json() if capabilities_digest is not None: - self._capabilities_digest.restore(result, capabilities_digest) + try: + self._capabilities_digest.restore(result, capabilities_digest) + except ServerCapabilitiesDigestError as unknown: + if capabilities_digest[0] is None: + raise + # The answer named capabilities this worker does not hold. + # Forget the held pair so later requests send ``none``. + self._capabilities_digest.forget() + if not progress_poll: + # Never resend a non-idempotent request (a completion, + # failure or heartbeat) to recover capabilities: its + # resend can answer a duplicate without what the first + # answer carried. Keep this answer and read the + # capabilities from discovery instead. + result = await self._substitute_unknown_capabilities(result, capabilities_digest, path) + else: + # A task poll is safe to repeat: ask again for the full object. + log.warning("worker poll answer from %s named unknown capabilities (%s); resending once " + "for the full capabilities", path, unknown) + capabilities_digest = (None, None) + resp = await _answer() + status_code = str(resp.status_code) + if resp.status_code == 204 or not resp.content: + outcome = "ok" + return None + result = resp.json() + self._capabilities_digest.restore(result, capabilities_digest) if (worker and activity_authority is not None and activity_authority.governs(self, self.namespace, path) and path.endswith("/heartbeat") and isinstance(result, dict) @@ -3078,6 +3115,29 @@ async def _do_request() -> httpx.Response: self.metrics.increment(CLIENT_REQUESTS, tags=tags) self.metrics.record(CLIENT_REQUEST_DURATION_SECONDS, time.perf_counter() - start, tags=tags) + async def _substitute_unknown_capabilities(self, result: Any, captured: Any, path: str) -> Any: + """Replace an answer's unknown capabilities digest with discovery's object. + + ``worker_protocol.server_capabilities`` from ``GET /api/cluster/info`` + is the object the worker plane elides. If discovery is unavailable the + field is dropped: no caller reads capabilities from a non-poll answer. + """ + try: + info = await self.get_cluster_info() + protocol = info.get("worker_protocol") + capabilities = protocol.get("server_capabilities") if isinstance(protocol, dict) else None + if not isinstance(capabilities, dict): + raise ServerCapabilitiesDigestError("discovery returned no worker_protocol.server_capabilities") + except Exception as error: + log.warning("worker answer from %s named unknown capabilities and discovery failed (%r); " + "keeping the answer without them", path, error) + self._capabilities_digest.restore(result, captured, drop_unknown=True) + return result + log.warning("worker answer from %s named unknown capabilities; kept the answer and read the " + "capabilities from cluster discovery", path) + self._capabilities_digest.restore(result, captured, substitute=capabilities) + return result + async def get_cluster_info(self) -> dict[str, Any]: """Fetch server build identity, capabilities, and protocol manifests. diff --git a/src/durable_workflow/worker.py b/src/durable_workflow/worker.py index 395dd6c..4276e4c 100644 --- a/src/durable_workflow/worker.py +++ b/src/durable_workflow/worker.py @@ -283,6 +283,57 @@ def _is_storage_admission_error(error: BaseException) -> bool: ) +# Retry schedule after a transient runtime failure, the same in every SDK: +# uniform(0, min(MAX, BASE * 2**(n - 1))) for the n-th consecutive failure. +_TRANSIENT_RETRY_BASE_SECONDS = 0.25 +_TRANSIENT_RETRY_MAX_SECONDS = 4.0 +_UPSTREAM_UNAVAILABLE_STATUSES = (502, 503, 504) +# Retry bound for an activity outcome whose lease end is unknown. +_UNKNOWN_LEASE_OUTCOME_RETRY_SECONDS = 60.0 +# The claimed activity task this dispatch publishes outcomes for. +_current_activity_claim: contextvars.ContextVar[dict[str, Any] | None] = contextvars.ContextVar( + "durable_workflow_activity_claim", default=None, +) +_TRANSIENT_UNAVAILABLE_REASONS = ("storage_unavailable", "backend_unavailable") + + +def _transient_runtime_error(error: BaseException) -> bool: + """Whether a failed request may succeed unchanged once the runtime recovers. + + Transient: a refused, reset or timed-out connection; an upstream 502, 503 + or 504 without a protocol body; a 503 whose reason is + ``storage_unavailable`` or ``backend_unavailable`` (native Server answers + every storage failure with that body, for example during warm-up); and + any 5xx or 429 that says ``retryable: true``. Never ``retryable: false``, + authentication or authorization failures, or other protocol refusals. + """ + if isinstance(error, httpx.TransportError): + return True + if not isinstance(error, ServerError): + return False + body = error.body if isinstance(error.body, Mapping) else None + retryable = body.get("retryable") if body is not None else None + if retryable is False or not (error.status >= 500 or error.status == 429): + return False + if retryable is True: + return True + if error.status == 503 and error.reason() in _TRANSIENT_UNAVAILABLE_REASONS: + return True + return error.status in _UPSTREAM_UNAVAILABLE_STATUSES and (body is None or "reason" not in body) + + +# Poll retries use the same full jitter with a 10 s cap, so a poller recovers +# quickly after an outage. A long outage escalates from warnings to errors. +_POLL_RETRY_MAX_SECONDS = 10.0 +_POLL_OUTAGE_ERROR_AFTER_SECONDS = 60.0 +_POLL_OUTAGE_ERROR_EVERY_SECONDS = 300.0 + + +def _transient_retry_delay(failures: int) -> float: + exponent = min(max(failures - 1, 0), 16) + return random.uniform(0.0, min(_TRANSIENT_RETRY_MAX_SECONDS, _TRANSIENT_RETRY_BASE_SECONDS * 2 ** exponent)) + + def _positive_seconds(value: object) -> bool: return ( isinstance(value, (int, float)) @@ -292,6 +343,34 @@ def _positive_seconds(value: object) -> bool: ) +def _auth_refusal(error: BaseException) -> bool: + return isinstance(error, Unauthorized) or isinstance(error, ServerError) and error.status in (401, 403) + + +def _committed_own_activity_completion(error: BaseException, task_id: str, attempt_id: str, worker_id: str) -> bool: + """Whether a refusal answers a resend of a completion this worker already committed. + + After a transient hid a committed completion, native Server answers the + resend with ``409 stale_attempt``, ``recorded: false``, the same task, + attempt and lease owner, and activity, attempt and task all ``completed`` + (the same rule as PHP's ``Worker::isCommittedActivityCompletion``). Any + other ``stale_attempt`` is a closed attempt, not this worker's completion. + """ + if not isinstance(error, ServerError) or error.status != 409 or not isinstance(error.body, Mapping): + return False + body = error.body + return (body.get("reason") == "stale_attempt" and body.get("recorded") is False + and body.get("outcome") == "completed" and body.get("task_id") == task_id + and body.get("activity_attempt_id") == attempt_id and body.get("lease_owner") == worker_id + and all(body.get(key) == "completed" for key in ("activity_status", "attempt_status", "task_status"))) + + +def _closed_activity_attempt(error: BaseException, attempt_id: str) -> bool: + """A ``409 stale_attempt`` for this attempt: it is settled, so resending cannot record anything.""" + return (isinstance(error, ServerError) and error.status == 409 and isinstance(error.body, Mapping) + and error.body.get("reason") == "stale_attempt" and error.body.get("activity_attempt_id") == attempt_id) + + def _poll_error_summary(error: BaseException) -> str: error_class = type(error).__name__ if isinstance(error, Unauthorized) or isinstance(error, ServerError) and error.status == 401: @@ -314,16 +393,46 @@ def __init__(self) -> None: self.reset() def reset(self) -> None: - self._ceiling = 1.0 + self._failures = 0 + self._outage_started_at: float | None = None self._last_summary: str | None = None self._last_warning_at = 0.0 + self._last_error_at: float | None = None def retry_delay(self, error: Exception, task_kind: str, *, minimum_delay: float = 0.0) -> float: - delay = max(minimum_delay, random.uniform(self._ceiling / 2, self._ceiling)) - self._ceiling = min(30.0, self._ceiling * 2) + # Full jitter up to 10 s: uniform(0, min(10, 0.25 * 2**(n - 1))). A + # bounded Server retry delay (minimum_delay) is still honoured. + self._failures += 1 + exponent = min(self._failures - 1, 16) + delay = max(minimum_delay, random.uniform( + 0.0, min(_POLL_RETRY_MAX_SECONDS, _TRANSIENT_RETRY_BASE_SECONDS * 2 ** exponent), + )) summary = _poll_error_summary(error) now = time.monotonic() - if summary != self._last_summary or now - self._last_warning_at >= 60.0: + if self._outage_started_at is None: + self._outage_started_at = now + outage = now - self._outage_started_at + if _auth_refusal(error): + # Every refusal is loud. The poller still retries so rotated + # credentials recover without a restart; PHP and Rust stop instead. + log.error( + "%s poll error: %s; route=/worker/%s-tasks/poll; retrying in %.2f s so rotated credentials " + "can recover", task_kind, summary, task_kind, delay, + ) + self._last_summary = summary + self._last_warning_at = now + elif outage >= _POLL_OUTAGE_ERROR_AFTER_SECONDS: + # A long outage is an error: once when it crosses the threshold, + # then every few minutes while it lasts. A success resets it. + if self._last_error_at is None or now - self._last_error_at >= _POLL_OUTAGE_ERROR_EVERY_SECONDS: + log.error( + "%s poll has failed for %.0f s: %s; route=/worker/%s-tasks/poll; still retrying, next in %.2f s", + task_kind, outage, summary, task_kind, delay, + ) + self._last_error_at = now + self._last_summary = summary + self._last_warning_at = now + elif summary != self._last_summary or now - self._last_warning_at >= 60.0: log.warning( "%s poll error: %s; route=/worker/%s-tasks/poll; retrying in %.2f s", task_kind, summary, task_kind, delay, @@ -3607,14 +3716,16 @@ async def _run_activity_task_body(self, task: dict[str, Any]) -> str: result = to_payload_value(result) log.info("completing activity task %s (%s) -> %r", task_id, activity_type, result) try: - await self.client.complete_activity_task( - task_id=task_id, - activity_attempt_id=attempt_id, - lease_owner=self.worker_id, - result=result, - codec=result_codec, - activity_name=activity_type, - **self._external_storage_completion_kwargs(), + await self._deliver_activity_outcome( + task_id, attempt_id, "completed", lambda: self.client.complete_activity_task( + task_id=task_id, + activity_attempt_id=attempt_id, + lease_owner=self.worker_id, + result=result, + codec=result_codec, + activity_name=activity_type, + **self._external_storage_completion_kwargs(), + ), ) except Exception as e: kept = _run_closed_completion_kept(e) @@ -3682,12 +3793,77 @@ async def _decode_activity_arguments(self, task: dict[str, Any]) -> tuple[str, l return None return codec, args if isinstance(args, list) else [args] + def _activity_outcome_window(self, task_id: str) -> float | None: + """Seconds of lease left for publishing this attempt's outcome, if known.""" + authority = current_activity_authority.get() + if authority is not None and authority.task_id == task_id: + try: + return authority.remaining() + except ActivityWorkerLost: + return 0.0 + claim = _current_activity_claim.get() + expires = claim.get("lease_expires_at") if claim is not None and claim.get("task_id") == task_id else None + try: + deadline = datetime.fromisoformat(str(expires).replace("Z", "+00:00")) + except ValueError: + return None + if deadline.tzinfo is None: + return None + return (deadline - datetime.now(timezone.utc)).total_seconds() + + async def _deliver_activity_outcome(self, task_id: str, attempt_id: str, outcome: str, + publish: Callable[[], Awaitable[Any]]) -> None: + """Publish an activity outcome, repeating it through transients while the lease lasts. + + A transient never turns the outcome into another one. A resend answered + as a duplicate of this worker's own recorded outcome counts as + delivered. Once the lease cannot cover another attempt the last + failure is logged as an error and raised; the Server then settles the + attempt at lease expiry, and the result is never dropped silently. + """ + failures = 0 + started = time.monotonic() + while True: + try: + await publish() + return + except Exception as error: + # The client's own transport retry can resend too, so this + # holds on the first round as well. + if outcome == "completed" and _committed_own_activity_completion( + error, task_id, attempt_id, self.worker_id, + ): + log.info("activity task %s: the Server had already committed this worker's completion " + "(resend answered 409 stale_attempt)", task_id) + return + if _closed_activity_attempt(error, attempt_id): + # Not provably this worker's outcome: report the closed attempt as it is. + log.warning("activity task %s %s outcome resend found the attempt already closed: %s", + task_id, outcome, _poll_error_summary(error)) + raise + if not _transient_runtime_error(error): + raise + failures += 1 + delay = _transient_retry_delay(failures) + window = self._activity_outcome_window(task_id) + if window is None: + window = _UNKNOWN_LEASE_OUTCOME_RETRY_SECONDS - (time.monotonic() - started) + summary = _poll_error_summary(error) + if window <= delay: + log.error("activity task %s %s outcome not delivered after %d transient failures (%s): " + "its lease ran out; the Server settles the attempt when the lease expires", + task_id, outcome, failures, summary) + raise + log.warning("activity task %s %s outcome failed transiently (%s); retrying in %.2f s", + task_id, outcome, summary, delay) + await asyncio.sleep(delay) + async def _report_activity_task_failure(self, task_id: str, attempt_id: str, context: str, **failure: Any) -> None: try: - await self.client.fail_activity_task( + await self._deliver_activity_outcome(task_id, attempt_id, "failed", lambda: self.client.fail_activity_task( task_id=task_id, activity_attempt_id=attempt_id, lease_owner=self.worker_id, **failure, - ) + )) except Exception as error: if _run_closed_refusal(error): log.info("%s for activity task %s not recorded: its workflow run closed", context, task_id) @@ -4342,6 +4518,13 @@ async def _poll_activity_tasks(self) -> None: self._track(self._dispatch_activity_task(task)) async def _dispatch_activity_task(self, task: dict[str, Any]) -> None: + claim = _current_activity_claim.set(task) + try: + await self._dispatch_claimed_activity_task(task) + finally: + _current_activity_claim.reset(claim) + + async def _dispatch_claimed_activity_task(self, task: dict[str, Any]) -> None: self._record_task_schedule_to_start("activity", task) task_start = time.perf_counter() outcome = "error" @@ -4707,12 +4890,47 @@ async def run(self) -> None: raise primary_error.with_traceback(primary_traceback) from shutdown_error raise + async def _register_through_transients(self) -> None: + """Register, repeating through transient runtime failures (sdk-php#448). + + A runtime that is warming up or briefly unavailable (for example + native Server's ``503 storage_unavailable``) must not end the worker. + Discovery and registration are repeated whole with the shared backoff; + any other failure is raised. Shutdown interrupts the wait and surfaces + the last failure. + """ + failures = 0 + while True: + try: + return await self._register() + except Exception as error: + # Discovery failures arrive wrapped as a compatibility error. + cause = error.__cause__ if isinstance(error, RuntimeError) else None + if not (_transient_runtime_error(error) or cause is not None and _transient_runtime_error(cause)): + raise + if self._stop.is_set(): + raise + failures += 1 + delay = _transient_retry_delay(failures) + log.warning("worker registration failed transiently: %s; retrying in %.2f s", + _poll_error_summary(cause if cause is not None else error), delay) + with contextlib.suppress(asyncio.TimeoutError): + await asyncio.wait_for(self._stop.wait(), timeout=delay) + if self._stop.is_set(): + raise + async def _register_for_run(self) -> None: try: - await self._register() + await self._register_through_transients() except BaseException as error: if isinstance(error, Exception): self._status.failed("registration") + if not self._stop.is_set(): + summary = (_poll_error_summary(error) + if isinstance(error, ServerError | Unauthorized | httpx.TransportError) + else f"{type(error).__name__}: {error}") + log.error("worker %s stopping: registration failed with a non-retryable error: %s", + self.worker_id, summary) if self._shutdown_task is None: # run() leaves no shutdown behind a failed registration. self._cancel_status_tick() @@ -4920,7 +5138,7 @@ async def run_until( async def _register_within(self, timeout: float) -> None: try: - await asyncio.wait_for(self._register(), timeout=timeout) + await asyncio.wait_for(self._register_through_transients(), timeout=timeout) except Exception: self._status.failed("registration") raise diff --git a/tests/test_capabilities_digest_unknown.py b/tests/test_capabilities_digest_unknown.py new file mode 100644 index 0000000..c3bf137 --- /dev/null +++ b/tests/test_capabilities_digest_unknown.py @@ -0,0 +1,270 @@ +"""Recovery from an answer naming an unknown capabilities digest (PHP and Rust parity). + +A non-idempotent request (completion, failure, heartbeat) is never resent: its +answer is kept and the capabilities are read from cluster discovery. Only task +polls, which are safe to repeat, are resent without the digest. +""" +from __future__ import annotations + +import asyncio +import copy +import logging +from typing import Any + +import httpx +import pytest + +from durable_workflow import Client, Worker, activity, serializer +from durable_workflow._capabilities_digest import HEADER, ServerCapabilitiesDigest +from durable_workflow.errors import ServerCapabilitiesDigestError, ServerError +from durable_workflow.retry_policy import TransportRetryPolicy + +from .test_worker import compatible_cluster_info + +D1, D2, GARBLED = "sha256:" + "1" * 64, "sha256:" + "2" * 64, "sha256:" + "f" * 64 +CAPS = {"activity_lease_renewal": False, "v": 1} +DISCOVERED = {"activity_lease_renewal": False, "v": "discovered"} + + +def tagged(digest: str, capabilities: dict[str, Any] | None = None) -> dict[str, Any]: + return {**copy.deepcopy(capabilities or CAPS), "digest": digest} + + +def test_drop_unknown_removes_the_field_and_forget_resets_the_header() -> None: + held = ServerCapabilitiesDigest() + held.restore({"server_capabilities": tagged(D1)}, held.capture()) + body = {"reason": "conflict", "server_capabilities": GARBLED, "answer": {"server_capabilities": D1}} + assert held.restore(body, held.capture(), drop_unknown=True) is True + assert "server_capabilities" not in body + assert body["answer"]["server_capabilities"] == CAPS + assert held.restore({"server_capabilities": D1}, held.capture(), drop_unknown=True) is False + held.forget() + assert held.header(held.capture()) == "none" + + +class GarblingServer: + """Answers worker requests per a script of (status, capabilities) after learning D1.""" + + def __init__(self, script: list[tuple[int, Any]] | None = None) -> None: + self.script = list(script or []) + self.sent: list[tuple[str, str | None]] = [] + self.current = D1 + self.discovery = 0 + + async def respond(self, request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/cluster/info"): + self.discovery += 1 + assert request.method == "GET" and HEADER not in request.headers + return httpx.Response(200, json={"worker_protocol": {"server_capabilities": DISCOVERED}}) + sent = request.headers.get(HEADER) + self.sent.append((request.url.path, sent)) + capabilities: Any + if self.script and sent != "none": + status, capabilities = self.script.pop(0) + else: + status, capabilities = 200, self.current if sent == self.current else tagged(self.current) + body = {"ok": True, "path": request.url.path, "server_capabilities": capabilities} + if status >= 400: + body = {"reason": "stale_attempt", "message": "refused", "server_capabilities": capabilities} + return httpx.Response(status, json=body) + + +def client_for(server: GarblingServer) -> Client: + client = Client("http://digest.invalid", namespace="17", worker_token="worker-token", + retry_policy=TransportRetryPolicy(max_attempts=1)) + client._http = httpx.AsyncClient(base_url="http://digest.invalid", transport=httpx.MockTransport(server.respond)) + return client + + +async def learned(client: Client) -> None: + await client._request("POST", "/worker/learn", worker=True, json={}) + assert client._capabilities_digest.capture()[0] == D1 + + +async def test_a_garbled_answer_to_a_completion_is_kept_and_not_resent() -> None: + server = GarblingServer() + async with client_for(server) as client: + await learned(client) + server.script = [(200, GARBLED)] + answer = await client._request("POST", "/worker/activity-tasks/t/complete", worker=True, json={}) + after = await client._request("POST", "/worker/b", worker=True, json={}) + assert server.sent[1:] == [("/api/worker/activity-tasks/t/complete", D1), ("/api/worker/b", "none")], ( + "the completion is sent once; the held pair is forgotten afterwards" + ) + assert answer["path"] == "/api/worker/activity-tasks/t/complete", "the completion's own answer is used" + assert answer["server_capabilities"] == DISCOVERED, "capabilities come from cluster discovery" + assert server.discovery == 1 + assert after["server_capabilities"] == CAPS + + +async def test_without_discovery_the_answer_is_kept_without_its_capabilities() -> None: + async def respond(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/cluster/info"): + return httpx.Response(503, json={"reason": "storage_unavailable"}) + return httpx.Response(200, json={"recorded": True, "server_capabilities": GARBLED}) + + async with Client("http://digest.invalid", worker_token="w", + retry_policy=TransportRetryPolicy(max_attempts=1)) as client: + client._http = httpx.AsyncClient(base_url="http://digest.invalid", transport=httpx.MockTransport(respond)) + client._capabilities_digest.restore({"server_capabilities": tagged(D1)}, (None, None)) + answer = await client._request("POST", "/worker/heartbeat", worker=True, json={}) + assert answer == {"recorded": True} + + +async def test_a_garbled_poll_answer_is_resent_once_without_the_digest() -> None: + server = GarblingServer() + async with client_for(server) as client: + await learned(client) + server.script = [(200, GARBLED)] + server.current = D2 # the capabilities changed; the garbled answer names neither + poll = {"worker_id": "w", "task_queue": "q"} + answer = await client._request("POST", "/worker/query-tasks/poll", worker=True, json=poll) + after = await client._request("POST", "/worker/b", worker=True, json={}) + assert server.sent[1:] == [ + ("/api/worker/query-tasks/poll", D1), ("/api/worker/query-tasks/poll", "none"), ("/api/worker/b", D2), + ] + assert answer["server_capabilities"] == CAPS, "the resend's full copy, not the garbled digest" + assert server.discovery == 0 + assert after["server_capabilities"] == CAPS + + +async def test_an_unknown_digest_on_the_poll_resend_is_a_protocol_error() -> None: + async def respond(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={"task": None, "server_capabilities": GARBLED}) + + poll = {"worker_id": "w", "task_queue": "q"} + async with Client("http://digest.invalid", worker_token="w") as client: + client._http = httpx.AsyncClient(base_url="http://digest.invalid", transport=httpx.MockTransport(respond)) + client._capabilities_digest.restore({"server_capabilities": tagged(D1)}, (None, None)) + with pytest.raises(ServerCapabilitiesDigestError): + await client._request("POST", "/worker/query-tasks/poll", worker=True, json=poll) + # Sent with ``none`` already: no resend, the answer is a protocol error. + with pytest.raises(ServerCapabilitiesDigestError): + await client._request("POST", "/worker/query-tasks/poll", worker=True, json=poll) + + +async def test_a_garbled_error_body_surfaces_the_http_error_unchanged() -> None: + server = GarblingServer() + async with client_for(server) as client: + await learned(client) + server.script = [(409, GARBLED)] + with pytest.raises(ServerError) as refusal: + await client._request("POST", "/worker/refused", worker=True, json={}) + await client._request("POST", "/worker/next", worker=True, json={}) + assert refusal.value.status == 409 + assert refusal.value.body == {"reason": "stale_attempt", "message": "refused"} + assert server.sent[1:] == [("/api/worker/refused", D1), ("/api/worker/next", "none")], "no resend" + + +@activity.defn(name="digest.echo") +def echo(value: str) -> str: + calls.append(value) + return value + + +calls: list[str] = [] + + +class Runtime: + """A worker runtime whose first poll and first completion answers are garbled.""" + + def __init__(self, *, garble_poll: bool, garble_completion: bool) -> None: + info = compatible_cluster_info() + info["worker_protocol"]["server_capabilities"].pop("worker_deregistration_fencing", None) # type: ignore[index] + self.info = info + self.garble_poll, self.garble_completion = garble_poll, garble_completion + self.polls: list[str | None] = [] + self.completions: list[str | None] = [] + self.failures = 0 + self.task_served = False + + def caps(self, sent: str | None) -> Any: + return D1 if sent == D1 else tagged(D1) + + async def respond(self, request: httpx.Request) -> httpx.Response: + path, sent = request.url.path, request.headers.get(HEADER) + if path.endswith("/cluster/info"): + return httpx.Response(200, json=self.info) + if path.endswith("/register"): + return httpx.Response(200, json={"worker_id": "worker-1", "registered": True, + "server_capabilities": self.caps(sent)}) + if path.endswith("/activity-tasks/poll"): + self.polls.append(sent) + await asyncio.sleep(0.01) + caps = self.caps(sent) + if self.garble_poll and sent == D1: + self.garble_poll = False + caps = GARBLED + task = None + if not self.task_served and self.garble_completion and sent is not None and caps != GARBLED: + self.task_served = True + task = { + "task_id": "task-1", "activity_attempt_id": "attempt-1", "attempt_number": 1, + "activity_type": "digest.echo", "workflow_id": "wf-1", "run_id": "run-1", + "lease_owner": "worker-1", "task_queue": "q", + "arguments": serializer.envelope(["once"], codec="avro"), "payload_codec": "avro", + } + return httpx.Response(200, json={"task": task, "server_capabilities": caps}) + if path.endswith("/complete"): + self.completions.append(sent) + if len(self.completions) == 1: + # Committed, but the answer names capabilities the worker never held. + return httpx.Response(200, json={"outcome": "completed", "recorded": True, + "server_capabilities": GARBLED}) + # A resend would only get this duplicate; it must never be sent. + return httpx.Response(409, json={ + "task_id": "task-1", "activity_attempt_id": "attempt-1", "outcome": "completed", + "recorded": False, "reason": "stale_attempt", "next_task_id": None, "lease_owner": "worker-1", + "activity_status": "completed", "attempt_status": "completed", "task_status": "completed", + "server_capabilities": tagged(D2), + }) + if path.endswith("/fail"): + self.failures += 1 + return httpx.Response(200, json={"outcome": "failed", "recorded": True}) + if path.endswith("/registrations/worker-1"): + return httpx.Response(200, json={"worker_id": "worker-1", "outcome": "deregistered", + "recovered_workflow_task_count": 0}) + return httpx.Response(200, json={"worker_id": "worker-1", "acknowledged": True, + "server_capabilities": self.caps(sent)}) + + +async def run_worker_until(runtime: Runtime, done: Any) -> Worker: + client = Client("http://runtime.invalid", token="token", retry_policy=TransportRetryPolicy(max_attempts=1)) + client._http = httpx.AsyncClient(base_url="http://runtime.invalid", transport=httpx.MockTransport(runtime.respond)) + worker = Worker(client, task_queue="q", workflows=[], activities=[echo], worker_id="worker-1") + running = asyncio.create_task(worker.run()) + try: + for _ in range(400): + if done(): + break + await asyncio.sleep(0.01) + assert not running.done(), running.exception() if running.done() else None + return worker + finally: + await worker.stop() + running.cancel() + await asyncio.gather(running, return_exceptions=True) + await client.aclose() + + +async def test_a_garbled_poll_answer_is_resent_in_full_and_the_worker_keeps_polling() -> None: + runtime = Runtime(garble_poll=True, garble_completion=False) + await run_worker_until(runtime, lambda: len(runtime.polls) >= 6) + garbled_at = runtime.polls.index(D1) + assert runtime.polls[garbled_at + 1] == "none", "the garbled poll is resent without the digest" + assert len(runtime.polls) >= 6, "the worker keeps polling" + + +async def test_a_garbled_completion_answer_is_kept_without_a_resend( + caplog: pytest.LogCaptureFixture, +) -> None: + caplog.set_level(logging.INFO, logger="durable_workflow.worker") + calls.clear() + runtime = Runtime(garble_poll=False, garble_completion=True) + await run_worker_until(runtime, lambda: len(runtime.completions) >= 1 and len(runtime.polls) >= 4) + assert calls == ["once"], "the activity ran once" + assert runtime.completions == [D1], "the completion is never resent" + assert runtime.failures == 0, "no failure report" + assert len(runtime.polls) >= 4, "the worker keeps running" + assert runtime.polls[-1] in ("none", D1), "later requests re-learn the capabilities" + assert "failed to complete" not in caplog.text, "the completion's own recorded answer is used" diff --git a/tests/test_poll_failure_backoff.py b/tests/test_poll_failure_backoff.py index f6058ca..af9cab3 100644 --- a/tests/test_poll_failure_backoff.py +++ b/tests/test_poll_failure_backoff.py @@ -78,7 +78,7 @@ async def sleep(delay: float) -> None: await asyncio.wait_for(worker._poll_workflow_tasks(), timeout=2) await asyncio.gather(*worker._in_flight) await real_sleep(0) # Let task done callbacks return their reservations. - assert delays == [1, 2, 4] + assert delays == [0.25, 0.5, 1] client.complete_workflow_task.assert_awaited_once() publication = client.complete_workflow_task.call_args.kwargs assert publication["task_id"] == "recovered-task" @@ -137,7 +137,8 @@ async def dispatch(task: object, **_: object) -> None: await getattr(worker, f"_poll_{kind}_tasks")() await asyncio.gather(*worker._in_flight) - assert delays == [1, 2, 4, 8, 16, 30, 30, 30, 30, 1] + # Full jitter capped at 10 s: uniform(0, min(10, 0.25 * 2**(n - 1))). + assert delays == [0.25, 0.5, 1, 2, 4, 8, 10, 10, 10, 0.25] assert calls == 12 assert len(dispatched) == int(successful_task) # Failed/empty polls must release their reservations before retrying. @@ -155,8 +156,43 @@ def uniform(low: float, high: float) -> float: monkeypatch.setattr(random, "uniform", uniform) backoff = worker_module._PollFailureBackoff() delays = [backoff.retry_delay(Unauthorized(), "workflow") for _ in range(8)] - assert delays == [0.5, 2, 2, 8, 8, 30, 15, 30] - assert bounds == [(0.5, 1), (1, 2), (2, 4), (4, 8), (8, 16), (15, 30), (15, 30), (15, 30)] + assert delays == [0, 0.5, 0, 2, 0, 8, 0, 10] + assert bounds == [(0, 0.25), (0, 0.5), (0, 1), (0, 2), (0, 4), (0, 8), (0, 10), (0, 10)] + + +def test_a_bounded_server_retry_delay_is_still_honoured(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(random, "uniform", lambda low, high: high) + backoff = worker_module._PollFailureBackoff() + assert backoff.retry_delay(ServerError(503, {}), "workflow", minimum_delay=7) == 7 + assert backoff.retry_delay(ServerError(503, {}), "workflow", minimum_delay=0.1) == 0.5 + + +def test_a_long_outage_escalates_to_errors_and_resets_after_success( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, +) -> None: + now = [1000.0] + clock = SimpleNamespace(monotonic=lambda: now[0], perf_counter=time.perf_counter) + monkeypatch.setattr(worker_module, "time", clock) + backoff = worker_module._PollFailureBackoff() + error = ServerError(503, {"reason": "storage_unavailable"}) + levels: list[tuple[float, int]] = [] + with caplog.at_level(logging.WARNING, logger="durable_workflow.worker"): + for at in (0, 30, 59, 61, 120, 300, 362, 500, 663): + now[0] = 1000.0 + at + before = len(caplog.records) + backoff.retry_delay(error, "activity") + levels.extend((at, record.levelno) for record in caplog.records[before:]) + backoff.reset() # a successful poll + now[0] = 2000.0 + before = len(caplog.records) + backoff.retry_delay(error, "activity") + levels.extend(("after success", record.levelno) for record in caplog.records[before:]) + # Warning at the start; error once the outage reaches 60 s, then every 5 minutes. + assert levels == [ + (0, logging.WARNING), (61, logging.ERROR), (362, logging.ERROR), (663, logging.ERROR), + ("after success", logging.WARNING), + ] + assert "activity poll has failed for 61 s" in caplog.text @pytest.mark.parametrize("kind", POLL_KINDS) @@ -176,13 +212,15 @@ async def sleep(delay: float) -> None: worker._stop.set() monkeypatch.setattr(worker_module.asyncio, "sleep", sleep) + monkeypatch.setattr(random, "uniform", lambda low, high: high) with caplog.at_level(logging.WARNING, logger="durable_workflow.worker"): await getattr(worker, f"_poll_{kind}_tasks")() - warnings = [r.getMessage() for r in caplog.records if r.name == "durable_workflow.worker"] - assert len(warnings) == 2 - assert "HTTP 401" in warnings[0] - assert "HTTP 403" in warnings[1] - assert delays[2] >= 2 # A changed failure does not reset the request backoff. + records = [r for r in caplog.records if r.name == "durable_workflow.worker"] + # Every auth refusal is an error, not only a changed one. + assert [r.levelno for r in records] == [logging.ERROR] * 4 + assert ["HTTP 401" in r.getMessage() for r in records] == [True, True, False, False] + assert ["HTTP 403" in r.getMessage() for r in records] == [False, False, True, True] + assert delays == [0.25, 0.5, 1, 2] # A changed failure does not reset the request backoff. @pytest.mark.parametrize("kind", POLL_KINDS) @@ -190,7 +228,7 @@ async def sleep(delay: float) -> None: (Unauthorized("private response"), "authentication failed (HTTP 401)"), (ServerError(403, {"message": "private response"}), "authorization failed (HTTP 403)"), ]) -async def test_identical_auth_warnings_are_rate_limited_and_reset_after_success( +async def test_every_auth_refusal_is_an_error_and_polling_recovers_after_success( kind: str, error: Exception, diagnostic: str, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, ) -> None: @@ -223,12 +261,16 @@ async def sleep(delay: float) -> None: with caplog.at_level(logging.WARNING, logger="durable_workflow.worker"): await getattr(worker, f"_poll_{kind}_tasks")() - warnings = [r.getMessage() for r in caplog.records if r.name == "durable_workflow.worker"] - # First refusal at 0, reminders at 61, 121 and 181, then the post-success streak. - assert len(warnings) == 5 - assert all(diagnostic in message and "credentials" in message for message in warnings) + records = [r for r in caplog.records if r.name == "durable_workflow.worker"] + # Unlike other poll errors, auth refusals are never rate-limited: each of + # the 12 is logged at ERROR, including the one after a later success. The + # poller keeps retrying so rotated credentials recover without a restart. + assert len(records) == 12 + assert all(r.levelno == logging.ERROR for r in records) + assert all(diagnostic in r.getMessage() and "rotated credentials" in r.getMessage() for r in records) assert "private response" not in caplog.text assert outcomes.count("error") == 12 + assert outcomes.count("empty") == 2, "polling recovered after the refusals" @pytest.mark.parametrize("kind", POLL_KINDS) @@ -257,7 +299,7 @@ async def test_shutdown_interrupts_a_capped_backoff(kind: str, monkeypatch: pyte real_sleep = asyncio.sleep async def sleep(delay: float) -> None: - if delay == 30: + if delay == 10: waiting.set() await real_sleep(delay) @@ -312,4 +354,4 @@ async def sleep(delay: float) -> None: async with client: await getattr(worker, f"_poll_{kind}_tasks")() assert calls == 6 - assert delays == [1, 2, 4, 1] + assert delays == [0.25, 0.5, 1, 0.25] diff --git a/tests/test_worker.py b/tests/test_worker.py index 6c2959b..a7fe974 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -965,7 +965,7 @@ async def test_malformed_poll_warning_omits_large_server_response( ) async def stop_after_backoff(seconds: float) -> None: - assert 0.5 <= seconds <= 1.0 + assert 0.0 <= seconds <= 0.25 # first retry: uniform(0, 0.25) worker._stop.set() monkeypatch.setattr(worker_module.asyncio, "sleep", stop_after_backoff) diff --git a/tests/test_worker_transient_runtime.py b/tests/test_worker_transient_runtime.py new file mode 100644 index 0000000..09fac2a --- /dev/null +++ b/tests/test_worker_transient_runtime.py @@ -0,0 +1,318 @@ +"""A runtime that is warming up or briefly unavailable does not end the worker (sdk-php#448).""" +from __future__ import annotations + +import asyncio +import logging +from typing import Any + +import httpx +import pytest + +from durable_workflow import Client, Worker, activity +from durable_workflow.errors import ServerError, Unauthorized +from durable_workflow.retry_policy import TransportRetryPolicy +from durable_workflow.worker import _transient_retry_delay, _transient_runtime_error + +from .test_worker import compatible_cluster_info + +STORAGE_UNAVAILABLE = {"reason": "storage_unavailable", "message": "storage_unavailable"} + + +@pytest.mark.parametrize(("error", "transient"), [ + (httpx.ConnectError("refused"), True), + (httpx.ReadTimeout("timed out"), True), + (httpx.RemoteProtocolError("reset"), True), + (ServerError(503, STORAGE_UNAVAILABLE), True), + (ServerError(503, {"reason": "backend_unavailable"}), True), + (ServerError(502, "bad gateway"), True), + (ServerError(504, {}), True), + (ServerError(500, {"reason": "x", "retryable": True}), True), + (ServerError(429, {"reason": "rate_limited", "retryable": True}), True), + (ServerError(503, {"reason": "storage_unavailable", "retryable": False}), False), + (ServerError(503, {"reason": "storage_pressure"}), False), + (ServerError(500, {"reason": "internal"}), False), + (ServerError(500, "boom"), False), + (ServerError(429, {"reason": "rate_limited"}), False), + (ServerError(401, {"reason": "unauthorized", "retryable": True}), False), + (ServerError(403, {}), False), + (ServerError(409, {"reason": "conflict"}), False), + (Unauthorized(), False), + (RuntimeError("incompatible"), False), +]) +def test_transient_classification(error: Exception, transient: bool) -> None: + assert _transient_runtime_error(error) is transient + + +@pytest.mark.parametrize(("failures", "ceiling"), [(1, 0.25), (2, 0.5), (3, 1.0), (4, 2.0), (5, 4.0), (12, 4.0)]) +def test_transient_retry_uses_the_shared_full_jitter_schedule( + monkeypatch: pytest.MonkeyPatch, failures: int, ceiling: float, +) -> None: + calls: list[tuple[float, float]] = [] + monkeypatch.setattr("durable_workflow.worker.random.uniform", lambda low, high: calls.append((low, high)) or high) + assert _transient_retry_delay(failures) == ceiling + assert calls == [(0.0, ceiling)] + + +@activity.defn(name="transient.noop") +def noop() -> None: + return None + + +class FlakyRuntime: + def __init__(self, path: str, failures: int, response: Any = None) -> None: + self.path, self.failures = path, failures + self.response = response if response is not None else (503, STORAGE_UNAVAILABLE) + self.served = 0 + self.polls = 0 + info = compatible_cluster_info() + info["worker_protocol"]["server_capabilities"].pop("worker_deregistration_fencing", None) # type: ignore[index] + self.info = info + + async def respond(self, request: httpx.Request) -> httpx.Response: + path = request.url.path + if path.endswith(self.path) and self.served < self.failures: + self.served += 1 + if self.response == "refused": + raise httpx.ConnectError("connection refused", request=request) + status, body = self.response + return httpx.Response(status, json=body) + if path.endswith("/cluster/info"): + return httpx.Response(200, json=self.info) + if path.endswith("/register"): + return httpx.Response(200, json={"worker_id": "worker-1", "registered": True}) + if path.endswith("/poll"): + self.polls += 1 + await asyncio.sleep(0.01) + return httpx.Response(200, json={"task": None}) + if path.endswith("/registrations/worker-1"): + return httpx.Response(200, json={"worker_id": "worker-1", "outcome": "deregistered", + "recovered_workflow_task_count": 0}) + return httpx.Response(200, json={"worker_id": "worker-1", "acknowledged": True}) + + +def worker_for(runtime: FlakyRuntime) -> tuple[Client, Worker]: + client = Client("http://runtime.invalid", token="token", retry_policy=TransportRetryPolicy(max_attempts=1)) + client._http = httpx.AsyncClient(base_url="http://runtime.invalid", transport=httpx.MockTransport(runtime.respond)) + return client, Worker(client, task_queue="q", workflows=[], activities=[noop], worker_id="worker-1") + + +@pytest.fixture +def fast_retries(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("durable_workflow.worker._TRANSIENT_RETRY_BASE_SECONDS", 0.005) + monkeypatch.setattr("durable_workflow.worker._TRANSIENT_RETRY_MAX_SECONDS", 0.02) + + +@pytest.mark.usefixtures("fast_retries") +@pytest.mark.parametrize(("path", "response", "failures"), [ + ("/cluster/info", None, 8), + ("/register", None, 8), + ("/register", "refused", 8), + ("/register", (502, None), 8), + ("/register", (503, {"reason": "backend_unavailable"}), 8), + # Pollers already ride out failures with their own backoff (1 s, doubling). + ("/activity-tasks/poll", None, 2), +]) +async def test_transient_failures_then_recovery_leave_the_worker_running( + path: str, response: Any, failures: int, caplog: pytest.LogCaptureFixture, +) -> None: + runtime = FlakyRuntime(path, failures=failures, response=response) + client, worker = worker_for(runtime) + running = asyncio.create_task(worker.run()) + try: + for _ in range(500): + if runtime.served == runtime.failures and runtime.polls >= 3: + break + await asyncio.sleep(0.01) + assert not running.done(), running.exception() if running.done() else None + assert runtime.served == runtime.failures + assert runtime.polls >= 3, "the worker polls once the runtime recovers" + assert worker._registered is True + assert not [record for record in caplog.records if record.levelno >= logging.ERROR] + finally: + await worker.stop() + running.cancel() + await asyncio.gather(running, return_exceptions=True) + await client.aclose() + + +@pytest.mark.usefixtures("fast_retries") +@pytest.mark.parametrize(("path", "response", "expected"), [ + ("/register", (401, {"reason": "unauthorized", "message": "bad token"}), Unauthorized), + ("/register", (403, {"reason": "forbidden"}), ServerError), + ("/register", (503, {"reason": "storage_unavailable", "retryable": False}), ServerError), + ("/cluster/info", (401, {"message": "bad token"}), RuntimeError), +]) +async def test_fatal_failures_stop_the_worker_with_a_clear_log( + path: str, response: Any, expected: type[Exception], caplog: pytest.LogCaptureFixture, +) -> None: + runtime = FlakyRuntime(path, failures=100, response=response) + client, worker = worker_for(runtime) + try: + with caplog.at_level(logging.ERROR, logger="durable_workflow.worker"), pytest.raises(expected): + await asyncio.wait_for(worker.run(), timeout=5) + assert runtime.served == 1, "a fatal failure is not retried" + assert runtime.polls == 0 + errors = [record.getMessage() for record in caplog.records if record.levelno >= logging.ERROR] + assert any("stopping: registration failed with a non-retryable error" in message for message in errors) + finally: + await client.aclose() + + +@pytest.mark.usefixtures("fast_retries") +async def test_shutdown_interrupts_a_transient_retry() -> None: + runtime = FlakyRuntime("/cluster/info", failures=10_000) + client, worker = worker_for(runtime) + running = asyncio.create_task(worker.run()) + try: + while runtime.served < 3: + await asyncio.sleep(0.005) + worker._stop.set() + with pytest.raises(RuntimeError, match="unable to read /api/cluster/info"): + await asyncio.wait_for(running, timeout=2) + finally: + running.cancel() + await asyncio.gather(running, return_exceptions=True) + await client.aclose() + + +# ── Activity outcomes through transients ────────────────────────────────── + +def activity_task(lease_seconds: float = 300.0) -> dict[str, Any]: + from datetime import datetime, timedelta, timezone + + from durable_workflow import serializer + + expires = datetime.now(timezone.utc) + timedelta(seconds=lease_seconds) + return { + "task_id": "task-1", "activity_attempt_id": "attempt-1", "attempt_number": 1, + "activity_type": "transient.echo", "workflow_id": "wf-1", "run_id": "run-1", + "lease_owner": "worker-1", "task_queue": "q", + "lease_expires_at": expires.isoformat(timespec="microseconds").replace("+00:00", "Z"), + "arguments": serializer.envelope(["value"], codec="avro"), "payload_codec": "avro", + } + + +@activity.defn(name="transient.echo") +def echo(value: str) -> str: + return value + + +@activity.defn(name="transient.boom") +def boom(value: str) -> str: + raise ValueError("application failure") + + +def committed_duplicate(**changes: Any) -> ServerError: + body = { + "task_id": "task-1", "activity_attempt_id": "attempt-1", "outcome": "completed", "recorded": False, + "reason": "stale_attempt", "next_task_id": None, "lease_owner": "worker-1", + "activity_status": "completed", "attempt_status": "completed", "task_status": "completed", + } + return ServerError(409, {**body, **changes}) + + +def outcome_worker(monkeypatch: pytest.MonkeyPatch, *activities: Any) -> tuple[Any, Worker, list[float]]: + from unittest.mock import AsyncMock + + client = AsyncMock(spec=Client) + client.namespace = "default" + client.external_storage = None + client._cluster_info = {} + worker = Worker(client, task_queue="q", workflows=[], activities=list(activities) or [echo], worker_id="worker-1") + delays: list[float] = [] + + async def sleep(delay: float) -> None: + delays.append(delay) + + monkeypatch.setattr("durable_workflow.worker.asyncio.sleep", sleep) + monkeypatch.setattr("durable_workflow.worker.random.uniform", lambda low, high: high) + return client, worker, delays + + +async def run_activity(worker: Worker, task: dict[str, Any]) -> str: + from durable_workflow.worker import _current_activity_claim + + claim = _current_activity_claim.set(task) + try: + return await worker._run_activity_task(task) + finally: + _current_activity_claim.reset(claim) + if worker._remote_activity_executor is not None: + worker._remote_activity_executor.shutdown(wait=True) + + +async def test_completion_is_repeated_through_transients_with_the_shared_backoff( + monkeypatch: pytest.MonkeyPatch, +) -> None: + client, worker, delays = outcome_worker(monkeypatch) + client.complete_activity_task.side_effect = [ + ServerError(503, STORAGE_UNAVAILABLE), httpx.ConnectError("refused"), ServerError(502, None), + {"outcome": "completed", "recorded": True}, + ] + assert await run_activity(worker, activity_task()) == "completed" + assert client.complete_activity_task.await_count == 4 + assert delays == [0.25, 0.5, 1.0] + client.fail_activity_task.assert_not_awaited() # a transient never becomes a failure report + + +async def test_a_resend_answered_as_our_committed_completion_is_success( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, +) -> None: + client, worker, _ = outcome_worker(monkeypatch) + client.complete_activity_task.side_effect = [ServerError(503, STORAGE_UNAVAILABLE), committed_duplicate()] + with caplog.at_level(logging.INFO, logger="durable_workflow.worker"): + assert await run_activity(worker, activity_task()) == "completed" + assert "already committed this worker's completion" in caplog.text + client.fail_activity_task.assert_not_awaited() + + +@pytest.mark.parametrize("changes", [ + {"lease_owner": "worker-2"}, + {"attempt_status": "expired", "activity_status": "pending"}, + {"task_status": "failed"}, + {"activity_attempt_id": "attempt-0"}, + {"outcome": "failed"}, +]) +async def test_any_other_stale_attempt_is_a_closed_attempt_not_success( + monkeypatch: pytest.MonkeyPatch, changes: dict[str, Any], +) -> None: + client, worker, delays = outcome_worker(monkeypatch) + client.complete_activity_task.side_effect = [ServerError(503, STORAGE_UNAVAILABLE), committed_duplicate(**changes)] + assert await run_activity(worker, activity_task()) == "complete_error" + assert client.complete_activity_task.await_count == 2, "a closed attempt is not resent again" + client.fail_activity_task.assert_not_awaited() + + +async def test_completion_retries_stop_and_log_when_the_lease_runs_out( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, +) -> None: + client, worker, delays = outcome_worker(monkeypatch) + client.complete_activity_task.side_effect = ServerError(503, STORAGE_UNAVAILABLE) + with caplog.at_level(logging.ERROR, logger="durable_workflow.worker"): + assert await run_activity(worker, activity_task(lease_seconds=0.9)) == "complete_error" + assert delays == [0.25, 0.5], "retried only while the lease covered the next wait" + assert client.complete_activity_task.await_count == 3 + assert "completed outcome not delivered after 3 transient failures" in caplog.text + assert "lease ran out" in caplog.text + client.fail_activity_task.assert_not_awaited() + + +async def test_completion_is_not_retried_on_a_protocol_refusal(monkeypatch: pytest.MonkeyPatch) -> None: + client, worker, delays = outcome_worker(monkeypatch) + client.complete_activity_task.side_effect = ServerError(422, {"reason": "invalid_result"}) + assert await run_activity(worker, activity_task()) == "complete_error" + assert client.complete_activity_task.await_count == 1 + assert delays == [] + + +async def test_failure_report_is_repeated_through_transients(monkeypatch: pytest.MonkeyPatch) -> None: + client, worker, delays = outcome_worker(monkeypatch, boom) + task = {**activity_task(), "activity_type": "transient.boom"} + client.fail_activity_task.side_effect = [ + ServerError(503, {"reason": "backend_unavailable"}), httpx.ReadTimeout("slow"), + {"outcome": "failed", "recorded": True}, + ] + assert await run_activity(worker, task) == "failed" + assert client.fail_activity_task.await_count == 3 + assert delays == [0.25, 0.5] + client.complete_activity_task.assert_not_awaited()