From 4fb7503641c0b081c8393b8cf5e6c2eb905612c1 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 20 Nov 2026 10:11:55 +0000 Subject: [PATCH] Treat cluster-info read failures at worker startup as transient GET /api/cluster/info has no side effects, so any connection failure and any 5xx or 429 is now transient for the startup discovery read (unless retryable: false); before, only the narrower transient set was, so a plain 500 stopped the worker. Startup retries use the shared backoff and now escalate like poll outages: ERROR after 60 s, then every 5 minutes. Typed refusals (401/403) stay fatal. After registration the worker uses the discovery it registered with and does not read cluster info again. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 ++ docs/reference/worker.md | 13 ++++- src/durable_workflow/worker.py | 35 ++++++++++-- tests/test_worker_transient_runtime.py | 73 ++++++++++++++++++++++++++ 4 files changed, 119 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44734c5..c0eff47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- A cluster-info read failure at worker startup no longer stops the worker. + - **Retried:** `GET /api/cluster/info` has no side effects, so `Worker.run()` and `run_until()` now treat any failed connection and any 5xx or 429 answer to it as transient, unless the Server says `retryable: false`. Before, only the narrower transient set did (for example 503 `storage_unavailable`), so a plain `500` stopped the worker. The read is retried with the shared backoff `uniform(0, min(4, 0.25 * 2**(n - 1)))` seconds until the runtime answers. + - **Logging:** startup retries now escalate like poll outages, to ERROR after 60 s and then every 5 minutes. + - **Still fatal:** typed refusals keep their behavior. A 401 or 403 is fatal at startup. - A worker no longer exits when the runtime is warming up or briefly unavailable while it registers (same issue as durable-workflow/sdk-php#448). Before, `Worker.run()` and `Worker.run_until()` raised once the client's three transport attempts had failed on `GET /api/cluster/info` or `POST /api/worker/register`. That covered native Server's `503 {"reason":"storage_unavailable"}` during warm-up, a refused connection and an upstream 502/503/504. - **Retried:** discovery and registration are now repeated with the backoff shared across SDKs, `uniform(0, min(4, 0.25 * 2**(n - 1)))` seconds, logging a warning each time. The transient failures are: refused, reset or timed-out connections; upstream 502/503/504 without a protocol body; 503 `storage_unavailable` or `backend_unavailable`; and any 5xx or 429 with `retryable: true`. - **Still fatal:** `retryable: false`, HTTP 401/403, other 4xx refusals and an incompatible Server still stop the worker. It now logs `worker stopping: registration failed with a non-retryable error: ...` at error level first. diff --git a/docs/reference/worker.md b/docs/reference/worker.md index fcda5eb..417c95d 100644 --- a/docs/reference/worker.md +++ b/docs/reference/worker.md @@ -140,9 +140,18 @@ registration are repeated whole after a transient failure: (native Server answers every storage failure with this body) - any 5xx or 429 that says `retryable: true` +Cluster discovery (`GET /api/cluster/info`) is a read with no side effects, so +any refused or failed connection, and any 5xx or 429 answer to it, counts as +transient unless the Server says `retryable: false`. + The wait before the *n*-th retry is `uniform(0, min(4, 0.25 * 2**(n - 1)))` -seconds. This is the backoff schedule shared by every Durable Workflow SDK. Each -retry logs a warning, and shutdown interrupts the wait. +seconds. This is the backoff schedule shared by every Durable Workflow SDK. +Each retry logs a warning during the first minute. After 60 seconds the outage +is logged at ERROR level, then again every 5 minutes while it lasts, the same +escalation pollers use. Shutdown interrupts the wait. + +After registration the worker uses the discovery it registered with. It does +not read cluster info again, so a later cluster-info outage cannot stop it. Some failures are never retried: diff --git a/src/durable_workflow/worker.py b/src/durable_workflow/worker.py index 4276e4c..123c392 100644 --- a/src/durable_workflow/worker.py +++ b/src/durable_workflow/worker.py @@ -322,6 +322,21 @@ def _transient_runtime_error(error: BaseException) -> bool: return error.status in _UPSTREAM_UNAVAILABLE_STATUSES and (body is None or "reason" not in body) +def _transient_discovery_error(error: BaseException) -> bool: + """Whether a failed cluster-info read may succeed once the runtime recovers. + + The read is a side-effect-free GET, so every connection failure and every + 5xx or 429 is transient unless the Server says ``retryable: false``. + Authentication, authorization and other typed refusals are not. + """ + if _transient_runtime_error(error): + return True + if not isinstance(error, ServerError) or not (error.status >= 500 or error.status == 429): + return False + body = error.body if isinstance(error.body, Mapping) else None + return body is None or body.get("retryable") is not False + + # Poll retries use the same full jitter with a 10 s cap, so a poller recovers # quickly after an outage. A long outage escalates from warnings to errors. _POLL_RETRY_MAX_SECONDS = 10.0 @@ -4900,20 +4915,32 @@ async def _register_through_transients(self) -> None: the last failure. """ failures = 0 + started = time.monotonic() + last_error_at: float | None = None while True: try: return await self._register() except Exception as error: - # Discovery failures arrive wrapped as a compatibility error. + # Discovery failures arrive wrapped as a compatibility error; + # a cluster-info read is transient on any 5xx or connection error. cause = error.__cause__ if isinstance(error, RuntimeError) else None - if not (_transient_runtime_error(error) or cause is not None and _transient_runtime_error(cause)): + if not (_transient_runtime_error(error) or cause is not None and _transient_discovery_error(cause)): raise if self._stop.is_set(): raise failures += 1 delay = _transient_retry_delay(failures) - log.warning("worker registration failed transiently: %s; retrying in %.2f s", - _poll_error_summary(cause if cause is not None else error), delay) + summary = _poll_error_summary(cause if cause is not None else error) + now = time.monotonic() + if now - started >= _POLL_OUTAGE_ERROR_AFTER_SECONDS: + # The same escalation as a poll outage: an error once the + # runtime has been unavailable for 60 s, then every 5 min. + if last_error_at is None or now - last_error_at >= _POLL_OUTAGE_ERROR_EVERY_SECONDS: + log.error("worker registration has failed for %.0f s: %s; still retrying, next in %.2f s", + now - started, summary, delay) + last_error_at = now + else: + log.warning("worker registration failed transiently: %s; retrying in %.2f s", summary, delay) with contextlib.suppress(asyncio.TimeoutError): await asyncio.wait_for(self._stop.wait(), timeout=delay) if self._stop.is_set(): diff --git a/tests/test_worker_transient_runtime.py b/tests/test_worker_transient_runtime.py index 09fac2a..a7f4a17 100644 --- a/tests/test_worker_transient_runtime.py +++ b/tests/test_worker_transient_runtime.py @@ -64,12 +64,15 @@ def __init__(self, path: str, failures: int, response: Any = None) -> None: self.response = response if response is not None else (503, STORAGE_UNAVAILABLE) self.served = 0 self.polls = 0 + self.discovery = 0 info = compatible_cluster_info() info["worker_protocol"]["server_capabilities"].pop("worker_deregistration_fencing", None) # type: ignore[index] self.info = info async def respond(self, request: httpx.Request) -> httpx.Response: path = request.url.path + if path.endswith("/cluster/info"): + self.discovery += 1 if path.endswith(self.path) and self.served < self.failures: self.served += 1 if self.response == "refused": @@ -141,6 +144,8 @@ async def test_transient_failures_then_recovery_leave_the_worker_running( ("/register", (403, {"reason": "forbidden"}), ServerError), ("/register", (503, {"reason": "storage_unavailable", "retryable": False}), ServerError), ("/cluster/info", (401, {"message": "bad token"}), RuntimeError), + ("/cluster/info", (403, {"reason": "forbidden"}), RuntimeError), + ("/cluster/info", (500, {"reason": "internal", "retryable": False}), RuntimeError), ]) async def test_fatal_failures_stop_the_worker_with_a_clear_log( path: str, response: Any, expected: type[Exception], caplog: pytest.LogCaptureFixture, @@ -316,3 +321,71 @@ async def test_failure_report_is_repeated_through_transients(monkeypatch: pytest assert client.fail_activity_task.await_count == 3 assert delays == [0.25, 0.5] client.complete_activity_task.assert_not_awaited() + + +@pytest.mark.usefixtures("fast_retries") +@pytest.mark.parametrize("response", [ + (500, {"message": "Server Error"}), + (500, {"reason": "internal"}), + (501, None), + (429, {"reason": "rate_limited"}), + "refused", +], ids=["500", "500-reason", "501", "429", "refused"]) +async def test_cluster_info_failures_at_startup_wait_then_start(response: Any) -> None: + runtime = FlakyRuntime("/cluster/info", failures=6, response=response) + client, worker = worker_for(runtime) + running = asyncio.create_task(worker.run()) + try: + for _ in range(500): + if runtime.polls >= 3: + break + await asyncio.sleep(0.01) + assert not running.done(), running.exception() if running.done() else None + assert runtime.served == 6 + assert worker._registered is True + assert runtime.polls >= 3 + # At run time the worker uses the discovery it registered with; it + # does not read cluster info again, so nothing later can fail on it. + assert runtime.discovery == 7 + finally: + await worker.stop() + running.cancel() + await asyncio.gather(running, return_exceptions=True) + await client.aclose() + + +async def test_a_long_startup_outage_escalates_to_errors( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, +) -> None: + from types import SimpleNamespace + from unittest.mock import AsyncMock + + import durable_workflow.worker as worker_module + + clock = [0.0] + real_time = worker_module.time + monkeypatch.setattr(worker_module, "time", SimpleNamespace( + monotonic=lambda: clock[0], perf_counter=real_time.perf_counter, time=real_time.time, + )) + monkeypatch.setattr(worker_module, "_transient_retry_delay", lambda failures: 0.0) + calls = 0 + + async def cluster_info() -> dict[str, Any]: + nonlocal calls + calls += 1 + clock[0] += 31.0 # each attempt takes the clock 31 s further into the outage + if calls <= 12: + raise ServerError(500, {"message": "Server Error"}) + raise RuntimeError("incompatible runtime: stop here") + + client = AsyncMock(spec=Client) + client.get_cluster_info.side_effect = cluster_info + worker = Worker(client, task_queue="q", workflows=[], activities=[noop], worker_id="worker-1") + with caplog.at_level(logging.WARNING, logger="durable_workflow.worker"), pytest.raises(RuntimeError): + await worker._register_through_transients() + levels = [record.levelno for record in caplog.records if "registration" in record.getMessage()] + # Warnings for the first minute, an error at 62 s, then every 5 minutes (372 s). + assert levels.count(logging.WARNING) == 1 + errors = [record.getMessage() for record in caplog.records if record.levelno == logging.ERROR + and "has failed for" in record.getMessage()] + assert [message.split("failed for ")[1].split(" s")[0] for message in errors] == ["62", "372"]