From d6c1084fa5f5661adf8c724d2139b6c9ff1ee54e Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 20 Nov 2026 01:40:11 +0000 Subject: [PATCH 1/4] Keep activity claims through transient lease renewal failures (#352) A transport timeout, connection reset or 5xx/429 on the renew-lease confirmation sent right after an activity poll raised ActivityLeaseAuthorityLost out of the poll, and the worker discarded the claimed task without completing, failing or releasing it. The Server then held the lease until expiry and, with the default single attempt, failed the workflow (same bug as durable-workflow/sdk-php#445). Now the activity runs under the poll's grant while confirmation is retried with backoff inside the lease. Every renewal is bounded to leave a report reserve (at most 10 s, at most a fifth of the grant). If renewal has still not succeeded when only that reserve is left, the worker stops the execution and fails the attempt retryably through the Server (ActivityLeaseAuthorityLost / activity_lease_renewal_unavailable) instead of dropping it. Typed Server refusals still end the claim. Worker-side lease handling only; no commands or history change. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 5 + src/durable_workflow/_activity_lease.py | 89 ++++++++++++++-- src/durable_workflow/worker.py | 47 ++++++++- tests/test_activity_lease_worker.py | 130 +++++++++++++++++++++++- 4 files changed, 260 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e9027b1a..2fd64fa1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Replay safety:** every worker of this release follows a marker a run already recorded, whether or not recording is on, for every workflow task, query and update. A marked run needs no per-group decision, because the first failure a task sees in append-only history is the first every later replay sees. A run whose first such group was recorded without the marker keeps the earlier rule for its whole life. Its settled groups raise the lowest-position failure, and a failure raised while siblings are open still records the #338 decision. This holds even when a task recorded no command after catching the failure, or another worker answered an update in between. A run with a cancellation request when it reaches its first group records no marker. - **Regression evidence:** fixtures recorded by published 2.19.1 on a Server built from main replay under the earlier rule. They cover a settled group in flight, a catch followed by a task that recorded no command, an update answered by 2.19.1 before or between later tasks, and a select() cancellation. Marked runs recorded on native and PHP Server replay under the new rule without the setting. New fixtures cover the first task with recording on and off. Existing fixtures, including `parallel-group-metadata-emission`, the completion-limit fixtures and `business-clock-parallel-failure-selection-gap`, are unchanged. +### Fixed + +- An activity worker no longer silently drops a claimed task when lease renewal fails transiently (#352, same bug as durable-workflow/sdk-php#445). Before, if the `renew-lease` confirmation sent right after an activity poll timed out, hit a connection reset or got a 5xx/429, the poll raised `ActivityLeaseAuthorityLost("activity_initial_lease_unavailable")` and the worker discarded the task. The Server then held the lease (300 s by default) until it expired and, with the default single attempt, failed the workflow. Now the activity runs under the lease the poll claimed, and confirmation is retried with backoff (1, 2, 4, 8 s) inside that lease, as steady-state renewal already did. Each renewal is bounded so that a reserve of the lease (at most 10 s, at most a fifth of the grant) is left. If renewal still has not succeeded when only that reserve is left, at start or later, the worker stops the execution and fails the attempt through the Server with a retryable `ActivityLeaseAuthorityLost` failure whose message names `activity_lease_renewal_unavailable`. The Server's retry policy then applies right away, not at lease expiry, and the claim is never dropped without a Server-visible outcome. A typed Server refusal (for example `lease_expired` or `run_closed`) still ends the claim as before. + - **Replay safety:** worker-side lease handling only. No workflow commands or recorded history change. + ## [2.20.0] - 2026-11-19 Both additions need **native Server 3.15.0 or later**, the first Server release that advertises `capabilities.signal_with_start` and the `signal_external_workflow` worker command. Each is discovered from the Server's capabilities and refused before anything is sent on an older Server, so the SDK's supported Server baselines are unchanged and every other feature keeps working on older Servers. Worker protocol (1.20) and control-plane (2) versions are unchanged. diff --git a/src/durable_workflow/_activity_lease.py b/src/durable_workflow/_activity_lease.py index cd74b9bb..40ae11c5 100644 --- a/src/durable_workflow/_activity_lease.py +++ b/src/durable_workflow/_activity_lease.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import logging import re import time from collections.abc import Awaitable, Callable, Mapping @@ -18,6 +19,27 @@ _T = TypeVar("_T") _UTC6 = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z\Z") +log = logging.getLogger("durable_workflow.worker") + +#: Reason of the :class:`ActivityLeaseAuthorityLost` raised when renewal kept +#: failing until only the failure-report reserve of the lease was left. The +#: worker then fails the attempt through the Server instead of dropping it. +ACTIVITY_LEASE_RENEWAL_EXHAUSTED = "activity_lease_renewal_unavailable" +# Part of the lease kept back for that failure report: at most this many +# seconds, and at most a fifth of the granted lease. +_REPORT_RESERVE_SECONDS = 10.0 +_REPORT_RESERVE_FRACTION = 0.2 +# Backoff between renewal attempts after a transient failure. +_RETRY_BASE_SECONDS = 1.0 +_RETRY_MAX_SECONDS = 8.0 +# Upper bound of one renewal exchange (the client accepts at most five). +_RENEWAL_TIMEOUT_SECONDS = 5.0 + + +def _transient(error: BaseException) -> bool: + """A failed exchange that grants nothing but does not revoke the claim.""" + return (isinstance(error, (httpx.TransportError, asyncio.TimeoutError)) + or isinstance(error, ServerError) and (error.status >= 500 or error.status == 429)) def _timestamp(value: Any) -> datetime: @@ -64,6 +86,10 @@ class ActivityLeaseAuthority: heartbeat_deadline: datetime | None last_renewal: tuple[datetime, datetime, float] | None = None lost_reason: str | None = field(default=None, init=False) + # Length of the latest confirmed grant, and consecutive transient renewal + # failures since it. Neither changes recorded history. + lease_seconds: float = field(default=0.0, init=False) + renewal_failures: int = field(default=0, init=False) @classmethod async def confirm_delivery(cls, client: Any, namespace: str, task: Mapping[str, Any], *, owner: str, @@ -75,14 +101,28 @@ async def confirm_delivery(cls, client: Any, namespace: str, task: Mapping[str, server_time=server_time, request_started=time.monotonic()) started = time.monotonic() try: + budget = provisional.renewal_budget() + if budget <= 0: + raise asyncio.TimeoutError("no time left to confirm the claim outside the report reserve") reply = await client.renew_activity_task_lease( task_id=provisional.task_id, activity_attempt_id=provisional.identity["activity_attempt_id"], - lease_owner=owner, timeout=min(5.0, provisional.remaining()), + lease_owner=owner, timeout=budget, ) provisional.acknowledge(reply, started) except ActivityLeaseAuthorityLost: raise except Exception as error: + if _transient(error): + # A slow or flaky link is not a lost claim (sdk-php#445). Keep + # the claim under the poll's grant and let maintenance retry + # the confirmation with backoff. If the grant runs down to the + # report reserve unconfirmed, the worker fails the attempt + # through the Server; it never drops the claim silently. + provisional.renewal_failures = 1 + log.warning("activity task %s initial lease confirmation failed (%s); continuing under the " + "claimed lease and retrying", provisional.task_id, repr(error)) + provisional.remaining() + return provisional reason = error.reason() if isinstance(error, ServerError) else None raise ActivityLeaseAuthorityLost( provisional.task_id, reason or "activity_initial_lease_unavailable", @@ -135,6 +175,7 @@ def delivery(cls, client: Any, namespace: str, task: Mapping[str, Any], *, owner ) except (KeyError, TypeError, ValueError, OverflowError) as error: raise ActivityLeaseAuthorityLost(task_id, "invalid_activity_claim") from error + result.lease_seconds = duration result.remaining() return result @@ -207,6 +248,20 @@ def remaining(self) -> float: raise ActivityLeaseAuthorityLost(self.task_id, self.lost_reason) return remaining + def report_reserve(self) -> float: + """Lease time kept back to fail the attempt if renewal keeps failing.""" + return min(_REPORT_RESERVE_SECONDS, self.lease_seconds * _REPORT_RESERVE_FRACTION) + + def renewal_budget(self) -> float: + """Bound one renewal so a timeout still leaves the report reserve.""" + return min(_RENEWAL_TIMEOUT_SECONDS, self.remaining(), + self.lease_until - time.monotonic() - self.report_reserve()) + + def renewal_exhausted(self) -> bool: + """Renewal has failed and only the report reserve of the lease is left.""" + return (self.renewal_failures > 0 + and self.lease_until - time.monotonic() <= self.report_reserve()) + def acknowledge(self, reply: Any, request_started: float) -> None: self.remaining() received = time.monotonic() @@ -235,6 +290,7 @@ def acknowledge(self, reply: Any, request_started: float) -> None: return raise ValueError("replayed activity renewal") self.lease_until = max(self.lease_until, request_started + duration) + self.lease_seconds = duration self._observe_progress(reply, server, request_started, received) self.server_time, self.lease_expiry, self.last_renewal = server, expiry, confirmation except (TypeError, ValueError, OverflowError) as error: @@ -260,34 +316,49 @@ def _observe_progress(self, reply: Mapping[str, Any], server: datetime, started: async def renew(self) -> bool: started = time.monotonic() try: + budget = self.renewal_budget() + if budget <= 0: + # No time for a renewal that would still leave the reserve. + self.renewal_failures += 1 + return False reply = await asyncio.wait_for(self.client.renew_activity_task_lease( task_id=self.task_id, activity_attempt_id=self.identity["activity_attempt_id"], - lease_owner=self.identity["lease_owner"], timeout=min(5.0, self.remaining()), - ), timeout=min(5.0, self.remaining())) + lease_owner=self.identity["lease_owner"], timeout=budget, + ), timeout=budget) self.acknowledge(reply, started) + self.renewal_failures = 0 return True except ActivityLeaseAuthorityLost: raise except Exception as error: - if (isinstance(error, (httpx.TransportError, asyncio.TimeoutError)) - or isinstance(error, ServerError) and (error.status >= 500 or error.status == 429)): + if _transient(error): # A failed exchange grants nothing. The existing lease may # still authorize work until its original local bound. + self.renewal_failures += 1 self.remaining() return False self.lost_reason = "activity_renewal_unavailable" raise ActivityLeaseAuthorityLost(self.task_id, self.lost_reason) from error async def _maintain(self, stopped: asyncio.Event) -> None: - confirmed = True while not stopped.is_set(): - delay = min(30.0 if confirmed else 1.0, max(0.0, self.lease_until - time.monotonic()) / 3) + lease_left = max(0.0, self.lease_until - time.monotonic()) + if self.renewal_failures: + # Back off on a struggling link, but keep retrying inside the + # grant, ahead of the reserve kept for the failure report. + backoff = min(_RETRY_MAX_SECONDS, _RETRY_BASE_SECONDS * 2 ** min(self.renewal_failures - 1, 8)) + delay = min(backoff, max(0.0, lease_left - self.report_reserve()) / 3) + else: + delay = min(30.0, lease_left / 3) if self.heartbeat_timeout is not None: delay = min(delay, self.heartbeat_timeout / 3) await asyncio.sleep(min(delay, self.remaining())) if stopped.is_set(): return - confirmed = await self.renew() + if self.renewal_exhausted(): + self.lost_reason = ACTIVITY_LEASE_RENEWAL_EXHAUSTED + return + await self.renew() async def run(self, operation: Callable[[], Awaitable[_T]]) -> _T: self.remaining() @@ -300,7 +371,7 @@ async def run(self, operation: Callable[[], Awaitable[_T]]) -> _T: return_when=asyncio.FIRST_COMPLETED) if maintenance in done: await maintenance - raise ActivityLeaseAuthorityLost(self.task_id, "activity_renewal_stopped") + raise ActivityLeaseAuthorityLost(self.task_id, self.lost_reason or "activity_renewal_stopped") if execution in done: result = await execution self.remaining() diff --git a/src/durable_workflow/worker.py b/src/durable_workflow/worker.py index 51fb0f7e..16993b18 100644 --- a/src/durable_workflow/worker.py +++ b/src/durable_workflow/worker.py @@ -42,7 +42,12 @@ import httpx from . import serializer -from ._activity_lease import activity_lease_renewal_supported, current_activity_authority +from ._activity_lease import ( + ACTIVITY_LEASE_RENEWAL_EXHAUSTED, + ActivityLeaseAuthority, + activity_lease_renewal_supported, + current_activity_authority, +) from ._activity_process import CallbackFailure, CallbackInvocation, CallbackProcessLost, SupervisedCallback from ._cancellation_scope_history import CancellationScopeBudget, CancellationScopeDeliveryReceipt from ._cooperative_cancellation import CancellationRequest, read_cancellation_history @@ -3476,9 +3481,49 @@ async def _run_activity_task(self, task: dict[str, Any]) -> str: try: return await authority.run(lambda: self._run_activity_task_body(task)) except ActivityLeaseAuthorityLost as error: + if error.reason == ACTIVITY_LEASE_RENEWAL_EXHAUSTED: + return await self._fail_unrenewable_activity_claim(task, authority, error) log.warning("activity claim abandoned: %s", error) return "claim_aborted" + async def _fail_unrenewable_activity_claim(self, task: dict[str, Any], authority: ActivityLeaseAuthority, + error: ActivityLeaseAuthorityLost) -> str: + """Fail, never silently drop, a claim whose lease could not be renewed. + + Renewal kept failing transiently until only the report reserve of the + lease was left (sdk-php#445). The execution is already stopped and + fenced; this retryable failure, sent inside the remaining lease, lets + the Server apply the retry policy now instead of at lease expiry. + """ + task_id = str(task["task_id"]) + attempt_id = task.get("activity_attempt_id") or task.get("attempt_id", "") + window = authority.lease_until - time.monotonic() + if window <= 0: + log.warning("activity claim %s lost its lease before its renewal failure could be reported: %s", + task_id, error) + return "claim_aborted" + # The authority fences its own publications once lost. This report is + # the deliberate exception, bounded by the lease that is still left. + fence = current_activity_authority.set(None) + try: + await asyncio.wait_for(self.client.fail_activity_task( + task_id=task_id, activity_attempt_id=attempt_id, lease_owner=self.worker_id, + message=f"activity lease could not be renewed before it expired ({error.reason})", + failure_type=type(error).__name__, failure_class=_exception_class_name(error), + non_retryable=False, + ), timeout=window) + except Exception as report_error: + if _run_closed_refusal(report_error): + log.info("renewal failure for activity task %s not recorded: its workflow run closed", task_id) + else: + log.warning("activity claim %s could not be renewed and its failure was not recorded (%r); " + "the Server settles it when the lease expires", task_id, report_error) + return "claim_aborted" + finally: + current_activity_authority.reset(fence) + log.warning("activity claim %s failed retryably: %s", task_id, error) + return "failed" + async def _run_activity_task_body(self, task: dict[str, Any]) -> str: self._track_worker_session_from_task(task) task_id: str = task["task_id"] diff --git a/tests/test_activity_lease_worker.py b/tests/test_activity_lease_worker.py index 42debe7b..d5ede25e 100644 --- a/tests/test_activity_lease_worker.py +++ b/tests/test_activity_lease_worker.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import json import threading import time from datetime import datetime, timedelta, timezone @@ -61,6 +62,11 @@ def __init__(self, *, lease: float = 0.18, deadline: float | None = None) -> Non self.delay_renewal = 0.0 self.fail_completion = False self.publication_times: list[float] = [] + # Transient faults applied to successive renew-lease exchanges: + # "timeout" stalls past the caller's budget, "reset" drops the + # connection, "503" answers a temporary server error. + self.renewal_faults: list[str | None] = [] + self.fail_bodies: list[dict[str, Any]] = [] async def respond(self, request: httpx.Request) -> httpx.Response: self.paths.append(request.url.path) @@ -86,6 +92,13 @@ async def respond(self, request: httpx.Request) -> httpx.Response: }) if request.url.path.endswith("/renew-lease"): self.renewals += 1 + fault = self.renewal_faults.pop(0) if self.renewal_faults else None + if fault == "timeout": + await asyncio.sleep(30) + if fault == "reset": + raise httpx.ConnectError("connection reset by peer", request=request) + if fault == "503": + return httpx.Response(503, json={"reason": "temporary_failure"}) await asyncio.sleep(self.delay_renewal if self.renewals > 1 else 0) now = datetime.now(timezone.utc) denied = (self.deny and self.renewals > 1 @@ -117,6 +130,8 @@ async def respond(self, request: httpx.Request) -> httpx.Response: "lease_expires_at": timestamp(now + timedelta(seconds=self.lease)), }) if request.url.path.endswith("/fail"): + self.publication_times.append(time.monotonic() - self.monotonic_started) + self.fail_bodies.append(json.loads(request.content)) return httpx.Response(200, json={"outcome": "failed", "recorded": True}) raise AssertionError(f"Unexpected request {request.method} {request.url.path}") @@ -205,7 +220,7 @@ async def loading() -> str: assert authority.remaining() > 0 -@pytest.mark.parametrize("loss", ["denied", "slow-renewal", "business-deadline"]) +@pytest.mark.parametrize("loss", ["denied", "business-deadline"]) async def test_lost_claim_cannot_publish_a_late_result_or_failure(loss: str) -> None: server = LeaseServer(deadline=0.13 if loss == "business-deadline" else None) server.deny = loss == "denied" @@ -321,3 +336,116 @@ async def handler() -> str: assert await execute(server, handler) == "completed" assert len(server.publications()) == 1 + + +def assert_retryable_renewal_failure(server: LeaseServer) -> None: + assert server.publications() == ["/api/worker/activity-tasks/task/17/fail"] + assert len(server.fail_bodies) == 1 + body = server.fail_bodies[0] + assert body["activity_attempt_id"] == "attempt-017" + assert body["lease_owner"] == "worker-1" + assert body["failure"]["type"] == "ActivityLeaseAuthorityLost" + assert "activity_lease_renewal_unavailable" in body["failure"]["message"] + assert "non_retryable" not in body["failure"] + # Reported inside the claimed lease, so the Server can still accept it. + assert all(elapsed < server.lease for elapsed in server.publication_times) + + +async def test_slow_renewal_fails_the_claim_retryably_instead_of_dropping_it() -> None: + server = LeaseServer() + server.delay_renewal = 0.3 + cancelled = False + + @activity.defn(name="maintained") + async def handler() -> str: + nonlocal cancelled + try: + await asyncio.sleep(0.45) + except asyncio.CancelledError: + cancelled = True + await asyncio.sleep(0.02) + return "late result" + + assert await execute(server, handler) == "failed" + assert cancelled is True + assert_retryable_renewal_failure(server) + + +@pytest.mark.parametrize("faults", [ + ["timeout"], ["reset"], ["503"], ["timeout", "reset", "503", "timeout"], +]) +async def test_transient_initial_confirmation_failure_keeps_the_claim( + monkeypatch: pytest.MonkeyPatch, faults: list[str], +) -> None: + # sdk-php#445: one stalled renewal at activity start must not drop the + # claim. The activity runs under the claimed lease while confirmation is + # retried, and its completion is accepted. + monkeypatch.setattr("durable_workflow._activity_lease._RENEWAL_TIMEOUT_SECONDS", 0.05) + monkeypatch.setattr("durable_workflow._activity_lease._RETRY_BASE_SECONDS", 0.02) + server = LeaseServer(lease=1.0) + server.renewal_faults = list(faults) + started = False + + @activity.defn(name="maintained") + async def handler() -> str: + nonlocal started + started = True + await asyncio.sleep(0.6) + return "completed despite a flaky start" + + assert await execute(server, handler) == "completed" + assert started is True + assert server.renewals > len(faults), "confirmation must be retried until it succeeds" + assert server.publications() == ["/api/worker/activity-tasks/task/17/complete"] + + +async def test_flaky_renewal_mid_execution_keeps_the_claim_beyond_the_original_lease( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr("durable_workflow._activity_lease._RENEWAL_TIMEOUT_SECONDS", 0.05) + monkeypatch.setattr("durable_workflow._activity_lease._RETRY_BASE_SECONDS", 0.02) + server = LeaseServer(lease=0.6) + server.renewal_faults = [None, "503", "timeout", "reset"] + + @activity.defn(name="maintained") + async def handler() -> str: + await asyncio.sleep(1.0) + return "outlived the original lease" + + assert await execute(server, handler) == "completed" + assert server.renewals > 4 + assert server.publications() == ["/api/worker/activity-tasks/task/17/complete"] + + +@pytest.mark.parametrize("fault", ["timeout", "reset", "503"]) +async def test_renewal_that_never_recovers_fails_the_claim_through_the_server( + monkeypatch: pytest.MonkeyPatch, fault: str, +) -> None: + monkeypatch.setattr("durable_workflow._activity_lease._RENEWAL_TIMEOUT_SECONDS", 0.05) + monkeypatch.setattr("durable_workflow._activity_lease._RETRY_BASE_SECONDS", 0.02) + server = LeaseServer(lease=0.5) + server.renewal_faults = [fault] * 100 + + @activity.defn(name="maintained") + async def handler() -> str: + await asyncio.sleep(2.0) + return "must not be published" + + assert await execute(server, handler) == "failed" + assert server.renewals >= 2, "renewal must be retried before the claim is given up" + assert_retryable_renewal_failure(server) + + +async def test_initial_confirmation_refused_by_the_server_still_raises() -> None: + # A typed refusal is the Server's own decision about the claim, not a + # transient failure, so the claim is not kept. + server = LeaseServer(lease=0.06) + server.delay_poll = 0.1 + + @activity.defn(name="maintained") + async def handler() -> str: + return "must not run" + + with pytest.raises(Exception, match="lease_expired"): + await execute(server, handler) + assert server.publications() == [] From 7179930637c3b30b3eaa2100847f5b26d7717f55 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 20 Nov 2026 02:57:49 +0000 Subject: [PATCH 2/4] Release an unmaintainable claim as worker_lost; add lease_loss_reason (#352) Align the released-claim failure with the cross-SDK wire contract shared with PHP and Rust: failure.type "worker_lost", failure.class "ActivityWorkerLost", non_retryable false, the local cause (activity_lease_renewal_unavailable) in the message, and details {"reason": "worker_lost", "cause": }. Rename ActivityLeaseAuthorityLost to ActivityWorkerLost, keeping the old name as a deprecated alias of the same class so existing handlers keep working. Add ActivityFailed.lease_loss_reason(): "worker_lost" for a released claim, the recorded failure_reason ("lease_expired" or "worker_deregistered", default "lease_expired") for a Server-settled ActivityLeaseExpiredException, otherwise None. ActivityFailed now keeps the recorded failure_reason. The default retry policy is unchanged. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 9 +++- README.md | 3 +- docs/reference/activity.md | 2 +- docs/sdk-reference.md | 2 +- src/durable_workflow/_activity_lease.py | 24 ++++----- src/durable_workflow/activity.py | 6 +-- src/durable_workflow/client.py | 2 +- src/durable_workflow/errors.py | 41 ++++++++++++++- src/durable_workflow/worker.py | 19 ++++--- src/durable_workflow/workflow.py | 1 + tests/test_activity_checkpoint.py | 4 +- tests/test_activity_heartbeat_authority.py | 8 +-- tests/test_activity_lease_authority.py | 10 ++-- tests/test_activity_lease_worker.py | 7 ++- tests/test_activity_run_closed_refusal.py | 4 +- tests/test_activity_worker_lost.py | 59 ++++++++++++++++++++++ tests/test_threaded_heartbeat_example.py | 4 +- 17 files changed, 159 insertions(+), 46 deletions(-) create mode 100644 tests/test_activity_worker_lost.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fd64fa1..1e96dd2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,10 +12,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Default off; enable it only after every worker on the task queue runs this release or later.** Published 2.20.0 and earlier cannot replay a marked run. The Server marks the workflow task replay-blocked with no retry, and the run waits until it is repaired, which can hand it to an old worker again. With recording off, new runs send exactly the commands 2.20.0 sends. Turning recording off again is safe. Rolling back below this release requires marked runs to be drained first. The next minor release turns recording on by default (#351). - **Replay safety:** every worker of this release follows a marker a run already recorded, whether or not recording is on, for every workflow task, query and update. A marked run needs no per-group decision, because the first failure a task sees in append-only history is the first every later replay sees. A run whose first such group was recorded without the marker keeps the earlier rule for its whole life. Its settled groups raise the lowest-position failure, and a failure raised while siblings are open still records the #338 decision. This holds even when a task recorded no command after catching the failure, or another worker answered an update in between. A run with a cancellation request when it reaches its first group records no marker. - **Regression evidence:** fixtures recorded by published 2.19.1 on a Server built from main replay under the earlier rule. They cover a settled group in flight, a catch followed by a task that recorded no command, an update answered by 2.19.1 before or between later tasks, and a select() cancellation. Marked runs recorded on native and PHP Server replay under the new rule without the setting. New fixtures cover the first task with recording on and off. Existing fixtures, including `parallel-group-metadata-emission`, the completion-limit fixtures and `business-clock-parallel-failure-selection-gap`, are unchanged. +- `ActivityFailed.lease_loss_reason()` tells workflow code whether an activity attempt failed because its worker lost the claim. It returns `"worker_lost"` when a worker released a claim it could not maintain (failure type `worker_lost`, see Fixed). For a lease the Server settled (`Workflow\V2\Exceptions\ActivityLeaseExpiredException`), it returns the recorded `failure_reason`, `"lease_expired"` or `"worker_deregistered"`, defaulting to `"lease_expired"`. It returns `None` for any other failure. `ActivityFailed.failure_reason` now carries that recorded field. The default activity retry policy is unchanged. + +### Changed + +- `ActivityLeaseAuthorityLost` is renamed `ActivityWorkerLost`. The old name stays importable from `durable_workflow.errors` as a deprecated alias of the same class, so existing `except ActivityLeaseAuthorityLost` handlers keep working. `durable_workflow.errors.WORKER_LOST_FAILURE_TYPE` (`"worker_lost"`) and `ACTIVITY_LEASE_EXPIRED_EXCEPTION_CLASS` name the two recorded forms of a lost claim. ### Fixed -- An activity worker no longer silently drops a claimed task when lease renewal fails transiently (#352, same bug as durable-workflow/sdk-php#445). Before, if the `renew-lease` confirmation sent right after an activity poll timed out, hit a connection reset or got a 5xx/429, the poll raised `ActivityLeaseAuthorityLost("activity_initial_lease_unavailable")` and the worker discarded the task. The Server then held the lease (300 s by default) until it expired and, with the default single attempt, failed the workflow. Now the activity runs under the lease the poll claimed, and confirmation is retried with backoff (1, 2, 4, 8 s) inside that lease, as steady-state renewal already did. Each renewal is bounded so that a reserve of the lease (at most 10 s, at most a fifth of the grant) is left. If renewal still has not succeeded when only that reserve is left, at start or later, the worker stops the execution and fails the attempt through the Server with a retryable `ActivityLeaseAuthorityLost` failure whose message names `activity_lease_renewal_unavailable`. The Server's retry policy then applies right away, not at lease expiry, and the claim is never dropped without a Server-visible outcome. A typed Server refusal (for example `lease_expired` or `run_closed`) still ends the claim as before. +- An activity worker no longer silently drops a claimed task when lease renewal fails transiently (#352, same bug as durable-workflow/sdk-php#445). Before, if the `renew-lease` confirmation sent right after an activity poll timed out, hit a connection reset or got a 5xx/429, the poll raised `ActivityLeaseAuthorityLost("activity_initial_lease_unavailable")` and the worker discarded the task. The Server then held the lease (300 s by default) until it expired and, with the default single attempt, failed the workflow. + - **Keeps working:** the activity now runs under the lease the poll claimed, and confirmation is retried with backoff (1, 2, 4, 8 s) inside that lease, as steady-state renewal already did. Each renewal is bounded so that a reserve of the lease (at most 10 s, at most a fifth of the grant) is left. + - **Releases through the Server:** if renewal still has not succeeded when only that reserve is left, at start or later, the worker stops the execution and releases the claim through the Server, following the cross-SDK contract shared with PHP and Rust. It sends `POST /worker/activity-tasks/{id}/fail` with `failure.type = "worker_lost"` (recorded as `exception_type`, and matchable in `non_retryable_error_types`), `failure.class = "ActivityWorkerLost"`, `non_retryable = false`, the local cause `activity_lease_renewal_unavailable` in the message, and `details = {"reason": "worker_lost", "cause": "activity_lease_renewal_unavailable"}`. The Server's retry policy then applies right away, not at lease expiry, and the claim is never dropped without a Server-visible outcome. A typed Server refusal (for example `lease_expired` or `run_closed`) still ends the claim as before. - **Replay safety:** worker-side lease handling only. No workflow commands or recorded history change. ## [2.20.0] - 2026-11-19 diff --git a/README.md b/README.md index 9c35bcd7..e86ebb90 100644 --- a/README.md +++ b/README.md @@ -160,7 +160,8 @@ after cancellation. The disposal guarantees require SDK 2.12.2 or later. Both `await activity.context().heartbeat()` and synchronous `activity.context().heartbeat_sync()` stop at an explicit Server refusal. -They raise `ActivityLeaseAuthorityLost` with its original `.reason` for an +They raise `ActivityWorkerLost` (formerly `ActivityLeaseAuthorityLost`, still +importable under that name) with its original `.reason` for an elapsed deadline, closed attempt or closed run. Cancellation continues to raise `ActivityCancelled`. The worker submits no result or failure for an attempt whose heartbeat refused continuation, even if application code catches that diff --git a/docs/reference/activity.md b/docs/reference/activity.md index c27220bd..56bc4667 100644 --- a/docs/reference/activity.md +++ b/docs/reference/activity.md @@ -111,7 +111,7 @@ The helper records the original exception, sets the cancellation event, and stops heartbeating when cancellation, lease-authority loss or a transport error occurs. After the work unwinds, it joins the helper and re-raises that exception before a result can be returned. In particular, do not swallow `ActivityCancelled` -or `ActivityLeaseAuthorityLost`, replace them with an application error, or +or `ActivityWorkerLost`, replace them with an application error, or return a successful result after either. `ActivityCancelled` inherits from `BaseException`, so `except Exception` alone does not catch it. Ordinary return and work exceptions also stop and join the helper. diff --git a/docs/sdk-reference.md b/docs/sdk-reference.md index aeb80b2e..8f281695 100644 --- a/docs/sdk-reference.md +++ b/docs/sdk-reference.md @@ -1516,7 +1516,7 @@ storage-admission handling remain separate. See [Worker recovery](reference/work Use `await activity.context().heartbeat(details)` in an async activity or `activity.context().heartbeat_sync(details)` in a synchronous activity thread. Both raise `ActivityCancelled` when cancellation is requested. An explicit -`can_continue=false` instead raises `ActivityLeaseAuthorityLost` with the +`can_continue=false` instead raises `ActivityWorkerLost` with the original `.reason`, such as `activity_deadline_elapsed`, `attempt_closed` or `run_closed`. This is lost execution authority, not a new activity failure. The worker releases the slot after the callback exits and submits neither a diff --git a/src/durable_workflow/_activity_lease.py b/src/durable_workflow/_activity_lease.py index 40ae11c5..0bfe9053 100644 --- a/src/durable_workflow/_activity_lease.py +++ b/src/durable_workflow/_activity_lease.py @@ -15,13 +15,13 @@ import httpx -from .errors import ActivityLeaseAuthorityLost, ServerError +from .errors import ActivityWorkerLost, ServerError _T = TypeVar("_T") _UTC6 = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z\Z") log = logging.getLogger("durable_workflow.worker") -#: Reason of the :class:`ActivityLeaseAuthorityLost` raised when renewal kept +#: Reason of the :class:`ActivityWorkerLost` raised when renewal kept #: failing until only the failure-report reserve of the lease was left. The #: worker then fails the attempt through the Server instead of dropping it. ACTIVITY_LEASE_RENEWAL_EXHAUSTED = "activity_lease_renewal_unavailable" @@ -109,7 +109,7 @@ async def confirm_delivery(cls, client: Any, namespace: str, task: Mapping[str, lease_owner=owner, timeout=budget, ) provisional.acknowledge(reply, started) - except ActivityLeaseAuthorityLost: + except ActivityWorkerLost: raise except Exception as error: if _transient(error): @@ -124,7 +124,7 @@ async def confirm_delivery(cls, client: Any, namespace: str, task: Mapping[str, provisional.remaining() return provisional reason = error.reason() if isinstance(error, ServerError) else None - raise ActivityLeaseAuthorityLost( + raise ActivityWorkerLost( provisional.task_id, reason or "activity_initial_lease_unavailable", ) from error observed = dict(task) @@ -174,7 +174,7 @@ def delivery(cls, client: Any, namespace: str, task: Mapping[str, Any], *, owner heartbeat, ) except (KeyError, TypeError, ValueError, OverflowError) as error: - raise ActivityLeaseAuthorityLost(task_id, "invalid_activity_claim") from error + raise ActivityWorkerLost(task_id, "invalid_activity_claim") from error result.lease_seconds = duration result.remaining() return result @@ -235,17 +235,17 @@ def acknowledge_progress(self, reply: Any, request_started: float) -> None: self.lease_expiry = max(self.lease_expiry, expiry) except (TypeError, ValueError, OverflowError) as error: self.lost_reason = "invalid_activity_heartbeat" - raise ActivityLeaseAuthorityLost(self.task_id, self.lost_reason) from error + raise ActivityWorkerLost(self.task_id, self.lost_reason) from error def remaining(self) -> float: if self.lost_reason is not None: - raise ActivityLeaseAuthorityLost(self.task_id, self.lost_reason) + raise ActivityWorkerLost(self.task_id, self.lost_reason) bounds = [self.lease_until, *(value for value in (self.business_until, self.heartbeat_until) if value is not None)] remaining = min(bounds) - time.monotonic() if remaining <= 0: self.lost_reason = "activity_execution_budget_expired" - raise ActivityLeaseAuthorityLost(self.task_id, self.lost_reason) + raise ActivityWorkerLost(self.task_id, self.lost_reason) return remaining def report_reserve(self) -> float: @@ -295,7 +295,7 @@ def acknowledge(self, reply: Any, request_started: float) -> None: self.server_time, self.lease_expiry, self.last_renewal = server, expiry, confirmation except (TypeError, ValueError, OverflowError) as error: self.lost_reason = "invalid_activity_renewal" - raise ActivityLeaseAuthorityLost(self.task_id, self.lost_reason) from error + raise ActivityWorkerLost(self.task_id, self.lost_reason) from error def _observe_progress(self, reply: Mapping[str, Any], server: datetime, started: float, received: float) -> None: if self.heartbeat_until is None or self.heartbeat_timeout is None: @@ -328,7 +328,7 @@ async def renew(self) -> bool: self.acknowledge(reply, started) self.renewal_failures = 0 return True - except ActivityLeaseAuthorityLost: + except ActivityWorkerLost: raise except Exception as error: if _transient(error): @@ -338,7 +338,7 @@ async def renew(self) -> bool: self.remaining() return False self.lost_reason = "activity_renewal_unavailable" - raise ActivityLeaseAuthorityLost(self.task_id, self.lost_reason) from error + raise ActivityWorkerLost(self.task_id, self.lost_reason) from error async def _maintain(self, stopped: asyncio.Event) -> None: while not stopped.is_set(): @@ -371,7 +371,7 @@ async def run(self, operation: Callable[[], Awaitable[_T]]) -> _T: return_when=asyncio.FIRST_COMPLETED) if maintenance in done: await maintenance - raise ActivityLeaseAuthorityLost(self.task_id, self.lost_reason or "activity_renewal_stopped") + raise ActivityWorkerLost(self.task_id, self.lost_reason or "activity_renewal_stopped") if execution in done: result = await execution self.remaining() diff --git a/src/durable_workflow/activity.py b/src/durable_workflow/activity.py index 4719cd79..2aec47c2 100644 --- a/src/durable_workflow/activity.py +++ b/src/durable_workflow/activity.py @@ -10,7 +10,7 @@ from ._activity_checkpoint import ActivityHeartbeatCheckpoint, _HeartbeatResume, claim_resume from ._activity_lease import current_activity_authority -from .errors import ActivityCancelled, ActivityLeaseAuthorityLost +from .errors import ActivityCancelled, ActivityWorkerLost if TYPE_CHECKING: from .client import Client @@ -190,7 +190,7 @@ async def heartbeat(self, details: dict[str, Any] | None = None) -> None: owning workflow has requested cancellation, so the activity can exit cleanly at its next natural break point. - Raises :class:`~durable_workflow.errors.ActivityLeaseAuthorityLost` + Raises :class:`~durable_workflow.errors.ActivityWorkerLost` with the original Server reason when the attempt cannot continue, including an elapsed deadline, closed attempt or closed run. """ @@ -225,7 +225,7 @@ def _raise_if_cannot_continue(self) -> None: and authority.identity["attempt_number"] == self._info.attempt_number and authority.identity["lease_owner"] == self._info.worker_id): authority.lost_reason = authority.lost_reason or self._cannot_continue_reason - raise ActivityLeaseAuthorityLost(self._info.task_id, self._cannot_continue_reason) + raise ActivityWorkerLost(self._info.task_id, self._cannot_continue_reason) def context() -> ActivityContext: diff --git a/src/durable_workflow/client.py b/src/durable_workflow/client.py index 5652bc53..8de3bba4 100644 --- a/src/durable_workflow/client.py +++ b/src/durable_workflow/client.py @@ -7226,7 +7226,7 @@ async def heartbeat_activity_task( Worker-plane endpoint. Most code calls :meth:`~durable_workflow.ActivityContext.heartbeat` instead, which additionally raises :class:`~durable_workflow.errors.ActivityCancelled` - for cancellation or :class:`~durable_workflow.errors.ActivityLeaseAuthorityLost` + for cancellation or :class:`~durable_workflow.errors.ActivityWorkerLost` with the original reason when execution cannot continue. """ body: dict[str, Any] = { diff --git a/src/durable_workflow/errors.py b/src/durable_workflow/errors.py index 308859c8..81a651e9 100644 --- a/src/durable_workflow/errors.py +++ b/src/durable_workflow/errors.py @@ -64,8 +64,16 @@ def __init__(self, query_task_id: str, reason: str) -> None: self.reason = reason -class ActivityLeaseAuthorityLost(DurableWorkflowError): - """An activity delivery lost its original live execution authority.""" +class ActivityWorkerLost(DurableWorkflowError): + """An activity delivery lost its original live execution authority. + + ``reason`` is the local cause, for example ``attempt_closed`` from a + heartbeat refusal or ``activity_lease_renewal_unavailable`` when renewal + kept failing. When the worker releases such a claim through the Server it + reports the cross-SDK failure type :data:`WORKER_LOST_FAILURE_TYPE` + (``"worker_lost"``) with class ``ActivityWorkerLost`` and this cause in + the message and details. + """ def __init__(self, task_id: str, reason: str) -> None: super().__init__(f"activity task {task_id!r} cannot continue: {reason}") @@ -73,6 +81,18 @@ def __init__(self, task_id: str, reason: str) -> None: self.reason = reason +#: Deprecated alias of :class:`ActivityWorkerLost`, kept so existing +#: ``except ActivityLeaseAuthorityLost`` handlers keep working. +ActivityLeaseAuthorityLost = ActivityWorkerLost + +#: Failure type of an activity attempt whose worker released a claim it could +#: not maintain. Recorded as ``exception_type``; it can be listed in an +#: activity retry policy's ``non_retryable_error_types``. +WORKER_LOST_FAILURE_TYPE = "worker_lost" +#: Exception class the Server records when it settles a lost activity lease. +ACTIVITY_LEASE_EXPIRED_EXCEPTION_CLASS = "Workflow\\V2\\Exceptions\\ActivityLeaseExpiredException" + + class InvalidWorkflowDefinition(DurableWorkflowError, ValueError): """Workflow authoring uses an unsupported entry point or result.""" @@ -1028,8 +1048,10 @@ def __init__( code: Any | None = None, exception_payload: dict[str, Any] | None = None, activity: dict[str, Any] | None = None, + failure_reason: str | None = None, ) -> None: super().__init__(message) + self.failure_reason = failure_reason self.activity_type = activity_type self.step_sequence = step_sequence self.activity_execution_id = activity_execution_id @@ -1043,6 +1065,21 @@ def __init__( self.exception_payload = exception_payload self.activity = activity + def lease_loss_reason(self) -> str | None: + """Why this attempt failed because its worker lost the claim, if it did. + + ``"worker_lost"`` when a worker released a claim it could not maintain + (failure type ``worker_lost``). For a lease the Server settled + (``ActivityLeaseExpiredException``), the recorded ``failure_reason``: + ``"lease_expired"`` or ``"worker_deregistered"``, defaulting to + ``"lease_expired"``. ``None`` for any other failure. + """ + if self.exception_type == WORKER_LOST_FAILURE_TYPE: + return WORKER_LOST_FAILURE_TYPE + if self.exception_class == ACTIVITY_LEASE_EXPIRED_EXCEPTION_CLASS: + return self.failure_reason or "lease_expired" + return None + class WorkflowTerminated(DurableWorkflowError): """A workflow was terminated by operator action. diff --git a/src/durable_workflow/worker.py b/src/durable_workflow/worker.py index 16993b18..04f22cf6 100644 --- a/src/durable_workflow/worker.py +++ b/src/durable_workflow/worker.py @@ -79,9 +79,10 @@ _worker_poll_http_timeout, ) from .errors import ( + WORKER_LOST_FAILURE_TYPE, ActivityCancelled, ActivityFailed, - ActivityLeaseAuthorityLost, + ActivityWorkerLost, AvroNotInstalledError, DuplicateRegistrationError, DurableWorkflowError, @@ -184,7 +185,7 @@ def _run_closed_refusal(error: BaseException) -> bool: """Whether a claim or outcome request was refused because its run closed.""" - if isinstance(error, ActivityLeaseAuthorityLost): + if isinstance(error, ActivityWorkerLost): return error.reason == _RUN_CLOSED_REFUSAL return isinstance(error, ServerError) and error.reason() == _RUN_CLOSED_REFUSAL @@ -3480,14 +3481,14 @@ async def _run_activity_task(self, task: dict[str, Any]) -> str: return "claim_aborted" try: return await authority.run(lambda: self._run_activity_task_body(task)) - except ActivityLeaseAuthorityLost as error: + except ActivityWorkerLost as error: if error.reason == ACTIVITY_LEASE_RENEWAL_EXHAUSTED: return await self._fail_unrenewable_activity_claim(task, authority, error) log.warning("activity claim abandoned: %s", error) return "claim_aborted" async def _fail_unrenewable_activity_claim(self, task: dict[str, Any], authority: ActivityLeaseAuthority, - error: ActivityLeaseAuthorityLost) -> str: + error: ActivityWorkerLost) -> str: """Fail, never silently drop, a claim whose lease could not be renewed. Renewal kept failing transiently until only the report reserve of the @@ -3506,11 +3507,13 @@ async def _fail_unrenewable_activity_claim(self, task: dict[str, Any], authority # the deliberate exception, bounded by the lease that is still left. fence = current_activity_authority.set(None) try: + # The cross-SDK released-claim contract: type worker_lost, the + # idiomatic class name, retryable, local cause in message/details. await asyncio.wait_for(self.client.fail_activity_task( task_id=task_id, activity_attempt_id=attempt_id, lease_owner=self.worker_id, - message=f"activity lease could not be renewed before it expired ({error.reason})", - failure_type=type(error).__name__, failure_class=_exception_class_name(error), - non_retryable=False, + message=f"activity worker released the claim: lease could not be renewed ({error.reason})", + failure_type=WORKER_LOST_FAILURE_TYPE, failure_class=ActivityWorkerLost.__name__, + non_retryable=False, details={"reason": WORKER_LOST_FAILURE_TYPE, "cause": error.reason}, ), timeout=window) except Exception as report_error: if _run_closed_refusal(report_error): @@ -3563,7 +3566,7 @@ async def _run_activity_task_body(self, task: dict[str, Any]) -> str: # A caught heartbeat refusal cannot restore outcome authority, # including when user code subsequently raises another error. act_ctx._raise_if_cannot_continue() - except ActivityLeaseAuthorityLost as error: + except ActivityWorkerLost as error: log.warning("activity %s cannot continue after heartbeat: %s", task_id, error.reason) return "claim_aborted" except _RemoteActivityExecutionAborted as error: diff --git a/src/durable_workflow/workflow.py b/src/durable_workflow/workflow.py index c6739af5..e4d803ca 100644 --- a/src/durable_workflow/workflow.py +++ b/src/durable_workflow/workflow.py @@ -5185,6 +5185,7 @@ def _activity_failed_from_payload(payload: Mapping[str, Any]) -> ActivityFailed: code=payload.get("code"), exception_payload=exposed_exception, activity=activity, + failure_reason=_optional_str(payload.get("failure_reason")), ) diff --git a/tests/test_activity_checkpoint.py b/tests/test_activity_checkpoint.py index 9ed8c056..77a32362 100644 --- a/tests/test_activity_checkpoint.py +++ b/tests/test_activity_checkpoint.py @@ -171,8 +171,8 @@ async def test_heartbeat_outcomes_never_advance_the_received_checkpoint(outcome: if outcome == "success": await ctx.heartbeat({"cursor": 2}) else: - from durable_workflow.errors import ActivityLeaseAuthorityLost - with pytest.raises(ActivityLeaseAuthorityLost): + from durable_workflow.errors import ActivityWorkerLost + with pytest.raises(ActivityWorkerLost): await ctx.heartbeat({"cursor": 2}) assert ctx.last_heartbeat_details() == {"cursor": 1} assert ctx.last_heartbeat_at() == AT diff --git a/tests/test_activity_heartbeat_authority.py b/tests/test_activity_heartbeat_authority.py index 3f540af8..9311415e 100644 --- a/tests/test_activity_heartbeat_authority.py +++ b/tests/test_activity_heartbeat_authority.py @@ -13,7 +13,7 @@ from durable_workflow._activity_lease import ActivityLeaseAuthority, current_activity_authority from durable_workflow.activity import ActivityContext, ActivityInfo from durable_workflow.client import Client -from durable_workflow.errors import ActivityLeaseAuthorityLost, NonRetryableError +from durable_workflow.errors import ActivityWorkerLost, NonRetryableError from durable_workflow.worker import Worker REASONS = ("activity_deadline_elapsed", "attempt_closed", "run_closed") @@ -38,7 +38,7 @@ async def test_heartbeat_refusal_carries_original_reason(reason: str, synchronou invocation = (asyncio.to_thread(ctx.heartbeat_sync, {"progress": 1}) if synchronous else ctx.heartbeat({"progress": 1})) - with pytest.raises(ActivityLeaseAuthorityLost) as refusal: + with pytest.raises(ActivityWorkerLost) as refusal: await invocation assert refusal.value.task_id == "original-task" @@ -115,10 +115,10 @@ def timestamp(value: datetime) -> str: task_id="original-task", activity_type="heartbeat-refusal", activity_attempt_id="original-attempt", attempt_number=1, task_queue="original-queue", worker_id="original-worker", ), client=client) - with pytest.raises(ActivityLeaseAuthorityLost, match="attempt_closed"): + with pytest.raises(ActivityWorkerLost, match="attempt_closed"): await ctx.heartbeat() if same_original: - with pytest.raises(ActivityLeaseAuthorityLost, match="attempt_closed"): + with pytest.raises(ActivityWorkerLost, match="attempt_closed"): await client.complete_activity_task( task_id="original-task", activity_attempt_id="original-attempt", lease_owner="original-worker", result="late result", diff --git a/tests/test_activity_lease_authority.py b/tests/test_activity_lease_authority.py index 43b0416d..f9436e23 100644 --- a/tests/test_activity_lease_authority.py +++ b/tests/test_activity_lease_authority.py @@ -7,7 +7,7 @@ import pytest from durable_workflow._activity_lease import ActivityLeaseAuthority -from durable_workflow.errors import ActivityLeaseAuthorityLost, ServerError +from durable_workflow.errors import ActivityWorkerLost, ServerError ORIGINAL = datetime(2026, 10, 21, tzinfo=timezone.utc) @@ -58,7 +58,7 @@ def reply(**changes: Any) -> dict[str, Any]: ]) def test_invalid_original_delivery_never_authorizes_execution(clock: list[float], field: str, value: Any) -> None: original = task(**{field: value}) - with pytest.raises(ActivityLeaseAuthorityLost): + with pytest.raises(ActivityWorkerLost): delivery(original) @@ -78,7 +78,7 @@ def test_wrong_or_unbounded_renewal_cannot_extend_the_grant( original = authority.lease_until clock[0] = 101.1 renewal = reply(**{field: value}) - with pytest.raises(ActivityLeaseAuthorityLost): + with pytest.raises(ActivityWorkerLost): authority.acknowledge(renewal, 101) assert authority.lease_until == original @@ -97,7 +97,7 @@ def test_acknowledgement_arriving_after_expiry_cannot_revive_the_claim(clock: li authority = delivery(task(deadlines={})) clock[0] = 105.1 renewal = reply(server_time=timestamp(4), lease_expires_at=timestamp(20)) - with pytest.raises(ActivityLeaseAuthorityLost, match="budget_expired"): + with pytest.raises(ActivityWorkerLost, match="budget_expired"): authority.acknowledge(renewal, 104) assert authority.lease_until == 105 @@ -110,7 +110,7 @@ def test_replaying_a_successful_reply_at_a_later_request_cannot_reset_latency(cl assert authority.lease_until == 111 clock[0] = 101.2 replayed = reply() - with pytest.raises(ActivityLeaseAuthorityLost, match="invalid_activity_renewal"): + with pytest.raises(ActivityWorkerLost, match="invalid_activity_renewal"): authority.acknowledge(replayed, 101.15) assert authority.lease_until == 111 diff --git a/tests/test_activity_lease_worker.py b/tests/test_activity_lease_worker.py index d5ede25e..f2bb6545 100644 --- a/tests/test_activity_lease_worker.py +++ b/tests/test_activity_lease_worker.py @@ -344,9 +344,14 @@ def assert_retryable_renewal_failure(server: LeaseServer) -> None: body = server.fail_bodies[0] assert body["activity_attempt_id"] == "attempt-017" assert body["lease_owner"] == "worker-1" - assert body["failure"]["type"] == "ActivityLeaseAuthorityLost" + # Cross-SDK released-claim contract (PHP, Python, Rust). + assert body["failure"]["type"] == "worker_lost" + assert body["failure"]["class"] == "ActivityWorkerLost" assert "activity_lease_renewal_unavailable" in body["failure"]["message"] assert "non_retryable" not in body["failure"] + assert serializer.decode_envelope(body["failure"]["details"]) == { + "reason": "worker_lost", "cause": "activity_lease_renewal_unavailable", + } # Reported inside the claimed lease, so the Server can still accept it. assert all(elapsed < server.lease for elapsed in server.publication_times) diff --git a/tests/test_activity_run_closed_refusal.py b/tests/test_activity_run_closed_refusal.py index e4f611fe..a6adc1e9 100644 --- a/tests/test_activity_run_closed_refusal.py +++ b/tests/test_activity_run_closed_refusal.py @@ -20,7 +20,7 @@ from durable_workflow import activity, serializer from durable_workflow.activity import ActivityContext, ActivityInfo from durable_workflow.client import Client -from durable_workflow.errors import ActivityCancelled, ActivityLeaseAuthorityLost, ServerError +from durable_workflow.errors import ActivityCancelled, ActivityWorkerLost, ServerError from durable_workflow.worker import Worker RUN_CLOSED_OUTCOME = { @@ -201,7 +201,7 @@ async def handler() -> str: async def poll(**_: object) -> None: polls.append("poll") if len(polls) == 1: - raise ActivityLeaseAuthorityLost("late-task", "run_closed") + raise ActivityWorkerLost("late-task", "run_closed") await worker.stop() return None diff --git a/tests/test_activity_worker_lost.py b/tests/test_activity_worker_lost.py new file mode 100644 index 00000000..408f85bd --- /dev/null +++ b/tests/test_activity_worker_lost.py @@ -0,0 +1,59 @@ +"""ActivityWorkerLost naming and the workflow-side lease-loss accessor.""" +from __future__ import annotations + +from typing import Any + +import pytest + +from durable_workflow import errors +from durable_workflow.errors import ActivityFailed, ActivityWorkerLost +from durable_workflow.workflow import _activity_failed_from_payload + +LEASE_EXPIRED = "Workflow\\V2\\Exceptions\\ActivityLeaseExpiredException" + + +def test_deprecated_name_is_the_same_class() -> None: + assert errors.ActivityLeaseAuthorityLost is ActivityWorkerLost + with pytest.raises(errors.ActivityLeaseAuthorityLost): + raise ActivityWorkerLost("task-1", "attempt_closed") + assert errors.WORKER_LOST_FAILURE_TYPE == "worker_lost" + assert errors.ACTIVITY_LEASE_EXPIRED_EXCEPTION_CLASS == LEASE_EXPIRED + + +@pytest.mark.parametrize(("fields", "expected"), [ + ({"exception_type": "worker_lost", "exception_class": "ActivityWorkerLost"}, "worker_lost"), + ({"exception_type": "worker_lost", "exception_class": "WorkerLost\\Php"}, "worker_lost"), + ({"exception_class": LEASE_EXPIRED, "failure_reason": "lease_expired"}, "lease_expired"), + ({"exception_class": LEASE_EXPIRED, "failure_reason": "worker_deregistered"}, "worker_deregistered"), + ({"exception_class": LEASE_EXPIRED}, "lease_expired"), + ({"exception_type": "ValueError", "exception_class": "builtins.ValueError"}, None), + ({"exception_type": "ActivityWorkerLost"}, None), + ({"failure_reason": "lease_expired"}, None), + ({}, None), +]) +def test_lease_loss_reason(fields: dict[str, Any], expected: str | None) -> None: + assert ActivityFailed("activity failed", **fields).lease_loss_reason() == expected + + +def test_lease_loss_reason_from_recorded_server_lease_expiry() -> None: + # The ActivityFailed payload native Server records when it settles a lost + # lease (activity_deadlines.rs): lease metadata merged at the top level. + failure = _activity_failed_from_payload({ + "activity_type": "charge", "sequence": 1, "message": "Activity charge lost its original task lease.", + "exception_class": LEASE_EXPIRED, "failure_category": "internal", "timeout_kind": "lease", + "failure_reason": "worker_deregistered", "retry_reason": "worker_deregistered", + "lease_owner": "worker-1", "lease_expires_at": "2026-11-20T00:00:00.000000Z", + }) + assert failure.failure_reason == "worker_deregistered" + assert failure.lease_loss_reason() == "worker_deregistered" + + +def test_lease_loss_reason_from_recorded_worker_lost_failure() -> None: + failure = _activity_failed_from_payload({ + "activity_type": "charge", "sequence": 1, + "exception": {"type": "worker_lost", "class": "ActivityWorkerLost", + "message": "activity worker released the claim: lease could not be renewed " + "(activity_lease_renewal_unavailable)"}, + }) + assert failure.exception_type == "worker_lost" + assert failure.lease_loss_reason() == "worker_lost" diff --git a/tests/test_threaded_heartbeat_example.py b/tests/test_threaded_heartbeat_example.py index eb2d4207..f45d527c 100644 --- a/tests/test_threaded_heartbeat_example.py +++ b/tests/test_threaded_heartbeat_example.py @@ -11,7 +11,7 @@ from durable_workflow import activity from durable_workflow.activity import ActivityContext, ActivityInfo, _set_context -from durable_workflow.errors import ActivityCancelled, ActivityLeaseAuthorityLost +from durable_workflow.errors import ActivityCancelled, ActivityWorkerLost SPEC = importlib.util.spec_from_file_location( "threaded_heartbeats_example", @@ -108,7 +108,7 @@ async def test_cancellation_stops_work_and_preserves_type(self) -> None: async def test_authority_loss_keeps_original_identity_and_reason(self) -> None: self.client.response = {"can_continue": False, "reason": "activity_attempt_replaced"} - with self.assertRaises(ActivityLeaseAuthorityLost) as caught: + with self.assertRaises(ActivityWorkerLost) as caught: await self.run_work(self.cancelled_work) self.assertEqual(caught.exception.task_id, "task-original") self.assertEqual(caught.exception.reason, "activity_attempt_replaced") From 0e6563ce1730063d8258e41e214e90dc68912e73 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 20 Nov 2026 04:07:41 +0000 Subject: [PATCH 3/4] Release worker_lost only after the execution is confirmed stopped (#352) Review follow-up, shared rule for all SDKs: a claim is released as worker_lost only after its execution has confirmed stopped inside the lease; otherwise nothing is sent and the Server settles lease_expired, so two executions of one attempt never overlap. - Thread futures and callback processes of a claim are tracked on its lease authority. On renewal exhaustion the execution is stopped (async cancelled, thread asked to stop via its next heartbeat_sync, callback stopped) and awaited for up to half of the release reserve. Only then is the release sent, within the remaining lease. - An unconfirmed grant is charged from the poll's start, the conservative bound. If it is already spent, the unstarted claim is released directly, and run() never starts an exhausted claim. - Renewal retries use the cross-SDK backoff: uniform(0, min(4, 0.25 * 2**n)) seconds (full jitter). Documented in the SDK reference. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 +- docs/sdk-reference.md | 47 ++++++++ src/durable_workflow/_activity_lease.py | 133 ++++++++++++++++++---- src/durable_workflow/client.py | 1 + src/durable_workflow/worker.py | 57 +++++----- tests/test_activity_lease_authority.py | 48 ++++++++ tests/test_activity_lease_worker.py | 142 ++++++++++++++++++++++-- 7 files changed, 371 insertions(+), 63 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e96dd2e..096ecee5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,8 +21,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - An activity worker no longer silently drops a claimed task when lease renewal fails transiently (#352, same bug as durable-workflow/sdk-php#445). Before, if the `renew-lease` confirmation sent right after an activity poll timed out, hit a connection reset or got a 5xx/429, the poll raised `ActivityLeaseAuthorityLost("activity_initial_lease_unavailable")` and the worker discarded the task. The Server then held the lease (300 s by default) until it expired and, with the default single attempt, failed the workflow. - - **Keeps working:** the activity now runs under the lease the poll claimed, and confirmation is retried with backoff (1, 2, 4, 8 s) inside that lease, as steady-state renewal already did. Each renewal is bounded so that a reserve of the lease (at most 10 s, at most a fifth of the grant) is left. - - **Releases through the Server:** if renewal still has not succeeded when only that reserve is left, at start or later, the worker stops the execution and releases the claim through the Server, following the cross-SDK contract shared with PHP and Rust. It sends `POST /worker/activity-tasks/{id}/fail` with `failure.type = "worker_lost"` (recorded as `exception_type`, and matchable in `non_retryable_error_types`), `failure.class = "ActivityWorkerLost"`, `non_retryable = false`, the local cause `activity_lease_renewal_unavailable` in the message, and `details = {"reason": "worker_lost", "cause": "activity_lease_renewal_unavailable"}`. The Server's retry policy then applies right away, not at lease expiry, and the claim is never dropped without a Server-visible outcome. A typed Server refusal (for example `lease_expired` or `run_closed`) still ends the claim as before. + - **Keeps working:** the activity now runs under the lease it already holds, and renewal is retried inside that lease. The backoff is the same in every SDK: the *n*-th retry waits `uniform(0, min(4, 0.25 * 2**n))` seconds, so 0.25 s doubling to a 4 s cap, with full jitter. Each renewal exchange is bounded to leave a reserve of the lease (at most 10 s, at most a fifth of the grant). An unconfirmed lease is charged from the poll's start, because the Server stamps it only after receiving the poll. + - **Stops before it releases:** if renewal still has not succeeded when only the reserve is left, the worker stops the execution. An async activity is cancelled. A thread activity is asked to stop: its next `heartbeat_sync()` raises `ActivityWorkerLost`. A callback process is stopped. The worker waits for that stop for up to half of the reserve. + - **Releases through the Server:** only once every execution of the claim is confirmed stopped does the worker release the claim, following the cross-SDK contract shared with PHP and Rust. It sends `POST /worker/activity-tasks/{id}/fail` with `failure.type = "worker_lost"` (recorded as `exception_type`, and matchable in `non_retryable_error_types`), `failure.class = "ActivityWorkerLost"`, `non_retryable = false`, the local cause `activity_lease_renewal_unavailable` in the message, and `details = {"reason": "worker_lost", "cause": "activity_lease_renewal_unavailable"}`. The retry policy then applies right away. + - **Waits for expiry if it cannot stop:** if the execution does not stop in time, for example a thread that never heartbeats, nothing is sent. The Server settles the attempt at lease expiry (`lease_expired`), so two executions of one attempt never overlap. A claim that never started is released directly. A typed Server refusal (for example `lease_expired` or `run_closed`) still ends the claim as before. See "Activity lease renewal" in the SDK reference. - **Replay safety:** worker-side lease handling only. No workflow commands or recorded history change. ## [2.20.0] - 2026-11-19 diff --git a/docs/sdk-reference.md b/docs/sdk-reference.md index 8f281695..b6511359 100644 --- a/docs/sdk-reference.md +++ b/docs/sdk-reference.md @@ -1528,6 +1528,53 @@ supervision. The SDK cannot forcibly stop a running native thread or undo its external effects. Keep side effects idempotent and heartbeat at useful progress boundaries. +## Activity lease renewal + +When the Server advertises `activity_lease_renewal`, the worker keeps each +claimed activity's lease alive in the background. It confirms the lease right +after the poll, then renews it about every third of the lease, at most every +30 seconds. Each renewal exchange is limited to 5 seconds, and part of the +lease is always held in reserve: at most 10 seconds, and at most a fifth of +the grant. + +A timeout, connection reset, 5xx or 429 on a renewal does not end the claim. +The activity keeps running under the lease it already holds. Renewal is retried +with the same backoff in every Durable Workflow SDK: + +- The *n*-th retry waits `uniform(0, min(4, 0.25 * 2**n))` seconds: 0.25 s, + doubling to a 4 s cap. +- Full random jitter keeps workers from retrying in step after an outage. +- Retries always stay inside the lease, ahead of the reserve. + +If the claim was never confirmed, its lease is charged from the start of the +poll, because the Server stamps the lease only after it receives the poll. + +If renewal has still not succeeded when only the reserve is left, the worker +stops the activity: + +- An async activity is cancelled. +- A thread activity is asked to stop: its next `heartbeat_sync()` raises + `ActivityWorkerLost`. +- A supervised callback process is stopped. + +The worker waits for that stop for up to half of the reserve. Once every +execution of the claim has stopped, it releases the claim with +`POST /worker/activity-tasks/{id}/fail`. The failure has type `worker_lost` +(recorded as `exception_type`, and matchable in `non_retryable_error_types`), +class `ActivityWorkerLost`, `non_retryable = false`, the local cause in the +message, and `details = {"reason": "worker_lost", "cause": ...}`. The +activity's retry policy then applies right away. + +If the execution does not stop in time, for example a thread that never +heartbeats, the worker sends nothing. The Server settles the attempt when its +lease expires (`lease_expired`), so two executions of one attempt never +overlap. A claim that never started running is released directly. + +In workflow code, `ActivityFailed.lease_loss_reason()` tells these outcomes +apart. It returns `"worker_lost"` for a released claim, `"lease_expired"` or +`"worker_deregistered"` when the Server settled the lease, and `None` for any +other failure. + ### Acknowledged heartbeat checkpoints SDK 2.18.0 adds `activity.context().last_heartbeat_checkpoint()`, diff --git a/src/durable_workflow/_activity_lease.py b/src/durable_workflow/_activity_lease.py index 0bfe9053..dbc8a5a8 100644 --- a/src/durable_workflow/_activity_lease.py +++ b/src/durable_workflow/_activity_lease.py @@ -4,6 +4,7 @@ import asyncio import logging +import random import re import time from collections.abc import Awaitable, Callable, Mapping @@ -15,27 +16,70 @@ import httpx -from .errors import ActivityWorkerLost, ServerError +from .errors import WORKER_LOST_FAILURE_TYPE, ActivityWorkerLost, ServerError _T = TypeVar("_T") _UTC6 = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z\Z") log = logging.getLogger("durable_workflow.worker") #: Reason of the :class:`ActivityWorkerLost` raised when renewal kept -#: failing until only the failure-report reserve of the lease was left. The -#: worker then fails the attempt through the Server instead of dropping it. +#: failing until only the release reserve of the lease was left. The worker +#: then stops the execution and, once it has confirmed the stop inside the +#: lease, releases the claim through the Server as ``worker_lost``. ACTIVITY_LEASE_RENEWAL_EXHAUSTED = "activity_lease_renewal_unavailable" -# Part of the lease kept back for that failure report: at most this many -# seconds, and at most a fifth of the granted lease. +# Part of the lease kept back to stop the execution and release the claim: at +# most this many seconds, and at most a fifth of the granted lease. The first +# half is for confirming the stop, the second for the release request. _REPORT_RESERVE_SECONDS = 10.0 _REPORT_RESERVE_FRACTION = 0.2 -# Backoff between renewal attempts after a transient failure. -_RETRY_BASE_SECONDS = 1.0 -_RETRY_MAX_SECONDS = 8.0 +# Renewal retry backoff after a transient failure, the same in every SDK: +# delay = uniform(0, min(MAX, BASE * 2**n)) for the n-th retry (full jitter). +_RETRY_BASE_SECONDS = 0.25 +_RETRY_MAX_SECONDS = 4.0 +# How often a stop confirmation re-checks threads and callback processes. +_STOP_POLL_SECONDS = 0.02 # Upper bound of one renewal exchange (the client accepts at most five). _RENEWAL_TIMEOUT_SECONDS = 5.0 +def renewal_retry_delay(failures: int) -> float: + """Full-jitter backoff before the retry after ``failures`` consecutive failures.""" + exponent = min(max(failures - 1, 0), 16) + return random.uniform(0.0, min(_RETRY_MAX_SECONDS, _RETRY_BASE_SECONDS * 2 ** exponent)) + + +async def release_worker_lost(client: Any, *, task_id: str, activity_attempt_id: str, lease_owner: str, + cause: str, timeout: float) -> bool: + """Release a claim the worker cannot maintain, by the cross-SDK contract. + + Call only after the execution is confirmed stopped (or never started): + the Server may hand the attempt to another worker as soon as it records + this retryable ``worker_lost`` failure. + """ + try: + await asyncio.wait_for(client.fail_activity_task( + task_id=task_id, activity_attempt_id=activity_attempt_id, lease_owner=lease_owner, + message=f"activity worker released the claim: lease could not be renewed ({cause})", + failure_type=WORKER_LOST_FAILURE_TYPE, failure_class=ActivityWorkerLost.__name__, + non_retryable=False, details={"reason": WORKER_LOST_FAILURE_TYPE, "cause": cause}, + ), timeout=timeout) + return True + except Exception as error: + if isinstance(error, ServerError) and error.reason() == "run_closed": + log.info("worker_lost release for activity task %s not recorded: its workflow run closed", task_id) + else: + log.warning("activity claim %s could not be released (%r); the Server settles it when the lease " + "expires", task_id, error) + return False + + +def _execution_stopped(execution: Any) -> bool: + done = getattr(execution, "done", None) + if callable(done): + return bool(done()) + return getattr(execution, "stopped", False) is True + + def _transient(error: BaseException) -> bool: """A failed exchange that grants nothing but does not revoke the claim.""" return (isinstance(error, (httpx.TransportError, asyncio.TimeoutError)) @@ -90,10 +134,15 @@ class ActivityLeaseAuthority: # failures since it. Neither changes recorded history. lease_seconds: float = field(default=0.0, init=False) renewal_failures: int = field(default=0, init=False) + # Physical executions (thread futures, callback processes) started for + # this claim. A release waits until every one has actually stopped. + executions: list[Any] = field(default_factory=list, init=False) + execution_started: bool = field(default=False, init=False) @classmethod async def confirm_delivery(cls, client: Any, namespace: str, task: Mapping[str, Any], *, owner: str, - task_queue: str, server_time: Any) -> ActivityLeaseAuthority: + task_queue: str, server_time: Any, + poll_started: float | None = None) -> ActivityLeaseAuthority: # A long poll may wait before a task or its lease exists. Establish the # first execution budget with a separate, original-claim confirmation; # its entire round trip is charged and it cannot revive an expired lease. @@ -115,14 +164,11 @@ async def confirm_delivery(cls, client: Any, namespace: str, task: Mapping[str, if _transient(error): # A slow or flaky link is not a lost claim (sdk-php#445). Keep # the claim under the poll's grant and let maintenance retry - # the confirmation with backoff. If the grant runs down to the - # report reserve unconfirmed, the worker fails the attempt - # through the Server; it never drops the claim silently. - provisional.renewal_failures = 1 - log.warning("activity task %s initial lease confirmation failed (%s); continuing under the " - "claimed lease and retrying", provisional.task_id, repr(error)) - provisional.remaining() - return provisional + # the confirmation with backoff. + return await cls._unconfirmed_delivery( + client, namespace, task, owner=owner, task_queue=task_queue, server_time=server_time, + poll_started=started if poll_started is None else poll_started, error=error, + ) reason = error.reason() if isinstance(error, ServerError) else None raise ActivityWorkerLost( provisional.task_id, reason or "activity_initial_lease_unavailable", @@ -136,6 +182,33 @@ async def confirm_delivery(cls, client: Any, namespace: str, task: Mapping[str, return cls.delivery(client, namespace, observed, owner=owner, task_queue=task_queue, server_time=reply["server_time"], request_started=started) + @classmethod + async def _unconfirmed_delivery(cls, client: Any, namespace: str, task: Mapping[str, Any], *, owner: str, + task_queue: str, server_time: Any, poll_started: float, + error: BaseException) -> ActivityLeaseAuthority: + # The Server stamps the lease after it receives the poll, so charging + # the grant from the poll's start is the conservative bound. + try: + grant = cls.delivery(client, namespace, task, owner=owner, task_queue=task_queue, + server_time=server_time, request_started=poll_started) + except ActivityWorkerLost: + grant = None + if grant is None or grant.lease_until - time.monotonic() <= grant.report_reserve(): + # Nothing has run, so releasing cannot cause a second concurrent + # execution; a stale release is refused by the Server. + task_id = str(task.get("task_id") or "") + log.warning("activity task %s initial lease confirmation failed (%r) and the conservative grant is " + "spent; releasing the unstarted claim", task_id, error) + await release_worker_lost( + client, task_id=task_id, activity_attempt_id=str(task.get("activity_attempt_id") or ""), + lease_owner=owner, cause=ACTIVITY_LEASE_RENEWAL_EXHAUSTED, timeout=_RENEWAL_TIMEOUT_SECONDS, + ) + raise ActivityWorkerLost(task_id, ACTIVITY_LEASE_RENEWAL_EXHAUSTED) from error + grant.renewal_failures = 1 + log.warning("activity task %s initial lease confirmation failed (%r); continuing under the claimed " + "lease and retrying", grant.task_id, error) + return grant + @classmethod def delivery(cls, client: Any, namespace: str, task: Mapping[str, Any], *, owner: str, task_queue: str, server_time: Any, request_started: float) -> ActivityLeaseAuthority: @@ -257,6 +330,19 @@ def renewal_budget(self) -> float: return min(_RENEWAL_TIMEOUT_SECONDS, self.remaining(), self.lease_until - time.monotonic() - self.report_reserve()) + def track(self, execution: Any) -> None: + """Register a thread future or callback process that runs this claim.""" + self.executions.append(execution) + + async def confirm_stopped(self, deadline: float) -> bool: + """Wait until every tracked execution has stopped, or ``deadline`` passes.""" + while not all(_execution_stopped(execution) for execution in self.executions): + left = deadline - time.monotonic() + if left <= 0: + return False + await asyncio.sleep(min(_STOP_POLL_SECONDS, left)) + return time.monotonic() < deadline + def renewal_exhausted(self) -> bool: """Renewal has failed and only the report reserve of the lease is left.""" return (self.renewal_failures > 0 @@ -344,10 +430,10 @@ async def _maintain(self, stopped: asyncio.Event) -> None: while not stopped.is_set(): lease_left = max(0.0, self.lease_until - time.monotonic()) if self.renewal_failures: - # Back off on a struggling link, but keep retrying inside the - # grant, ahead of the reserve kept for the failure report. - backoff = min(_RETRY_MAX_SECONDS, _RETRY_BASE_SECONDS * 2 ** min(self.renewal_failures - 1, 8)) - delay = min(backoff, max(0.0, lease_left - self.report_reserve()) / 3) + # Back off with full jitter on a struggling link, but keep + # retrying inside the grant, ahead of the release reserve. + delay = min(renewal_retry_delay(self.renewal_failures), + max(0.0, lease_left - self.report_reserve()) / 3) else: delay = min(30.0, lease_left / 3) if self.heartbeat_timeout is not None: @@ -362,7 +448,12 @@ async def _maintain(self, stopped: asyncio.Event) -> None: async def run(self, operation: Callable[[], Awaitable[_T]]) -> _T: self.remaining() + if self.renewal_exhausted(): + # Never start work that could not be confirmed stopped in time. + self.lost_reason = ACTIVITY_LEASE_RENEWAL_EXHAUSTED + raise ActivityWorkerLost(self.task_id, self.lost_reason) stopped = asyncio.Event() + self.execution_started = True execution = asyncio.ensure_future(operation()) maintenance = asyncio.create_task(self._maintain(stopped)) try: diff --git a/src/durable_workflow/client.py b/src/durable_workflow/client.py index 8de3bba4..b7083103 100644 --- a/src/durable_workflow/client.py +++ b/src/durable_workflow/client.py @@ -2316,6 +2316,7 @@ async def _do_request() -> httpx.Response: activity_authority = await ActivityLeaseAuthority.confirm_delivery( self, self.namespace, result["task"], owner=request_json["worker_id"], task_queue=request_json["task_queue"], server_time=result.get("server_time"), + poll_started=request_started, ) current_activity_authority.set(activity_authority) query_authority = None diff --git a/src/durable_workflow/worker.py b/src/durable_workflow/worker.py index 04f22cf6..ff37c89b 100644 --- a/src/durable_workflow/worker.py +++ b/src/durable_workflow/worker.py @@ -47,6 +47,7 @@ ActivityLeaseAuthority, activity_lease_renewal_supported, current_activity_authority, + release_worker_lost, ) from ._activity_process import CallbackFailure, CallbackInvocation, CallbackProcessLost, SupervisedCallback from ._cancellation_scope_history import CancellationScopeBudget, CancellationScopeDeliveryReceipt @@ -79,7 +80,6 @@ _worker_poll_http_timeout, ) from .errors import ( - WORKER_LOST_FAILURE_TYPE, ActivityCancelled, ActivityFailed, ActivityWorkerLost, @@ -3428,6 +3428,9 @@ async def observe_ownership() -> None: observability_supported=self._observability_claim(task, "activity"), remote=True, )) self._remote_activity_processes.add(callback) + claim = current_activity_authority.get() + if claim is not None: + claim.track(callback) invocation: asyncio.Task[Any] | None = None abandoned = False observation = asyncio.create_task(observe_ownership()) @@ -3489,42 +3492,37 @@ async def _run_activity_task(self, task: dict[str, Any]) -> str: async def _fail_unrenewable_activity_claim(self, task: dict[str, Any], authority: ActivityLeaseAuthority, error: ActivityWorkerLost) -> str: - """Fail, never silently drop, a claim whose lease could not be renewed. - - Renewal kept failing transiently until only the report reserve of the - lease was left (sdk-php#445). The execution is already stopped and - fenced; this retryable failure, sent inside the remaining lease, lets - the Server apply the retry policy now instead of at lease expiry. + """Release, never silently drop, a claim whose lease could not be renewed. + + Renewal kept failing transiently until only the release reserve of the + lease was left (sdk-php#445). The release is sent only after every + execution of the claim is confirmed stopped inside the lease. A Python + thread cannot be killed: it is asked to stop (its next heartbeat raises + ``ActivityWorkerLost``) and awaited until the first half of the reserve + is used. If it is still running, nothing is sent and the Server settles + the claim at lease expiry, so two executions never overlap. """ task_id = str(task["task_id"]) attempt_id = task.get("activity_attempt_id") or task.get("attempt_id", "") - window = authority.lease_until - time.monotonic() - if window <= 0: - log.warning("activity claim %s lost its lease before its renewal failure could be reported: %s", - task_id, error) + stop_deadline = authority.lease_until - authority.report_reserve() / 2 + if not await authority.confirm_stopped(stop_deadline): + log.warning("activity claim %s not released: its execution did not stop before the lease deadline; " + "the Server settles it when the lease expires (%s)", task_id, error.reason) return "claim_aborted" - # The authority fences its own publications once lost. This report is + window = authority.lease_until - time.monotonic() + # The authority fences its own publications once lost. This release is # the deliberate exception, bounded by the lease that is still left. fence = current_activity_authority.set(None) try: - # The cross-SDK released-claim contract: type worker_lost, the - # idiomatic class name, retryable, local cause in message/details. - await asyncio.wait_for(self.client.fail_activity_task( - task_id=task_id, activity_attempt_id=attempt_id, lease_owner=self.worker_id, - message=f"activity worker released the claim: lease could not be renewed ({error.reason})", - failure_type=WORKER_LOST_FAILURE_TYPE, failure_class=ActivityWorkerLost.__name__, - non_retryable=False, details={"reason": WORKER_LOST_FAILURE_TYPE, "cause": error.reason}, - ), timeout=window) - except Exception as report_error: - if _run_closed_refusal(report_error): - log.info("renewal failure for activity task %s not recorded: its workflow run closed", task_id) - else: - log.warning("activity claim %s could not be renewed and its failure was not recorded (%r); " - "the Server settles it when the lease expires", task_id, report_error) - return "claim_aborted" + released = await release_worker_lost( + self.client, task_id=task_id, activity_attempt_id=attempt_id, lease_owner=self.worker_id, + cause=error.reason, timeout=window, + ) finally: current_activity_authority.reset(fence) - log.warning("activity claim %s failed retryably: %s", task_id, error) + if not released: + return "claim_aborted" + log.warning("activity claim %s released as worker_lost: %s", task_id, error.reason) return "failed" async def _run_activity_task_body(self, task: dict[str, Any]) -> str: @@ -3746,6 +3744,9 @@ async def call_activity(ctx: ActivityInterceptorContext) -> Any: self._remote_activity_thread_slots.release() raise self._remote_activity_threads.add(future) + claim = current_activity_authority.get() + if claim is not None: + claim.track(future) def thread_finished(completed: Future[Any]) -> None: def release_capacity() -> None: diff --git a/tests/test_activity_lease_authority.py b/tests/test_activity_lease_authority.py index f9436e23..95901fac 100644 --- a/tests/test_activity_lease_authority.py +++ b/tests/test_activity_lease_authority.py @@ -174,3 +174,51 @@ async def test_temporary_renewal_failure_grants_nothing_but_does_not_revoke_a_li assert await authority.renew() is False assert authority.lease_until == original assert authority.remaining() > 0 + + +@pytest.mark.parametrize(("failures", "ceiling"), [(1, 0.25), (2, 0.5), (3, 1.0), (4, 2.0), (5, 4.0), (9, 4.0)]) +def test_renewal_retry_backoff_uses_full_jitter_up_to_four_seconds( + monkeypatch: pytest.MonkeyPatch, failures: int, ceiling: float, +) -> None: + from durable_workflow import _activity_lease + + calls: list[tuple[float, float]] = [] + monkeypatch.setattr(_activity_lease.random, "uniform", lambda low, high: calls.append((low, high)) or high) + assert _activity_lease.renewal_retry_delay(failures) == ceiling + assert calls == [(0.0, ceiling)] + + +def test_renewal_retry_backoff_spreads_retries() -> None: + from durable_workflow._activity_lease import renewal_retry_delay + + delays = [renewal_retry_delay(5) for _ in range(200)] + assert all(0.0 <= delay <= 4.0 for delay in delays) + assert len({round(delay, 3) for delay in delays}) > 50 + + +class _Callback: + def __init__(self, stopped: bool) -> None: + self.stopped = stopped + + +async def test_stop_confirmation_requires_every_execution_to_have_stopped() -> None: + import time + from concurrent.futures import Future + + authority = ActivityLeaseAuthority.delivery( + object(), "17", task(lease_expires_at=timestamp(60), deadlines={}), owner="worker-1", + task_queue="queue-1", server_time=timestamp(0), request_started=time.monotonic(), + ) + assert await authority.confirm_stopped(time.monotonic() + 0.05) is True + + running: Future[Any] = Future() + callback = _Callback(stopped=False) + authority.track(running) + authority.track(callback) + assert await authority.confirm_stopped(time.monotonic() + 0.05) is False + + running.set_result("done") + assert await authority.confirm_stopped(time.monotonic() + 0.05) is False, "callback process still running" + callback.stopped = True + assert await authority.confirm_stopped(time.monotonic() + 0.05) is True + assert await authority.confirm_stopped(time.monotonic() - 1) is False, "a stop after the deadline is too late" diff --git a/tests/test_activity_lease_worker.py b/tests/test_activity_lease_worker.py index f2bb6545..0884f2a9 100644 --- a/tests/test_activity_lease_worker.py +++ b/tests/test_activity_lease_worker.py @@ -67,6 +67,9 @@ def __init__(self, *, lease: float = 0.18, deadline: float | None = None) -> Non # connection, "503" answers a temporary server error. self.renewal_faults: list[str | None] = [] self.fail_bodies: list[dict[str, Any]] = [] + # When False, an application heartbeat is acknowledged without + # extending the lease, so it cannot stand in for failed renewals. + self.heartbeat_extends = True async def respond(self, request: httpx.Request) -> httpx.Response: self.paths.append(request.url.path) @@ -127,7 +130,8 @@ async def respond(self, request: httpx.Request) -> httpx.Response: "lease_owner": self.task["lease_owner"], "heartbeat_recorded": True, "can_continue": True, "cancel_requested": False, "reason": None, "last_heartbeat_at": timestamp(now), - "lease_expires_at": timestamp(now + timedelta(seconds=self.lease)), + "lease_expires_at": (timestamp(now + timedelta(seconds=self.lease)) if self.heartbeat_extends + else self.task["lease_expires_at"]), }) if request.url.path.endswith("/fail"): self.publication_times.append(time.monotonic() - self.monotonic_started) @@ -356,19 +360,24 @@ def assert_retryable_renewal_failure(server: LeaseServer) -> None: assert all(elapsed < server.lease for elapsed in server.publication_times) -async def test_slow_renewal_fails_the_claim_retryably_instead_of_dropping_it() -> None: - server = LeaseServer() - server.delay_renewal = 0.3 +def fast_retries(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("durable_workflow._activity_lease._RENEWAL_TIMEOUT_SECONDS", 0.05) + monkeypatch.setattr("durable_workflow._activity_lease._RETRY_BASE_SECONDS", 0.02) + + +async def test_slow_renewal_releases_a_cooperatively_stopped_claim() -> None: + server = LeaseServer(lease=0.5) + server.delay_renewal = 0.6 cancelled = False @activity.defn(name="maintained") async def handler() -> str: nonlocal cancelled try: - await asyncio.sleep(0.45) + await asyncio.sleep(2.0) except asyncio.CancelledError: cancelled = True - await asyncio.sleep(0.02) + raise return "late result" assert await execute(server, handler) == "failed" @@ -376,6 +385,118 @@ async def handler() -> str: assert_retryable_renewal_failure(server) +async def test_coroutine_that_outlives_the_stop_window_is_not_released() -> None: + # A coroutine may swallow cancellation. Until it has actually finished the + # claim is not released; past the deadline the lease is left to expire. + server = LeaseServer(lease=0.5) + server.delay_renewal = 0.6 + + @activity.defn(name="maintained") + async def handler() -> str: + try: + await asyncio.sleep(2.0) + except asyncio.CancelledError: + await asyncio.sleep(0.3) + return "late result" + + assert await execute(server, handler) == "claim_aborted" + assert server.publications() == [] + + +async def test_thread_that_stops_at_its_next_heartbeat_is_released(monkeypatch: pytest.MonkeyPatch) -> None: + fast_retries(monkeypatch) + server = LeaseServer(lease=0.6) + server.renewal_faults = [None] + ["503"] * 100 + server.heartbeat_extends = False + stopped_at: list[float] = [] + + @activity.defn(name="maintained") + def handler() -> str: + try: + while True: + activity.context().heartbeat_sync({"step": 1}) + time.sleep(0.01) + finally: + stopped_at.append(time.monotonic() - server.monotonic_started) + + assert await execute(server, handler) == "failed" + assert_retryable_renewal_failure(server) + assert len(stopped_at) == 1 + assert stopped_at[0] <= server.publication_times[0], "release only after the thread stopped" + + +async def test_thread_that_will_not_stop_is_left_to_lease_expiry(monkeypatch: pytest.MonkeyPatch) -> None: + fast_retries(monkeypatch) + server = LeaseServer(lease=0.6) + server.renewal_faults = [None] + ["503"] * 100 + started, release = threading.Event(), threading.Event() + + @activity.defn(name="maintained") + def handler() -> str: + started.set() + release.wait(timeout=5) + return "late thread result" + + async with Client("http://lease.invalid", namespace="17", worker_token="synthetic-worker-token") as client: + await client._http.aclose() + client._http = httpx.AsyncClient(base_url="http://lease.invalid", transport=httpx.MockTransport(server.respond)) + client._cluster_info = {"worker_protocol": {"server_capabilities": {"activity_lease_renewal": True}}} + worker = Worker(client, worker_id="worker-1", task_queue="queue-1", workflows=[], activities=[handler]) + try: + task = await client.poll_activity_task(worker_id="worker-1", task_queue="queue-1") + assert await worker._run_activity_task(task) == "claim_aborted" + assert started.is_set() + # Still running: a release now would let a second attempt overlap. + assert len(worker._remote_activity_threads) == 1 + assert server.publications() == [] + finally: + release.set() + if worker._remote_activity_executor is not None: + worker._remote_activity_executor.shutdown(wait=True) + assert server.publications() == [] + + +async def test_unconfirmed_grant_is_measured_from_the_poll_start(monkeypatch: pytest.MonkeyPatch) -> None: + fast_retries(monkeypatch) + server = LeaseServer(lease=1.0) + server.delay_poll = 0.2 + server.renewal_faults = ["timeout"] + async with Client("http://lease.invalid", namespace="17", worker_token="synthetic-worker-token") as client: + await client._http.aclose() + client._http = httpx.AsyncClient(base_url="http://lease.invalid", transport=httpx.MockTransport(server.respond)) + client._cluster_info = {"worker_protocol": {"server_capabilities": {"activity_lease_renewal": True}}} + poll_started = time.monotonic() + await client.poll_activity_task(worker_id="worker-1", task_queue="queue-1") + authority = current_activity_authority.get() + assert authority is not None and authority.renewal_failures == 1 + # Charged from the poll's start, not from the response's arrival 0.2 s later. + assert poll_started + 0.9 < authority.lease_until < poll_started + 1.1 + current_activity_authority.set(None) + + +async def test_unstarted_claim_with_a_spent_conservative_grant_is_released( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # The long poll waited longer than the lease, then confirmation failed. + # Nothing ran, so the claim is released rather than dropped. + fast_retries(monkeypatch) + server = LeaseServer(lease=0.25) + server.delay_before_claim = 0.3 + server.renewal_faults = ["timeout"] + called = False + + @activity.defn(name="maintained") + async def handler() -> str: + nonlocal called + called = True + return "must not run" + + with pytest.raises(Exception, match="activity_lease_renewal_unavailable"): + await execute(server, handler) + assert called is False + assert_retryable_renewal_failure(server) + + @pytest.mark.parametrize("faults", [ ["timeout"], ["reset"], ["503"], ["timeout", "reset", "503", "timeout"], ]) @@ -385,8 +506,7 @@ async def test_transient_initial_confirmation_failure_keeps_the_claim( # sdk-php#445: one stalled renewal at activity start must not drop the # claim. The activity runs under the claimed lease while confirmation is # retried, and its completion is accepted. - monkeypatch.setattr("durable_workflow._activity_lease._RENEWAL_TIMEOUT_SECONDS", 0.05) - monkeypatch.setattr("durable_workflow._activity_lease._RETRY_BASE_SECONDS", 0.02) + fast_retries(monkeypatch) server = LeaseServer(lease=1.0) server.renewal_faults = list(faults) started = False @@ -407,8 +527,7 @@ async def handler() -> str: async def test_flaky_renewal_mid_execution_keeps_the_claim_beyond_the_original_lease( monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setattr("durable_workflow._activity_lease._RENEWAL_TIMEOUT_SECONDS", 0.05) - monkeypatch.setattr("durable_workflow._activity_lease._RETRY_BASE_SECONDS", 0.02) + fast_retries(monkeypatch) server = LeaseServer(lease=0.6) server.renewal_faults = [None, "503", "timeout", "reset"] @@ -426,8 +545,7 @@ async def handler() -> str: async def test_renewal_that_never_recovers_fails_the_claim_through_the_server( monkeypatch: pytest.MonkeyPatch, fault: str, ) -> None: - monkeypatch.setattr("durable_workflow._activity_lease._RENEWAL_TIMEOUT_SECONDS", 0.05) - monkeypatch.setattr("durable_workflow._activity_lease._RETRY_BASE_SECONDS", 0.02) + fast_retries(monkeypatch) server = LeaseServer(lease=0.5) server.renewal_faults = [fault] * 100 From 6dc63b324268cc5878185e020c4a1306ec5eafde Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 20 Nov 2026 04:20:06 +0000 Subject: [PATCH 4/4] Track activity threads with a race-free guard instead of the future (#352) Wrap a claim's synchronous activity before it is handed to the thread pool. A thread that has not started yet is fenced when the release asks, so it never runs afterwards; a started one counts as stopped only once its function has returned or raised. This also keeps the shared thread-pool code path (a replay-guarded symbol) unchanged. Co-Authored-By: Claude Opus 5.5 --- src/durable_workflow/_activity_lease.py | 52 ++++++++++++++++++++++++- src/durable_workflow/worker.py | 7 ++-- tests/test_activity_lease_authority.py | 33 ++++++++++++++++ 3 files changed, 87 insertions(+), 5 deletions(-) diff --git a/src/durable_workflow/_activity_lease.py b/src/durable_workflow/_activity_lease.py index dbc8a5a8..2b738a74 100644 --- a/src/durable_workflow/_activity_lease.py +++ b/src/durable_workflow/_activity_lease.py @@ -3,9 +3,11 @@ from __future__ import annotations import asyncio +import functools import logging import random import re +import threading import time from collections.abc import Awaitable, Callable, Mapping from contextvars import ContextVar @@ -73,7 +75,47 @@ async def release_worker_lost(client: Any, *, task_id: str, activity_attempt_id: return False +class _ThreadRun: + """Whether a claim's activity thread has stopped, without a start race. + + A thread that has not started yet is fenced when asked, so it never runs + after a release; one that has started counts as stopped only once its + function has returned or raised. + """ + + def __init__(self, task_id: str) -> None: + self._task_id = task_id + self._lock = threading.Lock() + self._started = False + self._finished = False + self._fenced = False + + def wrap(self, fn: Callable[..., _T]) -> Callable[..., _T]: + @functools.wraps(fn) + def guarded(*args: Any, **kwargs: Any) -> _T: + with self._lock: + if self._fenced: + raise ActivityWorkerLost(self._task_id, "activity_execution_abandoned") + self._started = True + try: + return fn(*args, **kwargs) + finally: + with self._lock: + self._finished = True + return guarded + + def fence_or_finished(self) -> bool: + with self._lock: + if not self._started: + self._fenced = True + return True + return self._finished + + def _execution_stopped(execution: Any) -> bool: + fence_or_finished = getattr(execution, "fence_or_finished", None) + if callable(fence_or_finished): + return bool(fence_or_finished()) done = getattr(execution, "done", None) if callable(done): return bool(done()) @@ -134,7 +176,7 @@ class ActivityLeaseAuthority: # failures since it. Neither changes recorded history. lease_seconds: float = field(default=0.0, init=False) renewal_failures: int = field(default=0, init=False) - # Physical executions (thread futures, callback processes) started for + # Physical executions (activity threads, callback processes) started for # this claim. A release waits until every one has actually stopped. executions: list[Any] = field(default_factory=list, init=False) execution_started: bool = field(default=False, init=False) @@ -331,9 +373,15 @@ def renewal_budget(self) -> float: self.lease_until - time.monotonic() - self.report_reserve()) def track(self, execution: Any) -> None: - """Register a thread future or callback process that runs this claim.""" + """Register an execution (callback process, future) that runs this claim.""" self.executions.append(execution) + def guard_thread(self, fn: Callable[..., _T]) -> Callable[..., _T]: + """Wrap a synchronous activity so its thread is tracked for this claim.""" + run = _ThreadRun(self.task_id) + self.track(run) + return run.wrap(fn) + async def confirm_stopped(self, deadline: float) -> bool: """Wait until every tracked execution has stopped, or ``deadline`` passes.""" while not all(_execution_stopped(execution) for execution in self.executions): diff --git a/src/durable_workflow/worker.py b/src/durable_workflow/worker.py index ff37c89b..395dd6c4 100644 --- a/src/durable_workflow/worker.py +++ b/src/durable_workflow/worker.py @@ -3698,6 +3698,10 @@ async def _invoke_remote_activity(self, task: dict[str, Any], activity_type: str fn: Callable[..., Any], args: list[Any], info: ActivityInfo) -> Any: if self._cooperative_cancellation_supported: return await self._execute_cooperative_remote_callable(task, fn, tuple(args), info) + claim = current_activity_authority.get() + if claim is not None and not inspect.iscoroutinefunction(fn): + # A worker_lost release must wait until this thread has stopped. + fn = claim.guard_thread(fn) return await self._execute_activity_callable( task, activity_type, tuple(args), fn, run_sync_in_thread=True, remote=True, ) @@ -3744,9 +3748,6 @@ async def call_activity(ctx: ActivityInterceptorContext) -> Any: self._remote_activity_thread_slots.release() raise self._remote_activity_threads.add(future) - claim = current_activity_authority.get() - if claim is not None: - claim.track(future) def thread_finished(completed: Future[Any]) -> None: def release_capacity() -> None: diff --git a/tests/test_activity_lease_authority.py b/tests/test_activity_lease_authority.py index 95901fac..c9262256 100644 --- a/tests/test_activity_lease_authority.py +++ b/tests/test_activity_lease_authority.py @@ -222,3 +222,36 @@ async def test_stop_confirmation_requires_every_execution_to_have_stopped() -> N callback.stopped = True assert await authority.confirm_stopped(time.monotonic() + 0.05) is True assert await authority.confirm_stopped(time.monotonic() - 1) is False, "a stop after the deadline is too late" + + +def test_thread_guard_fences_a_thread_that_has_not_started() -> None: + import threading + import time + + authority = ActivityLeaseAuthority.delivery( + object(), "17", task(lease_expires_at=timestamp(60), deadlines={}), owner="worker-1", + task_queue="queue-1", server_time=timestamp(0), request_started=time.monotonic(), + ) + calls: list[str] = [] + queued = authority.guard_thread(lambda: calls.append("queued")) + # Asked before its thread starts: counts as stopped and never runs later. + assert authority.executions[0].fence_or_finished() is True + with pytest.raises(ActivityWorkerLost): + queued() + assert calls == [] + + entered, leave = threading.Event(), threading.Event() + + def running() -> str: + entered.set() + leave.wait(timeout=2) + return "done" + + guarded = authority.guard_thread(running) + thread = threading.Thread(target=guarded) + thread.start() + entered.wait(timeout=2) + assert authority.executions[1].fence_or_finished() is False + leave.set() + thread.join(timeout=2) + assert authority.executions[1].fence_or_finished() is True