Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

### Fixed

- `WorkflowHandle.result()` and `Client.get_result()` on a handle bound to a run that continued as new no longer silently return `None` (#356). They now follow the continue-as-new chain to the run that actually ended, as the PHP and Rust SDKs do. That run's result is returned, or its typed failure is raised. The chain is followed through each `WorkflowContinuedAsNew` event's `continued_to_run_id`; when no successor is recorded, the newest run named by the status read is used. A chain that cannot be followed raises `WorkflowFailed`.
- An activity worker no longer silently drops a claimed task when lease renewal fails transiently (#352, same bug as durable-workflow/sdk-php#445). Before, if the `renew-lease` confirmation sent right after an activity poll timed out, hit a connection reset or got a 5xx/429, the poll raised `ActivityLeaseAuthorityLost("activity_initial_lease_unavailable")` and the worker discarded the task. The Server then held the lease (300 s by default) until it expired and, with the default single attempt, failed the workflow.
- **Keeps working:** the activity now runs under the lease it already holds, and renewal is retried inside that lease. The backoff is the same in every SDK: the *n*-th retry waits `uniform(0, min(4, 0.25 * 2**n))` seconds, so 0.25 s doubling to a 4 s cap, with full jitter. Each renewal exchange is bounded to leave a reserve of the lease (at most 10 s, at most a fifth of the grant). An unconfirmed lease is charged from the poll's start, because the Server stamps it only after receiving the poll.
- **Stops before it releases:** if renewal still has not succeeded when only the reserve is left, the worker stops the execution. An async activity is cancelled. A thread activity is asked to stop: its next `heartbeat_sync()` raises `ActivityWorkerLost`. A callback process is stopped. The worker waits for that stop for up to half of the reserve.
Expand Down
106 changes: 66 additions & 40 deletions src/durable_workflow/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -1929,7 +1929,7 @@
values raise ``ValueError`` naming the option, without exposing its value.
"""

def __init__(

Check failure on line 1932 in src/durable_workflow/client.py

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Refactor this function to reduce its Cognitive Complexity from 17 to the 15 allowed.

See more on https://sonarcloud.io/project/issues?id=durable-workflow_sdk-python&issues=AaH4Ezw3UgvRZIAy31-q&open=AaH4Ezw3UgvRZIAy31-q&pullRequest=357
self,
base_url: str,
*,
Expand Down Expand Up @@ -5502,6 +5502,54 @@
)
return cast(dict[str, Any], data)

async def _run_terminal_outcome(

Check failure on line 5505 in src/durable_workflow/client.py

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Refactor this function to reduce its Cognitive Complexity from 31 to the 15 allowed.

See more on https://sonarcloud.io/project/issues?id=durable-workflow_sdk-python&issues=AaH3rYyqz_kWP0RXQs6x&open=AaH3rYyqz_kWP0RXQs6x&pullRequest=357
self, workflow_id: str, run_id: str, desc: WorkflowExecution, result_type: Any,
) -> tuple[str, Any]:
"""Read one run's terminal event from its history.

Returns ``("result", value)`` for a completed run,
``("continued", successor_run_id)`` for a run that continued as new, or
``("no_terminal_event", None)``; raises the typed error of a failed,
terminated, cancelled or timed-out run.
"""
page_token: str | None = None
seen_tokens: set[str] = set()
while True:
history = await self.get_history(workflow_id, run_id, page_size=1000, next_page_token=page_token)
events = history.get("events", [])
for ev in reversed(events):
etype = ev.get("event_type")
payload = ev.get("payload") or {}
if etype == "WorkflowCompleted":
output = serializer.decode_envelope(
payload.get("output"),
codec=payload.get("payload_codec") or desc.payload_codec,
external_storage=self.external_storage,
external_storage_cache=self.external_storage_cache,
)
if result_type is None:
return "result", output
return "result", decode_payload_value(output, result_type, "workflow result")
if etype == "WorkflowFailed":
raise WorkflowFailed(
payload.get("message", "workflow failed"),
payload.get("exception_class"),
)
if etype == "WorkflowTerminated":
raise WorkflowTerminated(payload.get("reason", "workflow was terminated"))
if etype == "WorkflowCancelled":
raise WorkflowCancelled(payload.get("reason", "workflow was cancelled"))
if etype == "WorkflowTimedOut":
raise WorkflowTimedOut()
if etype == "WorkflowContinuedAsNew":
return "continued", payload.get("continued_to_run_id")
page_token = history.get("next_page_token")
if not page_token:
return "no_terminal_event", None
if page_token in seen_tokens:
raise RuntimeError("history pagination repeated a page token")
seen_tokens.add(page_token)

async def get_result(
self,
handle: WorkflowHandle,
Expand Down Expand Up @@ -5542,47 +5590,25 @@
return decode_payload_value(desc.output, result_type, "workflow result")
if run_id is None:
raise WorkflowFailed("no run_id available to fetch history")
page_token: str | None = None
seen_tokens: set[str] = set()
# A handle bound to an earlier run follows its continue-as-new
# chain to the run that actually ended, as PHP and Rust do.
visited: set[str] = set()
while True:
history = await self.get_history(
handle.workflow_id, run_id, page_size=1000, next_page_token=page_token
)
events = history.get("events", [])
for ev in reversed(events):
etype = ev.get("event_type")
payload = ev.get("payload") or {}
if etype == "WorkflowCompleted":
output = serializer.decode_envelope(
payload.get("output"),
codec=payload.get("payload_codec") or desc.payload_codec,
external_storage=self.external_storage,
external_storage_cache=self.external_storage_cache,
)
if result_type is None:
return output
return decode_payload_value(output, result_type, "workflow result")
if etype == "WorkflowFailed":
raise WorkflowFailed(
payload.get("message", "workflow failed"),
payload.get("exception_class"),
)
if etype == "WorkflowTerminated":
raise WorkflowTerminated(
payload.get("reason", "workflow was terminated")
)
if etype == "WorkflowCancelled":
raise WorkflowCancelled(
payload.get("reason", "workflow was cancelled")
)
if etype == "WorkflowTimedOut":
raise WorkflowTimedOut()
page_token = history.get("next_page_token")
if not page_token:
return None
if page_token in seen_tokens:
raise RuntimeError("history pagination repeated a page token")
seen_tokens.add(page_token)
visited.add(run_id)
kind, value = await self._run_terminal_outcome(handle.workflow_id, run_id, desc, result_type)
if kind == "result":
return value
# Continued as new: follow the recorded successor, else the
# newest run the status read named.
successor = value if isinstance(value, str) and value else desc.run_id
if successor is None or successor in visited:
if kind == "no_terminal_event":
return None
raise WorkflowFailed(
f"workflow {handle.workflow_id} run {run_id} continued as new, but its "
"successor run could not be followed"
)
run_id = successor
if asyncio.get_running_loop().time() > deadline:
raise TimeoutError(
f"workflow {handle.workflow_id} not terminal after {timeout}s (status={status})"
Expand Down
81 changes: 81 additions & 0 deletions tests/test_result_continue_as_new.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
"""result() on a handle bound to an earlier run follows continue-as-new to the run that ended."""
from __future__ import annotations

from typing import Any

import httpx
import pytest

from durable_workflow import Client, serializer
from durable_workflow.errors import WorkflowFailed


def continued(to: str | None) -> dict[str, Any]:
payload: dict[str, Any] = {"sequence": 3, "arguments": serializer.envelope([1])}
if to is not None:
payload["continued_to_run_id"] = to
return {"event_type": "WorkflowContinuedAsNew", "payload": payload}


COMPLETED = {"event_type": "WorkflowCompleted", "payload": {"output": serializer.envelope({"final": True})}}
FAILED = {"event_type": "WorkflowFailed", "payload": {"message": "final run failed", "exception_class": "E"}}
STARTED = {"event_type": "WorkflowStarted", "payload": {}}


class ChainServer:
def __init__(self, current: str, status: str, histories: dict[str, list[dict[str, Any]]]) -> None:
self.current, self.status, self.histories = current, status, histories
self.paths: list[str] = []

async def respond(self, request: httpx.Request) -> httpx.Response:
path = request.url.path
self.paths.append(path)
if path.endswith("/history"):
run = path.split("/runs/")[1].split("/")[0]
return httpx.Response(200, json={"events": [STARTED, *self.histories[run]], "next_page_token": None})
return httpx.Response(200, json={"workflow_id": "wf", "run_id": self.current, "workflow_type": "t",
"status": self.status})


async def result_of(server: ChainServer, run_id: str | None) -> Any:
async with Client("http://chain.invalid", token="t") as client:
await client._http.aclose()
client._http = httpx.AsyncClient(base_url="http://chain.invalid", transport=httpx.MockTransport(server.respond))
return await client.get_workflow_handle("wf", run_id=run_id).result(timeout=1)


def history_reads(server: ChainServer) -> list[str]:
return [path.split("/runs/")[1].split("/")[0] for path in server.paths if path.endswith("/history")]


@pytest.mark.parametrize("run_id", ["run-1", None])
async def test_completed_after_continue_as_new_returns_the_final_result(run_id: str | None) -> None:
server = ChainServer("run-2", "completed", {"run-1": [continued("run-2")], "run-2": [COMPLETED]})
assert await result_of(server, run_id) == {"final": True}
assert history_reads(server) == (["run-1", "run-2"] if run_id else ["run-2"])


async def test_failed_after_continue_as_new_raises_the_final_failure() -> None:
server = ChainServer("run-2", "failed", {"run-1": [continued("run-2")], "run-2": [FAILED]})
with pytest.raises(WorkflowFailed, match="final run failed"):
await result_of(server, "run-1")
assert history_reads(server) == ["run-1", "run-2"]


async def test_a_longer_chain_is_followed_run_by_run() -> None:
server = ChainServer("run-3", "completed", {
"run-1": [continued("run-2")], "run-2": [continued("run-3")], "run-3": [COMPLETED],
})
assert await result_of(server, "run-1") == {"final": True}
assert history_reads(server) == ["run-1", "run-2", "run-3"]


async def test_without_a_recorded_successor_the_newest_run_is_used() -> None:
server = ChainServer("run-2", "completed", {"run-1": [continued(None)], "run-2": [COMPLETED]})
assert await result_of(server, "run-1") == {"final": True}


async def test_a_cycle_is_a_failure_not_a_silent_none() -> None:
server = ChainServer("run-2", "completed", {"run-1": [continued("run-2")], "run-2": [continued("run-1")]})
with pytest.raises(WorkflowFailed, match="successor run could not be followed"):
await result_of(server, "run-1")