diff --git a/CHANGELOG.md b/CHANGELOG.md index 42a70f6..8f78726 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- `WorkflowHandle.result()` and `Client.get_result()` on a handle bound to a run that continued as new no longer silently return `None` (#356). They now follow the continue-as-new chain to the run that actually ended, as the PHP and Rust SDKs do. That run's result is returned, or its typed failure is raised. The chain is followed through each `WorkflowContinuedAsNew` event's `continued_to_run_id`; when no successor is recorded, the newest run named by the status read is used. A chain that cannot be followed raises `WorkflowFailed`. - An activity worker no longer silently drops a claimed task when lease renewal fails transiently (#352, same bug as durable-workflow/sdk-php#445). Before, if the `renew-lease` confirmation sent right after an activity poll timed out, hit a connection reset or got a 5xx/429, the poll raised `ActivityLeaseAuthorityLost("activity_initial_lease_unavailable")` and the worker discarded the task. The Server then held the lease (300 s by default) until it expired and, with the default single attempt, failed the workflow. - **Keeps working:** the activity now runs under the lease it already holds, and renewal is retried inside that lease. The backoff is the same in every SDK: the *n*-th retry waits `uniform(0, min(4, 0.25 * 2**n))` seconds, so 0.25 s doubling to a 4 s cap, with full jitter. Each renewal exchange is bounded to leave a reserve of the lease (at most 10 s, at most a fifth of the grant). An unconfirmed lease is charged from the poll's start, because the Server stamps it only after receiving the poll. - **Stops before it releases:** if renewal still has not succeeded when only the reserve is left, the worker stops the execution. An async activity is cancelled. A thread activity is asked to stop: its next `heartbeat_sync()` raises `ActivityWorkerLost`. A callback process is stopped. The worker waits for that stop for up to half of the reserve. diff --git a/src/durable_workflow/client.py b/src/durable_workflow/client.py index fc4c102..7bb8636 100644 --- a/src/durable_workflow/client.py +++ b/src/durable_workflow/client.py @@ -5502,6 +5502,54 @@ async def _read_update_outcome( ) return cast(dict[str, Any], data) + async def _run_terminal_outcome( + self, workflow_id: str, run_id: str, desc: WorkflowExecution, result_type: Any, + ) -> tuple[str, Any]: + """Read one run's terminal event from its history. + + Returns ``("result", value)`` for a completed run, + ``("continued", successor_run_id)`` for a run that continued as new, or + ``("no_terminal_event", None)``; raises the typed error of a failed, + terminated, cancelled or timed-out run. + """ + page_token: str | None = None + seen_tokens: set[str] = set() + while True: + history = await self.get_history(workflow_id, run_id, page_size=1000, next_page_token=page_token) + events = history.get("events", []) + for ev in reversed(events): + etype = ev.get("event_type") + payload = ev.get("payload") or {} + if etype == "WorkflowCompleted": + output = serializer.decode_envelope( + payload.get("output"), + codec=payload.get("payload_codec") or desc.payload_codec, + external_storage=self.external_storage, + external_storage_cache=self.external_storage_cache, + ) + if result_type is None: + return "result", output + return "result", decode_payload_value(output, result_type, "workflow result") + if etype == "WorkflowFailed": + raise WorkflowFailed( + payload.get("message", "workflow failed"), + payload.get("exception_class"), + ) + if etype == "WorkflowTerminated": + raise WorkflowTerminated(payload.get("reason", "workflow was terminated")) + if etype == "WorkflowCancelled": + raise WorkflowCancelled(payload.get("reason", "workflow was cancelled")) + if etype == "WorkflowTimedOut": + raise WorkflowTimedOut() + if etype == "WorkflowContinuedAsNew": + return "continued", payload.get("continued_to_run_id") + page_token = history.get("next_page_token") + if not page_token: + return "no_terminal_event", None + if page_token in seen_tokens: + raise RuntimeError("history pagination repeated a page token") + seen_tokens.add(page_token) + async def get_result( self, handle: WorkflowHandle, @@ -5542,47 +5590,25 @@ async def get_result( return decode_payload_value(desc.output, result_type, "workflow result") if run_id is None: raise WorkflowFailed("no run_id available to fetch history") - page_token: str | None = None - seen_tokens: set[str] = set() + # A handle bound to an earlier run follows its continue-as-new + # chain to the run that actually ended, as PHP and Rust do. + visited: set[str] = set() while True: - history = await self.get_history( - handle.workflow_id, run_id, page_size=1000, next_page_token=page_token - ) - events = history.get("events", []) - for ev in reversed(events): - etype = ev.get("event_type") - payload = ev.get("payload") or {} - if etype == "WorkflowCompleted": - output = serializer.decode_envelope( - payload.get("output"), - codec=payload.get("payload_codec") or desc.payload_codec, - external_storage=self.external_storage, - external_storage_cache=self.external_storage_cache, - ) - if result_type is None: - return output - return decode_payload_value(output, result_type, "workflow result") - if etype == "WorkflowFailed": - raise WorkflowFailed( - payload.get("message", "workflow failed"), - payload.get("exception_class"), - ) - if etype == "WorkflowTerminated": - raise WorkflowTerminated( - payload.get("reason", "workflow was terminated") - ) - if etype == "WorkflowCancelled": - raise WorkflowCancelled( - payload.get("reason", "workflow was cancelled") - ) - if etype == "WorkflowTimedOut": - raise WorkflowTimedOut() - page_token = history.get("next_page_token") - if not page_token: - return None - if page_token in seen_tokens: - raise RuntimeError("history pagination repeated a page token") - seen_tokens.add(page_token) + visited.add(run_id) + kind, value = await self._run_terminal_outcome(handle.workflow_id, run_id, desc, result_type) + if kind == "result": + return value + # Continued as new: follow the recorded successor, else the + # newest run the status read named. + successor = value if isinstance(value, str) and value else desc.run_id + if successor is None or successor in visited: + if kind == "no_terminal_event": + return None + raise WorkflowFailed( + f"workflow {handle.workflow_id} run {run_id} continued as new, but its " + "successor run could not be followed" + ) + run_id = successor if asyncio.get_running_loop().time() > deadline: raise TimeoutError( f"workflow {handle.workflow_id} not terminal after {timeout}s (status={status})" diff --git a/tests/test_result_continue_as_new.py b/tests/test_result_continue_as_new.py new file mode 100644 index 0000000..a1b1a6a --- /dev/null +++ b/tests/test_result_continue_as_new.py @@ -0,0 +1,81 @@ +"""result() on a handle bound to an earlier run follows continue-as-new to the run that ended.""" +from __future__ import annotations + +from typing import Any + +import httpx +import pytest + +from durable_workflow import Client, serializer +from durable_workflow.errors import WorkflowFailed + + +def continued(to: str | None) -> dict[str, Any]: + payload: dict[str, Any] = {"sequence": 3, "arguments": serializer.envelope([1])} + if to is not None: + payload["continued_to_run_id"] = to + return {"event_type": "WorkflowContinuedAsNew", "payload": payload} + + +COMPLETED = {"event_type": "WorkflowCompleted", "payload": {"output": serializer.envelope({"final": True})}} +FAILED = {"event_type": "WorkflowFailed", "payload": {"message": "final run failed", "exception_class": "E"}} +STARTED = {"event_type": "WorkflowStarted", "payload": {}} + + +class ChainServer: + def __init__(self, current: str, status: str, histories: dict[str, list[dict[str, Any]]]) -> None: + self.current, self.status, self.histories = current, status, histories + self.paths: list[str] = [] + + async def respond(self, request: httpx.Request) -> httpx.Response: + path = request.url.path + self.paths.append(path) + if path.endswith("/history"): + run = path.split("/runs/")[1].split("/")[0] + return httpx.Response(200, json={"events": [STARTED, *self.histories[run]], "next_page_token": None}) + return httpx.Response(200, json={"workflow_id": "wf", "run_id": self.current, "workflow_type": "t", + "status": self.status}) + + +async def result_of(server: ChainServer, run_id: str | None) -> Any: + async with Client("http://chain.invalid", token="t") as client: + await client._http.aclose() + client._http = httpx.AsyncClient(base_url="http://chain.invalid", transport=httpx.MockTransport(server.respond)) + return await client.get_workflow_handle("wf", run_id=run_id).result(timeout=1) + + +def history_reads(server: ChainServer) -> list[str]: + return [path.split("/runs/")[1].split("/")[0] for path in server.paths if path.endswith("/history")] + + +@pytest.mark.parametrize("run_id", ["run-1", None]) +async def test_completed_after_continue_as_new_returns_the_final_result(run_id: str | None) -> None: + server = ChainServer("run-2", "completed", {"run-1": [continued("run-2")], "run-2": [COMPLETED]}) + assert await result_of(server, run_id) == {"final": True} + assert history_reads(server) == (["run-1", "run-2"] if run_id else ["run-2"]) + + +async def test_failed_after_continue_as_new_raises_the_final_failure() -> None: + server = ChainServer("run-2", "failed", {"run-1": [continued("run-2")], "run-2": [FAILED]}) + with pytest.raises(WorkflowFailed, match="final run failed"): + await result_of(server, "run-1") + assert history_reads(server) == ["run-1", "run-2"] + + +async def test_a_longer_chain_is_followed_run_by_run() -> None: + server = ChainServer("run-3", "completed", { + "run-1": [continued("run-2")], "run-2": [continued("run-3")], "run-3": [COMPLETED], + }) + assert await result_of(server, "run-1") == {"final": True} + assert history_reads(server) == ["run-1", "run-2", "run-3"] + + +async def test_without_a_recorded_successor_the_newest_run_is_used() -> None: + server = ChainServer("run-2", "completed", {"run-1": [continued(None)], "run-2": [COMPLETED]}) + assert await result_of(server, "run-1") == {"final": True} + + +async def test_a_cycle_is_a_failure_not_a_silent_none() -> None: + server = ChainServer("run-2", "completed", {"run-1": [continued("run-2")], "run-2": [continued("run-1")]}) + with pytest.raises(WorkflowFailed, match="successor run could not be followed"): + await result_of(server, "run-1")