From eeece44bea929a224bb88911b9bbf238ad905823 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 19 Nov 2026 16:27:43 +0000 Subject: [PATCH 1/3] Raise a settled all() group's first recorded failure (#307) When several members of an all() group had failed by the time a task reached it and every member had settled, the worker raised the failure with the lowest input position. Workflow's embedded engine raises the failure recorded first, so the exception a workflow caught depended on whether its task ran before or after the other members finished. A task reaching a settled group now raises its first recorded failure. It records the member it raised with the #338 decision side effect only when that member is not also the lowest-position failure, so its commands are unchanged whenever both rules agree. Replay raises the member a decision names. A settled group with no decision after which the run closed or did further durable work came from a released worker and keeps the position rule and its clock behavior. Native Server fixtures recorded by published 2.19.1 (position rule, complete and in flight) and by this version (decision, deciding task) cover both rules. The selection-gap fixture's frontier expectation is retired as same-protocol evidence under a corrected fixture. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 + docs/sdk-reference.md | 25 +- src/durable_workflow/workflow.py | 30 +- ...led-group-first-task-records-decision.json | 1047 ++++++++++++++ ...d-group-raises-first-recorded-failure.json | 144 ++ ...-group-records-first-recorded-failure.json | 1285 +++++++++++++++++ ...ttled-group-released-worker-in-flight.json | 1112 ++++++++++++++ ...p-released-worker-keeps-position-rule.json | 1276 ++++++++++++++++ tests/test_business_replay_clock_parallel.py | 41 +- .../test_parallel_completion_order_replay.py | 55 +- tests/test_parallel_group_policy.py | 15 +- ...t_parallel_member_failure_open_siblings.py | 121 +- tests/test_replay_regression_corpus.py | 14 +- tests/test_run_closed_settlement_replay.py | 8 +- 14 files changed, 5132 insertions(+), 47 deletions(-) create mode 100644 tests/fixtures/replay_regressions/settled-group-first-task-records-decision.json create mode 100644 tests/fixtures/replay_regressions/settled-group-raises-first-recorded-failure.json create mode 100644 tests/fixtures/replay_regressions/settled-group-records-first-recorded-failure.json create mode 100644 tests/fixtures/replay_regressions/settled-group-released-worker-in-flight.json create mode 100644 tests/fixtures/replay_regressions/settled-group-released-worker-keeps-position-rule.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f6569e1..f37224b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Replay safety:** replay matches `ExternalSignalRequested` at the command's sequence and returns the outcome from the following `ExternalSignalDelivered` or `ExternalSignalFailed`. A recorded signal is never sent again, including on a cold replacement worker and while its outcome is not yet recorded. A different target, run, signal name or arguments at that sequence raises `NonDeterministicReplayError`. Existing histories are unchanged. - **Server requirement:** the Worker submits the command only when the Server advertises `signal_external_workflow` in `worker_protocol.server_capabilities` (and lists it in `supported_workflow_task_commands` when that list is present). Otherwise it fails the workflow task with `signal_external_workflow_unavailable` before submitting, so the signal is never sent outside history. +### Changed + +- **Behavior change:** when several members of an `all()` group (`yield [...]`) have failed by the time the workflow resumes, the failure recorded first in history is now raised, even if every member has already settled. Workers up to 2.19.1 raised the failure with the lowest input position in that case, so the exception a workflow catches could depend on whether its task ran before or after the other members finished. This completes the fail-fast default: the first recorded member failure is raised as soon as it is recorded, the other members keep running and are not cancelled, and their outcomes are ignored, as in Workflow's embedded engine. Groups with one failure, or whose first failure also has the lowest position, are unaffected. (#307) + - **Replay safety:** the new choice is gated by the recorded decision from #338. A task records which member it raised (a side effect keyed `durable_workflow.parallel_failure_decision`, at the sequence right after the group) only when the first recorded failure is not the lowest-position one, so its commands are identical to 2.19.1 whenever both rules agree. Replay raises the member a decision names. A settled group with no decision, after which the run closed or did further durable work, is a history from a released worker and keeps the lowest-position rule, including its business-clock behavior. Released workers could not leave such a group without one or the other, since a workflow task that raises a failure either closes the run or blocks on a durable command. New regression fixtures replay a native Server history recorded by published 2.19.1 under the old rule and one recorded by this version under the new rule. The `business-clock-parallel-failure-selection-gap` fixture, whose expectation was the old rule at an unrecorded frontier, stays unchanged as evidence and its history now replays under `settled-group-raises-first-recorded-failure`. + - **Who is affected:** workflows that catch an `all()` failure and act on which member failed, when two or more members can fail. Code after a group that records a decision starts one workflow sequence later. The explicit `propagate`, `cancel_remaining` and `settle` policies remain tracked in #307 and durable-workflow/.github#232. + ## [2.19.1] - 2026-11-18 ### Fixed diff --git a/docs/sdk-reference.md b/docs/sdk-reference.md index 77e279ed..ea150f08 100644 --- a/docs/sdk-reference.md +++ b/docs/sdk-reference.md @@ -522,15 +522,22 @@ profile, (shipping, _) = results ``` A failed activity or child is thrown at the list-yield point as soon as it is -recorded. The other members keep running; their later outcomes do not change -the thrown failure or reach the workflow's next operations. If every member -had already settled when the workflow resumed, the failure with the lowest -durable input position is thrown. A group whose members are already scheduled -is never scheduled again on a later wake, such as a signal. When a failure is -raised while other members are open, the worker records which member it -raised as a side effect directly after the group, so every later replay raises -the same one. Code after such a group therefore starts one workflow sequence -later. +recorded. The other members keep running and are not cancelled; their later +outcomes do not change the thrown failure or reach the workflow's next +operations. When several members have failed by the time the workflow +resumes, the one whose failure was recorded first in history is thrown, even +if every member has already settled, as in Workflow's embedded engine. A +group whose members are already scheduled is never scheduled again on a later +wake, such as a signal. When a failure is raised while other members are open, +or when a settled group's first recorded failure is not the one with the +lowest input position, the worker records which member it raised as a side +effect directly after the group, so every later replay raises the same one. +Code after such a group therefore starts one workflow sequence later. + +Workers up to 2.19.1 threw the failure with the lowest input position once +every member had settled. Histories they recorded keep that choice on replay: +a settled group with no recorded decision, after which the run closed or did +further durable work, still throws the lowest-position failure. ## Saga compensation diff --git a/src/durable_workflow/workflow.py b/src/durable_workflow/workflow.py index 54fcbb11..272d2407 100644 --- a/src/durable_workflow/workflow.py +++ b/src/durable_workflow/workflow.py @@ -7820,12 +7820,22 @@ def _absent_policy_failure_decision(commands: list[Any]) -> tuple[bool, int | No task therefore records its decision as a side effect directly after the group, and replay raises exactly the member it names. + When every member has already settled by the time a task first reaches + the group, that task also raises the first recorded failure, as + Workflow's embedded engine does (sdk-python#307). It records the + decision only when that failure is not also the lowest-position one, + the member released workers chose; otherwise both rules raise the same + member and the task's commands are unchanged. + A history without a decision replays as it always has when every member settled before the workflow did any further durable work: the workflow waited for the whole group and the failure is chosen by member - position. A history without a decision that moved on while a member was - open came from another runtime, such as Workflow's embedded engine, and - raises its first recorded failure. + position. Released workers wrote no decision for a settled group, and a + task that raises a failure always either closes the run or records + durable work after the group before it can stop, so such a history + always shows that it moved on. A history without a decision that moved + on while a member was open came from another runtime, such as + Workflow's embedded engine, and raises its first recorded failure. """ marked = _parallel_failure_marker(commands) if marked is not None: @@ -7842,7 +7852,19 @@ def _absent_policy_failure_decision(commands: list[Any]) -> tuple[bool, int | No if len({recorded_steps[index].workflow_sequence for index in slots}) != len(commands): return True, None, not moved_on settled = [recorded_steps[index].history_index for index in slots] - if any(index is None for index in settled) or not moved_on: + if any(index is None for index in settled): + return False, None, False + if not moved_on and len(slots) == len(commands): + # This task is the first to act on the settled group. Raise the + # first recorded failure; record it only where the legacy + # position rule would choose another member (sdk-python#307). + failures = [index for index in slots + if isinstance(resolved_results[index], ActivityFailed | ChildWorkflowFailed)] + first_recorded = min(failures, key=lambda index: cast(int, recorded_steps[index].history_index)) + lowest_position = min(failures, key=lambda index: recorded_steps[index].workflow_sequence) + differs = first_recorded != lowest_position + return differs, None, differs + if not moved_on: return False, None, False return min(moved_on) < max(cast(list[int], settled)), None, False diff --git a/tests/fixtures/replay_regressions/settled-group-first-task-records-decision.json b/tests/fixtures/replay_regressions/settled-group-first-task-records-decision.json new file mode 100644 index 00000000..bb0d1d32 --- /dev/null +++ b/tests/fixtures/replay_regressions/settled-group-first-task-records-decision.json @@ -0,0 +1,1047 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "settled-group-first-task-records-decision", + "protocol_version": "1.20", + "bindings": [ + "python" + ], + "settled_group_first_recorded_failure": { + "issues": [ + "durable-workflow/sdk-python#307" + ], + "note": "The history the deciding task received (native Server built from durable-workflow/server main 52e3ec00): every member settled, member 1 failed first. The task records which member it raised and schedules the compensation for it." + }, + "workflow": { + "type": "py307.caught", + "input": [], + "payload_codec": "avro" + }, + "history": [ + { + "sequence": 1, + "event_type": "StartAccepted", + "timestamp": "2026-11-19T16:26:12.027000Z", + "principal": { + "type": "auth:token", + "id": "legacy-token", + "label": "Admin" + }, + "payload": { + "workflow_instance_id": "py307-caught-1795105571", + "workflow_run_id": "01M7TA2H5T1GTXE76ZS15AM165", + "workflow_type": "py307.caught", + "workflow_class": "py307.caught", + "workflow_command_id": "01M7TA2H5T1GTXE76ZS15AM166", + "memo": null, + "business_key": null, + "visibility_labels": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01M7TA2H5T1GTXE76ZS15AM166", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01M7TA2H5T1GTXE76ZS15AM165", + "payload_codec": "avro", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "source": "control_plane", + "context": { + "principal": { + "type": "auth:token", + "id": "legacy-token", + "label": "Admin" + } + }, + "caller_label": "Standalone Server", + "principal_type": "auth:token", + "principal_id": "legacy-token", + "principal_label": "Admin", + "auth_status": "authorized", + "auth_method": "token", + "request_method": "POST", + "request_path": "/api/workflows", + "request_fingerprint": "sha256:47edc9a9a51d6470d6a390ed2559a1fcda1699404c126e1e34ac6345d8076f0f", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-11-19T16:26:12.026000Z", + "applied_at": "2026-11-19T16:26:12.026000Z" + } + } + }, + { + "sequence": 2, + "event_type": "WorkflowStarted", + "timestamp": "2026-11-19T16:26:12.028000Z", + "principal": { + "type": "auth:token", + "id": "legacy-token", + "label": "Admin" + }, + "payload": { + "workflow_instance_id": "py307-caught-1795105571", + "workflow_run_id": "01M7TA2H5T1GTXE76ZS15AM165", + "workflow_type": "py307.caught", + "workflow_class": "py307.caught", + "workflow_command_id": "01M7TA2H5T1GTXE76ZS15AM166", + "execution_timeout_seconds": 3600, + "run_timeout_seconds": 600, + "execution_deadline_at": "2026-11-19T17:26:12.026Z", + "run_deadline_at": "2026-11-19T16:36:12.026Z", + "command": { + "id": "01M7TA2H5T1GTXE76ZS15AM166", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01M7TA2H5T1GTXE76ZS15AM165", + "payload_codec": "avro", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "source": "control_plane", + "context": { + "principal": { + "type": "auth:token", + "id": "legacy-token", + "label": "Admin" + } + }, + "caller_label": "Standalone Server", + "principal_type": "auth:token", + "principal_id": "legacy-token", + "principal_label": "Admin", + "auth_status": "authorized", + "auth_method": "token", + "request_method": "POST", + "request_path": "/api/workflows", + "request_fingerprint": "sha256:47edc9a9a51d6470d6a390ed2559a1fcda1699404c126e1e34ac6345d8076f0f", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-11-19T16:26:12.026000Z", + "applied_at": "2026-11-19T16:26:12.026000Z" + } + } + }, + { + "sequence": 3, + "event_type": "ActivityScheduled", + "timestamp": "2026-11-19T16:26:12.164000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA2HA40T3VYAMYBJSGH18V", + "activity_type": "py307.fail", + "activity_class": "py307.fail", + "sequence": 1, + "activity": { + "id": "01M7TA2HA40T3VYAMYBJSGH18V", + "idempotency_key": "01M7TA2HA40T3VYAMYBJSGH18V", + "sequence": 1, + "type": "py307.fail", + "class": "py307.fail", + "local_activity": false, + "activity_type": "py307.fail", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBgAAAAAAAPg/AA==" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "pending", + "attempt_count": 0, + "closed_at": null, + "attempt_id": null, + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 0 + } + ], + "task": { + "id": "01M7TA2H5WNWMXXSWJ6JJ4M0GS", + "type": "workflow", + "status": "leased", + "attempt_count": 1, + "repair_count": 0, + "queue": "py307-workflows", + "connection": "sqlite", + "available_at": "2026-11-19T16:26:12.028000Z", + "leased_at": "2026-11-19T16:26:12.086000Z", + "lease_expires_at": "2026-11-19T16:27:12.086000Z", + "last_dispatch_attempt_at": "2026-11-19T16:26:12.028000Z", + "last_dispatched_at": "2026-11-19T16:26:12.028000Z", + "lease_owner": "py-worker-89b000d4" + } + } + }, + { + "sequence": 4, + "event_type": "ActivityScheduled", + "timestamp": "2026-11-19T16:26:12.165000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA2HA54J8KXYXJEJ7S4S5T", + "activity_type": "py307.fail", + "activity_class": "py307.fail", + "sequence": 2, + "activity": { + "id": "01M7TA2HA54J8KXYXJEJ7S4S5T", + "idempotency_key": "01M7TA2HA54J8KXYXJEJ7S4S5T", + "sequence": 2, + "type": "py307.fail", + "class": "py307.fail", + "local_activity": false, + "activity_type": "py307.fail", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "pending", + "attempt_count": 0, + "closed_at": null, + "attempt_id": null, + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 1 + } + ], + "task": { + "id": "01M7TA2H5WNWMXXSWJ6JJ4M0GS", + "type": "workflow", + "status": "leased", + "attempt_count": 1, + "repair_count": 0, + "queue": "py307-workflows", + "connection": "sqlite", + "available_at": "2026-11-19T16:26:12.028000Z", + "leased_at": "2026-11-19T16:26:12.086000Z", + "lease_expires_at": "2026-11-19T16:27:12.086000Z", + "last_dispatch_attempt_at": "2026-11-19T16:26:12.028000Z", + "last_dispatched_at": "2026-11-19T16:26:12.028000Z", + "lease_owner": "py-worker-89b000d4" + } + } + }, + { + "sequence": 5, + "event_type": "ActivityScheduled", + "timestamp": "2026-11-19T16:26:12.166000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA2HA66FRY17YJN6HC6S52", + "activity_type": "py307.ok", + "activity_class": "py307.ok", + "sequence": 3, + "activity": { + "id": "01M7TA2HA66FRY17YJN6HC6S52", + "idempotency_key": "01M7TA2HA66FRY17YJN6HC6S52", + "sequence": 3, + "type": "py307.ok", + "class": "py307.ok", + "local_activity": false, + "activity_type": "py307.ok", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "pending", + "attempt_count": 0, + "closed_at": null, + "attempt_id": null, + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 2, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 2 + } + ], + "task": { + "id": "01M7TA2H5WNWMXXSWJ6JJ4M0GS", + "type": "workflow", + "status": "leased", + "attempt_count": 1, + "repair_count": 0, + "queue": "py307-workflows", + "connection": "sqlite", + "available_at": "2026-11-19T16:26:12.028000Z", + "leased_at": "2026-11-19T16:26:12.086000Z", + "lease_expires_at": "2026-11-19T16:27:12.086000Z", + "last_dispatch_attempt_at": "2026-11-19T16:26:12.028000Z", + "last_dispatched_at": "2026-11-19T16:26:12.028000Z", + "lease_owner": "py-worker-89b000d4" + } + } + }, + { + "sequence": 6, + "event_type": "ActivityScheduled", + "timestamp": "2026-11-19T16:26:12.166000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA2HA66FRY17YJN6HC6S55", + "activity_type": "py307.ok", + "activity_class": "py307.ok", + "sequence": 4, + "activity": { + "id": "01M7TA2HA66FRY17YJN6HC6S55", + "idempotency_key": "01M7TA2HA66FRY17YJN6HC6S55", + "sequence": 4, + "type": "py307.ok", + "class": "py307.ok", + "local_activity": false, + "activity_type": "py307.ok", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBgAAAAAAAOA/AA==" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "pending", + "attempt_count": 0, + "closed_at": null, + "attempt_id": null, + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 3, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 3 + } + ], + "task": { + "id": "01M7TA2H5WNWMXXSWJ6JJ4M0GS", + "type": "workflow", + "status": "leased", + "attempt_count": 1, + "repair_count": 0, + "queue": "py307-workflows", + "connection": "sqlite", + "available_at": "2026-11-19T16:26:12.028000Z", + "leased_at": "2026-11-19T16:26:12.086000Z", + "lease_expires_at": "2026-11-19T16:27:12.086000Z", + "last_dispatch_attempt_at": "2026-11-19T16:26:12.028000Z", + "last_dispatched_at": "2026-11-19T16:26:12.028000Z", + "lease_owner": "py-worker-89b000d4" + } + } + }, + { + "sequence": 7, + "event_type": "ActivityStarted", + "timestamp": "2026-11-19T16:26:12.449000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA2HA40T3VYAMYBJSGH18V", + "activity_type": "py307.fail", + "activity_class": "py307.fail", + "sequence": 1, + "activity": { + "id": "01M7TA2HA40T3VYAMYBJSGH18V", + "idempotency_key": "01M7TA2HA40T3VYAMYBJSGH18V", + "sequence": 1, + "type": "py307.fail", + "class": "py307.fail", + "local_activity": false, + "activity_type": "py307.fail", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBgAAAAAAAPg/AA==" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "running", + "attempt_count": 1, + "closed_at": null, + "attempt_id": "01M7TA2HK1H2Y7GACGE77Z49TH", + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities", + "started_at": "2026-11-19T16:26:12.449Z" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 0 + } + ], + "activity_attempt_id": "01M7TA2HK1H2Y7GACGE77Z49TH", + "attempt_number": 1, + "task": { + "id": "01M7TA2HA40T3VYAMYBJSGH18X", + "type": "activity", + "status": "leased", + "repair_count": 0, + "available_at": "2026-11-19T16:26:12.164Z", + "leased_at": "2026-11-19T16:26:12.448Z", + "lease_owner": "py-worker-9b247caa", + "lease_expires_at": "2026-11-19T16:31:12.448Z", + "attempt_count": 1, + "queue": "py307-activities", + "connection": "sqlite" + } + } + }, + { + "sequence": 8, + "event_type": "ActivityStarted", + "timestamp": "2026-11-19T16:26:12.497000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA2HA54J8KXYXJEJ7S4S5T", + "activity_type": "py307.fail", + "activity_class": "py307.fail", + "sequence": 2, + "activity": { + "id": "01M7TA2HA54J8KXYXJEJ7S4S5T", + "idempotency_key": "01M7TA2HA54J8KXYXJEJ7S4S5T", + "sequence": 2, + "type": "py307.fail", + "class": "py307.fail", + "local_activity": false, + "activity_type": "py307.fail", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "running", + "attempt_count": 1, + "closed_at": null, + "attempt_id": "01M7TA2HMHNJVVHNEW678ZZ1KW", + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities", + "started_at": "2026-11-19T16:26:12.497Z" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 1 + } + ], + "activity_attempt_id": "01M7TA2HMHNJVVHNEW678ZZ1KW", + "attempt_number": 1, + "task": { + "id": "01M7TA2HA54J8KXYXJEJ7S4S5W", + "type": "activity", + "status": "leased", + "repair_count": 0, + "available_at": "2026-11-19T16:26:12.165Z", + "leased_at": "2026-11-19T16:26:12.496Z", + "lease_owner": "py-worker-9b247caa", + "lease_expires_at": "2026-11-19T16:31:12.496Z", + "attempt_count": 1, + "queue": "py307-activities", + "connection": "sqlite" + } + } + }, + { + "sequence": 9, + "event_type": "ActivityStarted", + "timestamp": "2026-11-19T16:26:12.567000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA2HA66FRY17YJN6HC6S52", + "activity_type": "py307.ok", + "activity_class": "py307.ok", + "sequence": 3, + "activity": { + "id": "01M7TA2HA66FRY17YJN6HC6S52", + "idempotency_key": "01M7TA2HA66FRY17YJN6HC6S52", + "sequence": 3, + "type": "py307.ok", + "class": "py307.ok", + "local_activity": false, + "activity_type": "py307.ok", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "running", + "attempt_count": 1, + "closed_at": null, + "attempt_id": "01M7TA2HPQRHFPFHKK2D6JVY07", + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities", + "started_at": "2026-11-19T16:26:12.567Z" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 2, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 2 + } + ], + "activity_attempt_id": "01M7TA2HPQRHFPFHKK2D6JVY07", + "attempt_number": 1, + "task": { + "id": "01M7TA2HA66FRY17YJN6HC6S54", + "type": "activity", + "status": "leased", + "repair_count": 0, + "available_at": "2026-11-19T16:26:12.166Z", + "leased_at": "2026-11-19T16:26:12.567Z", + "lease_owner": "py-worker-9b247caa", + "lease_expires_at": "2026-11-19T16:31:12.567Z", + "attempt_count": 1, + "queue": "py307-activities", + "connection": "sqlite" + } + } + }, + { + "sequence": 10, + "event_type": "ActivityFailed", + "timestamp": "2026-11-19T16:26:12.580000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA2HA54J8KXYXJEJ7S4S5T", + "activity_type": "py307.fail", + "activity_class": "py307.fail", + "sequence": 2, + "activity": { + "id": "01M7TA2HA54J8KXYXJEJ7S4S5T", + "idempotency_key": "01M7TA2HA54J8KXYXJEJ7S4S5T", + "sequence": 2, + "type": "py307.fail", + "class": "py307.fail", + "local_activity": false, + "activity_type": "py307.fail", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "failed", + "attempt_count": 1, + "closed_at": "2026-11-19T16:26:12.579Z", + "attempt_id": "01M7TA2HMHNJVVHNEW678ZZ1KW", + "exception": { + "code": 0, + "type": "RuntimeError", + "message": "py307 failed after 0" + }, + "payload_codec": "avro", + "queue": "py307-activities", + "started_at": "2026-11-19T16:26:12.497Z" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 1 + } + ], + "activity_attempt_id": "01M7TA2HMHNJVVHNEW678ZZ1KW", + "attempt_number": 1, + "failure_id": "01M7TA2HQ4AZ38WZ2BVBJKHQKX", + "failure_category": "activity", + "non_retryable": false, + "exception_type": "RuntimeError", + "exception_class": "builtins.RuntimeError", + "message": "py307 failed after 0", + "code": 0, + "exception": { + "type": "RuntimeError", + "message": "py307 failed after 0", + "code": 0 + }, + "task": { + "id": "01M7TA2HA54J8KXYXJEJ7S4S5W", + "type": "activity", + "status": "leased", + "available_at": "2026-11-19T16:26:12.165Z", + "leased_at": "2026-11-19T16:26:12.496Z", + "lease_owner": "py-worker-9b247caa", + "lease_expires_at": "2026-11-19T16:31:12.524Z", + "attempt_count": 1, + "repair_count": 0, + "queue": "py307-activities", + "connection": "sqlite" + } + } + }, + { + "sequence": 11, + "event_type": "ActivityCompleted", + "timestamp": "2026-11-19T16:26:12.638000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA2HA66FRY17YJN6HC6S52", + "activity_type": "py307.ok", + "activity_class": "py307.ok", + "sequence": 3, + "activity": { + "id": "01M7TA2HA66FRY17YJN6HC6S52", + "idempotency_key": "01M7TA2HA66FRY17YJN6HC6S52", + "sequence": 3, + "type": "py307.ok", + "class": "py307.ok", + "local_activity": false, + "activity_type": "py307.ok", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "completed", + "attempt_count": 1, + "closed_at": "2026-11-19T16:26:12.636Z", + "attempt_id": "01M7TA2HPQRHFPFHKK2D6JVY07", + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities", + "started_at": "2026-11-19T16:26:12.567Z" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 2, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 2 + } + ], + "activity_attempt_id": "01M7TA2HPQRHFPFHKK2D6JVY07", + "attempt_number": 1, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoEb2s=" + }, + "payload_codec": "avro", + "task": { + "id": "01M7TA2HA66FRY17YJN6HC6S54", + "type": "activity", + "status": "leased", + "repair_count": 0, + "available_at": "2026-11-19T16:26:12.166Z", + "leased_at": "2026-11-19T16:26:12.567Z", + "lease_owner": "py-worker-9b247caa", + "lease_expires_at": "2026-11-19T16:31:12.615Z", + "attempt_count": 1, + "queue": "py307-activities", + "connection": "sqlite" + } + } + }, + { + "sequence": 12, + "event_type": "ActivityStarted", + "timestamp": "2026-11-19T16:26:12.662000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA2HA66FRY17YJN6HC6S55", + "activity_type": "py307.ok", + "activity_class": "py307.ok", + "sequence": 4, + "activity": { + "id": "01M7TA2HA66FRY17YJN6HC6S55", + "idempotency_key": "01M7TA2HA66FRY17YJN6HC6S55", + "sequence": 4, + "type": "py307.ok", + "class": "py307.ok", + "local_activity": false, + "activity_type": "py307.ok", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBgAAAAAAAOA/AA==" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "running", + "attempt_count": 1, + "closed_at": null, + "attempt_id": "01M7TA2HSPDGABA249E1XKK6EM", + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities", + "started_at": "2026-11-19T16:26:12.662Z" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 3, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 3 + } + ], + "activity_attempt_id": "01M7TA2HSPDGABA249E1XKK6EM", + "attempt_number": 1, + "task": { + "id": "01M7TA2HA77XPRQ2H05A6SRV55", + "type": "activity", + "status": "leased", + "repair_count": 0, + "available_at": "2026-11-19T16:26:12.167Z", + "leased_at": "2026-11-19T16:26:12.662Z", + "lease_owner": "py-worker-9b247caa", + "lease_expires_at": "2026-11-19T16:31:12.662Z", + "attempt_count": 1, + "queue": "py307-activities", + "connection": "sqlite" + } + } + }, + { + "sequence": 13, + "event_type": "ActivityCompleted", + "timestamp": "2026-11-19T16:26:13.261000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA2HA66FRY17YJN6HC6S55", + "activity_type": "py307.ok", + "activity_class": "py307.ok", + "sequence": 4, + "activity": { + "id": "01M7TA2HA66FRY17YJN6HC6S55", + "idempotency_key": "01M7TA2HA66FRY17YJN6HC6S55", + "sequence": 4, + "type": "py307.ok", + "class": "py307.ok", + "local_activity": false, + "activity_type": "py307.ok", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBgAAAAAAAOA/AA==" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "completed", + "attempt_count": 1, + "closed_at": "2026-11-19T16:26:13.261Z", + "attempt_id": "01M7TA2HSPDGABA249E1XKK6EM", + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities", + "started_at": "2026-11-19T16:26:12.662Z" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 3, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 3 + } + ], + "activity_attempt_id": "01M7TA2HSPDGABA249E1XKK6EM", + "attempt_number": 1, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoEb2s=" + }, + "payload_codec": "avro", + "task": { + "id": "01M7TA2HA77XPRQ2H05A6SRV55", + "type": "activity", + "status": "leased", + "repair_count": 0, + "available_at": "2026-11-19T16:26:12.167Z", + "leased_at": "2026-11-19T16:26:12.662Z", + "lease_owner": "py-worker-9b247caa", + "lease_expires_at": "2026-11-19T16:31:12.724Z", + "attempt_count": 1, + "queue": "py307-activities", + "connection": "sqlite" + } + } + }, + { + "sequence": 14, + "event_type": "ActivityFailed", + "timestamp": "2026-11-19T16:26:14.004000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA2HA40T3VYAMYBJSGH18V", + "activity_type": "py307.fail", + "activity_class": "py307.fail", + "sequence": 1, + "activity": { + "id": "01M7TA2HA40T3VYAMYBJSGH18V", + "idempotency_key": "01M7TA2HA40T3VYAMYBJSGH18V", + "sequence": 1, + "type": "py307.fail", + "class": "py307.fail", + "local_activity": false, + "activity_type": "py307.fail", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBgAAAAAAAPg/AA==" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "failed", + "attempt_count": 1, + "closed_at": "2026-11-19T16:26:14.003Z", + "attempt_id": "01M7TA2HK1H2Y7GACGE77Z49TH", + "exception": { + "code": 0, + "type": "RuntimeError", + "message": "py307 failed after 1.5" + }, + "payload_codec": "avro", + "queue": "py307-activities", + "started_at": "2026-11-19T16:26:12.449Z" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 0 + } + ], + "activity_attempt_id": "01M7TA2HK1H2Y7GACGE77Z49TH", + "attempt_number": 1, + "failure_id": "01M7TA2K3KSSXE8NCV02N8BP2J", + "failure_category": "activity", + "non_retryable": false, + "exception_type": "RuntimeError", + "exception_class": "builtins.RuntimeError", + "message": "py307 failed after 1.5", + "code": 0, + "exception": { + "type": "RuntimeError", + "message": "py307 failed after 1.5", + "code": 0 + }, + "task": { + "id": "01M7TA2HA40T3VYAMYBJSGH18X", + "type": "activity", + "status": "leased", + "available_at": "2026-11-19T16:26:12.164Z", + "leased_at": "2026-11-19T16:26:12.448Z", + "lease_owner": "py-worker-9b247caa", + "lease_expires_at": "2026-11-19T16:31:12.471Z", + "attempt_count": 1, + "repair_count": 0, + "queue": "py307-activities", + "connection": "sqlite" + } + } + } + ], + "expected": { + "command_sequence": [ + { + "type": "record_side_effect" + }, + { + "type": "schedule_activity", + "activity_type": "py307.compensate", + "arguments": [ + "py307.fail#2" + ] + } + ] + } +} diff --git a/tests/fixtures/replay_regressions/settled-group-raises-first-recorded-failure.json b/tests/fixtures/replay_regressions/settled-group-raises-first-recorded-failure.json new file mode 100644 index 00000000..a704200b --- /dev/null +++ b/tests/fixtures/replay_regressions/settled-group-raises-first-recorded-failure.json @@ -0,0 +1,144 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "settled-group-raises-first-recorded-failure", + "protocol_version": "1.20", + "bindings": [ + "python" + ], + "settled_group_first_recorded_failure": { + "issues": [ + "durable-workflow/sdk-python#307" + ], + "note": "The history of business-clock-parallel-failure-selection-gap: both members failed before a task reached the group, member 2 first. That fixture expected the lower position (member 1) with no clock, because member 1's failure was not the earliest. Like Workflow's embedded engine, the task now raises the first recorded failure, with that failure's clock, and records which member it raised because released workers would have chosen another member. A run that a released worker already moved past keeps the position rule." + }, + "workflow": { + "type": "tests.replay.business-clock-parallel-failure-gap", + "input": [], + "payload_codec": "avro", + "workflow_id": "business-clock-parallel-failure-selection-gap", + "run_id": "business-clock-parallel-failure-selection-gap-run" + }, + "history": [ + { + "event_type": "WorkflowStarted", + "recorded_at": "2026-01-01T00:00:00.000000+00:00", + "payload": {}, + "sequence": 1 + }, + { + "event_type": "ActivityScheduled", + "recorded_at": "2026-01-01T00:00:00.000000+00:00", + "payload": { + "sequence": 1, + "activity_execution_id": "group-execution-0", + "activity_type": "group.0", + "parallel_group_id": "parallel-activities:1:2", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:2", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 0 + } + ] + }, + "sequence": 2 + }, + { + "event_type": "ActivityScheduled", + "recorded_at": "2026-01-01T00:00:00.000000+00:00", + "payload": { + "sequence": 2, + "activity_execution_id": "group-execution-1", + "activity_type": "group.1", + "parallel_group_id": "parallel-activities:1:2", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:2", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 1 + } + ] + }, + "sequence": 3 + }, + { + "event_type": "ActivityFailed", + "recorded_at": "2026-01-01T00:00:01Z", + "payload": { + "sequence": 2, + "activity_execution_id": "group-execution-1", + "activity_type": "group.1", + "parallel_group_id": "parallel-activities:1:2", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:2", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 1 + } + ], + "message": "failure-1" + }, + "sequence": 4 + }, + { + "event_type": "ActivityFailed", + "recorded_at": "2026-01-01T00:00:03.123100+00:00", + "payload": { + "sequence": 1, + "activity_execution_id": "group-execution-0", + "activity_type": "group.0", + "parallel_group_id": "parallel-activities:1:2", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:2", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 0 + } + ], + "message": "failure-0" + }, + "sequence": 5 + } + ], + "expected": { + "command_sequence": [ + { + "type": "record_side_effect" + }, + { + "type": "complete_workflow", + "result": { + "activity_type": "group.1", + "message": "failure-1", + "clock": "2026-01-01T00:00:01.000000+00:00", + "legacy": "2026-01-01T00:00:00.000000+00:00" + } + } + ] + } +} diff --git a/tests/fixtures/replay_regressions/settled-group-records-first-recorded-failure.json b/tests/fixtures/replay_regressions/settled-group-records-first-recorded-failure.json new file mode 100644 index 00000000..ac3f020e --- /dev/null +++ b/tests/fixtures/replay_regressions/settled-group-records-first-recorded-failure.json @@ -0,0 +1,1285 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "settled-group-records-first-recorded-failure", + "protocol_version": "1.20", + "bindings": [ + "python" + ], + "settled_group_first_recorded_failure": { + "issues": [ + "durable-workflow/sdk-python#307" + ], + "note": "Recorded by this version on native Server built from durable-workflow/server main 52e3ec00, same scenario. The task raised member 1, whose failure was recorded first, as Workflow's embedded engine does, recorded that decision at sequence 5 and scheduled the compensation at 6." + }, + "workflow": { + "type": "py307.caught", + "input": [], + "payload_codec": "avro", + "query": { + "name": "caught", + "arguments": [] + } + }, + "history": [ + { + "sequence": 1, + "event_type": "StartAccepted", + "timestamp": "2026-11-19T16:26:12.027000Z", + "principal": { + "type": "auth:token", + "id": "legacy-token", + "label": "Admin" + }, + "payload": { + "workflow_instance_id": "py307-caught-1795105571", + "workflow_run_id": "01M7TA2H5T1GTXE76ZS15AM165", + "workflow_type": "py307.caught", + "workflow_class": "py307.caught", + "workflow_command_id": "01M7TA2H5T1GTXE76ZS15AM166", + "memo": null, + "business_key": null, + "visibility_labels": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01M7TA2H5T1GTXE76ZS15AM166", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01M7TA2H5T1GTXE76ZS15AM165", + "payload_codec": "avro", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "source": "control_plane", + "context": { + "principal": { + "type": "auth:token", + "id": "legacy-token", + "label": "Admin" + } + }, + "caller_label": "Standalone Server", + "principal_type": "auth:token", + "principal_id": "legacy-token", + "principal_label": "Admin", + "auth_status": "authorized", + "auth_method": "token", + "request_method": "POST", + "request_path": "/api/workflows", + "request_fingerprint": "sha256:47edc9a9a51d6470d6a390ed2559a1fcda1699404c126e1e34ac6345d8076f0f", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-11-19T16:26:12.026000Z", + "applied_at": "2026-11-19T16:26:12.026000Z" + } + } + }, + { + "sequence": 2, + "event_type": "WorkflowStarted", + "timestamp": "2026-11-19T16:26:12.028000Z", + "principal": { + "type": "auth:token", + "id": "legacy-token", + "label": "Admin" + }, + "payload": { + "workflow_instance_id": "py307-caught-1795105571", + "workflow_run_id": "01M7TA2H5T1GTXE76ZS15AM165", + "workflow_type": "py307.caught", + "workflow_class": "py307.caught", + "workflow_command_id": "01M7TA2H5T1GTXE76ZS15AM166", + "execution_timeout_seconds": 3600, + "run_timeout_seconds": 600, + "execution_deadline_at": "2026-11-19T17:26:12.026Z", + "run_deadline_at": "2026-11-19T16:36:12.026Z", + "command": { + "id": "01M7TA2H5T1GTXE76ZS15AM166", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01M7TA2H5T1GTXE76ZS15AM165", + "payload_codec": "avro", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "source": "control_plane", + "context": { + "principal": { + "type": "auth:token", + "id": "legacy-token", + "label": "Admin" + } + }, + "caller_label": "Standalone Server", + "principal_type": "auth:token", + "principal_id": "legacy-token", + "principal_label": "Admin", + "auth_status": "authorized", + "auth_method": "token", + "request_method": "POST", + "request_path": "/api/workflows", + "request_fingerprint": "sha256:47edc9a9a51d6470d6a390ed2559a1fcda1699404c126e1e34ac6345d8076f0f", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-11-19T16:26:12.026000Z", + "applied_at": "2026-11-19T16:26:12.026000Z" + } + } + }, + { + "sequence": 3, + "event_type": "ActivityScheduled", + "timestamp": "2026-11-19T16:26:12.164000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA2HA40T3VYAMYBJSGH18V", + "activity_type": "py307.fail", + "activity_class": "py307.fail", + "sequence": 1, + "activity": { + "id": "01M7TA2HA40T3VYAMYBJSGH18V", + "idempotency_key": "01M7TA2HA40T3VYAMYBJSGH18V", + "sequence": 1, + "type": "py307.fail", + "class": "py307.fail", + "local_activity": false, + "activity_type": "py307.fail", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBgAAAAAAAPg/AA==" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "pending", + "attempt_count": 0, + "closed_at": null, + "attempt_id": null, + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 0 + } + ], + "task": { + "id": "01M7TA2H5WNWMXXSWJ6JJ4M0GS", + "type": "workflow", + "status": "leased", + "attempt_count": 1, + "repair_count": 0, + "queue": "py307-workflows", + "connection": "sqlite", + "available_at": "2026-11-19T16:26:12.028000Z", + "leased_at": "2026-11-19T16:26:12.086000Z", + "lease_expires_at": "2026-11-19T16:27:12.086000Z", + "last_dispatch_attempt_at": "2026-11-19T16:26:12.028000Z", + "last_dispatched_at": "2026-11-19T16:26:12.028000Z", + "lease_owner": "py-worker-89b000d4" + } + } + }, + { + "sequence": 4, + "event_type": "ActivityScheduled", + "timestamp": "2026-11-19T16:26:12.165000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA2HA54J8KXYXJEJ7S4S5T", + "activity_type": "py307.fail", + "activity_class": "py307.fail", + "sequence": 2, + "activity": { + "id": "01M7TA2HA54J8KXYXJEJ7S4S5T", + "idempotency_key": "01M7TA2HA54J8KXYXJEJ7S4S5T", + "sequence": 2, + "type": "py307.fail", + "class": "py307.fail", + "local_activity": false, + "activity_type": "py307.fail", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "pending", + "attempt_count": 0, + "closed_at": null, + "attempt_id": null, + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 1 + } + ], + "task": { + "id": "01M7TA2H5WNWMXXSWJ6JJ4M0GS", + "type": "workflow", + "status": "leased", + "attempt_count": 1, + "repair_count": 0, + "queue": "py307-workflows", + "connection": "sqlite", + "available_at": "2026-11-19T16:26:12.028000Z", + "leased_at": "2026-11-19T16:26:12.086000Z", + "lease_expires_at": "2026-11-19T16:27:12.086000Z", + "last_dispatch_attempt_at": "2026-11-19T16:26:12.028000Z", + "last_dispatched_at": "2026-11-19T16:26:12.028000Z", + "lease_owner": "py-worker-89b000d4" + } + } + }, + { + "sequence": 5, + "event_type": "ActivityScheduled", + "timestamp": "2026-11-19T16:26:12.166000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA2HA66FRY17YJN6HC6S52", + "activity_type": "py307.ok", + "activity_class": "py307.ok", + "sequence": 3, + "activity": { + "id": "01M7TA2HA66FRY17YJN6HC6S52", + "idempotency_key": "01M7TA2HA66FRY17YJN6HC6S52", + "sequence": 3, + "type": "py307.ok", + "class": "py307.ok", + "local_activity": false, + "activity_type": "py307.ok", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "pending", + "attempt_count": 0, + "closed_at": null, + "attempt_id": null, + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 2, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 2 + } + ], + "task": { + "id": "01M7TA2H5WNWMXXSWJ6JJ4M0GS", + "type": "workflow", + "status": "leased", + "attempt_count": 1, + "repair_count": 0, + "queue": "py307-workflows", + "connection": "sqlite", + "available_at": "2026-11-19T16:26:12.028000Z", + "leased_at": "2026-11-19T16:26:12.086000Z", + "lease_expires_at": "2026-11-19T16:27:12.086000Z", + "last_dispatch_attempt_at": "2026-11-19T16:26:12.028000Z", + "last_dispatched_at": "2026-11-19T16:26:12.028000Z", + "lease_owner": "py-worker-89b000d4" + } + } + }, + { + "sequence": 6, + "event_type": "ActivityScheduled", + "timestamp": "2026-11-19T16:26:12.166000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA2HA66FRY17YJN6HC6S55", + "activity_type": "py307.ok", + "activity_class": "py307.ok", + "sequence": 4, + "activity": { + "id": "01M7TA2HA66FRY17YJN6HC6S55", + "idempotency_key": "01M7TA2HA66FRY17YJN6HC6S55", + "sequence": 4, + "type": "py307.ok", + "class": "py307.ok", + "local_activity": false, + "activity_type": "py307.ok", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBgAAAAAAAOA/AA==" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "pending", + "attempt_count": 0, + "closed_at": null, + "attempt_id": null, + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 3, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 3 + } + ], + "task": { + "id": "01M7TA2H5WNWMXXSWJ6JJ4M0GS", + "type": "workflow", + "status": "leased", + "attempt_count": 1, + "repair_count": 0, + "queue": "py307-workflows", + "connection": "sqlite", + "available_at": "2026-11-19T16:26:12.028000Z", + "leased_at": "2026-11-19T16:26:12.086000Z", + "lease_expires_at": "2026-11-19T16:27:12.086000Z", + "last_dispatch_attempt_at": "2026-11-19T16:26:12.028000Z", + "last_dispatched_at": "2026-11-19T16:26:12.028000Z", + "lease_owner": "py-worker-89b000d4" + } + } + }, + { + "sequence": 7, + "event_type": "ActivityStarted", + "timestamp": "2026-11-19T16:26:12.449000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA2HA40T3VYAMYBJSGH18V", + "activity_type": "py307.fail", + "activity_class": "py307.fail", + "sequence": 1, + "activity": { + "id": "01M7TA2HA40T3VYAMYBJSGH18V", + "idempotency_key": "01M7TA2HA40T3VYAMYBJSGH18V", + "sequence": 1, + "type": "py307.fail", + "class": "py307.fail", + "local_activity": false, + "activity_type": "py307.fail", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBgAAAAAAAPg/AA==" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "running", + "attempt_count": 1, + "closed_at": null, + "attempt_id": "01M7TA2HK1H2Y7GACGE77Z49TH", + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities", + "started_at": "2026-11-19T16:26:12.449Z" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 0 + } + ], + "activity_attempt_id": "01M7TA2HK1H2Y7GACGE77Z49TH", + "attempt_number": 1, + "task": { + "id": "01M7TA2HA40T3VYAMYBJSGH18X", + "type": "activity", + "status": "leased", + "repair_count": 0, + "available_at": "2026-11-19T16:26:12.164Z", + "leased_at": "2026-11-19T16:26:12.448Z", + "lease_owner": "py-worker-9b247caa", + "lease_expires_at": "2026-11-19T16:31:12.448Z", + "attempt_count": 1, + "queue": "py307-activities", + "connection": "sqlite" + } + } + }, + { + "sequence": 8, + "event_type": "ActivityStarted", + "timestamp": "2026-11-19T16:26:12.497000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA2HA54J8KXYXJEJ7S4S5T", + "activity_type": "py307.fail", + "activity_class": "py307.fail", + "sequence": 2, + "activity": { + "id": "01M7TA2HA54J8KXYXJEJ7S4S5T", + "idempotency_key": "01M7TA2HA54J8KXYXJEJ7S4S5T", + "sequence": 2, + "type": "py307.fail", + "class": "py307.fail", + "local_activity": false, + "activity_type": "py307.fail", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "running", + "attempt_count": 1, + "closed_at": null, + "attempt_id": "01M7TA2HMHNJVVHNEW678ZZ1KW", + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities", + "started_at": "2026-11-19T16:26:12.497Z" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 1 + } + ], + "activity_attempt_id": "01M7TA2HMHNJVVHNEW678ZZ1KW", + "attempt_number": 1, + "task": { + "id": "01M7TA2HA54J8KXYXJEJ7S4S5W", + "type": "activity", + "status": "leased", + "repair_count": 0, + "available_at": "2026-11-19T16:26:12.165Z", + "leased_at": "2026-11-19T16:26:12.496Z", + "lease_owner": "py-worker-9b247caa", + "lease_expires_at": "2026-11-19T16:31:12.496Z", + "attempt_count": 1, + "queue": "py307-activities", + "connection": "sqlite" + } + } + }, + { + "sequence": 9, + "event_type": "ActivityStarted", + "timestamp": "2026-11-19T16:26:12.567000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA2HA66FRY17YJN6HC6S52", + "activity_type": "py307.ok", + "activity_class": "py307.ok", + "sequence": 3, + "activity": { + "id": "01M7TA2HA66FRY17YJN6HC6S52", + "idempotency_key": "01M7TA2HA66FRY17YJN6HC6S52", + "sequence": 3, + "type": "py307.ok", + "class": "py307.ok", + "local_activity": false, + "activity_type": "py307.ok", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "running", + "attempt_count": 1, + "closed_at": null, + "attempt_id": "01M7TA2HPQRHFPFHKK2D6JVY07", + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities", + "started_at": "2026-11-19T16:26:12.567Z" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 2, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 2 + } + ], + "activity_attempt_id": "01M7TA2HPQRHFPFHKK2D6JVY07", + "attempt_number": 1, + "task": { + "id": "01M7TA2HA66FRY17YJN6HC6S54", + "type": "activity", + "status": "leased", + "repair_count": 0, + "available_at": "2026-11-19T16:26:12.166Z", + "leased_at": "2026-11-19T16:26:12.567Z", + "lease_owner": "py-worker-9b247caa", + "lease_expires_at": "2026-11-19T16:31:12.567Z", + "attempt_count": 1, + "queue": "py307-activities", + "connection": "sqlite" + } + } + }, + { + "sequence": 10, + "event_type": "ActivityFailed", + "timestamp": "2026-11-19T16:26:12.580000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA2HA54J8KXYXJEJ7S4S5T", + "activity_type": "py307.fail", + "activity_class": "py307.fail", + "sequence": 2, + "activity": { + "id": "01M7TA2HA54J8KXYXJEJ7S4S5T", + "idempotency_key": "01M7TA2HA54J8KXYXJEJ7S4S5T", + "sequence": 2, + "type": "py307.fail", + "class": "py307.fail", + "local_activity": false, + "activity_type": "py307.fail", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "failed", + "attempt_count": 1, + "closed_at": "2026-11-19T16:26:12.579Z", + "attempt_id": "01M7TA2HMHNJVVHNEW678ZZ1KW", + "exception": { + "code": 0, + "type": "RuntimeError", + "message": "py307 failed after 0" + }, + "payload_codec": "avro", + "queue": "py307-activities", + "started_at": "2026-11-19T16:26:12.497Z" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 1 + } + ], + "activity_attempt_id": "01M7TA2HMHNJVVHNEW678ZZ1KW", + "attempt_number": 1, + "failure_id": "01M7TA2HQ4AZ38WZ2BVBJKHQKX", + "failure_category": "activity", + "non_retryable": false, + "exception_type": "RuntimeError", + "exception_class": "builtins.RuntimeError", + "message": "py307 failed after 0", + "code": 0, + "exception": { + "type": "RuntimeError", + "message": "py307 failed after 0", + "code": 0 + }, + "task": { + "id": "01M7TA2HA54J8KXYXJEJ7S4S5W", + "type": "activity", + "status": "leased", + "available_at": "2026-11-19T16:26:12.165Z", + "leased_at": "2026-11-19T16:26:12.496Z", + "lease_owner": "py-worker-9b247caa", + "lease_expires_at": "2026-11-19T16:31:12.524Z", + "attempt_count": 1, + "repair_count": 0, + "queue": "py307-activities", + "connection": "sqlite" + } + } + }, + { + "sequence": 11, + "event_type": "ActivityCompleted", + "timestamp": "2026-11-19T16:26:12.638000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA2HA66FRY17YJN6HC6S52", + "activity_type": "py307.ok", + "activity_class": "py307.ok", + "sequence": 3, + "activity": { + "id": "01M7TA2HA66FRY17YJN6HC6S52", + "idempotency_key": "01M7TA2HA66FRY17YJN6HC6S52", + "sequence": 3, + "type": "py307.ok", + "class": "py307.ok", + "local_activity": false, + "activity_type": "py307.ok", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "completed", + "attempt_count": 1, + "closed_at": "2026-11-19T16:26:12.636Z", + "attempt_id": "01M7TA2HPQRHFPFHKK2D6JVY07", + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities", + "started_at": "2026-11-19T16:26:12.567Z" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 2, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 2 + } + ], + "activity_attempt_id": "01M7TA2HPQRHFPFHKK2D6JVY07", + "attempt_number": 1, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoEb2s=" + }, + "payload_codec": "avro", + "task": { + "id": "01M7TA2HA66FRY17YJN6HC6S54", + "type": "activity", + "status": "leased", + "repair_count": 0, + "available_at": "2026-11-19T16:26:12.166Z", + "leased_at": "2026-11-19T16:26:12.567Z", + "lease_owner": "py-worker-9b247caa", + "lease_expires_at": "2026-11-19T16:31:12.615Z", + "attempt_count": 1, + "queue": "py307-activities", + "connection": "sqlite" + } + } + }, + { + "sequence": 12, + "event_type": "ActivityStarted", + "timestamp": "2026-11-19T16:26:12.662000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA2HA66FRY17YJN6HC6S55", + "activity_type": "py307.ok", + "activity_class": "py307.ok", + "sequence": 4, + "activity": { + "id": "01M7TA2HA66FRY17YJN6HC6S55", + "idempotency_key": "01M7TA2HA66FRY17YJN6HC6S55", + "sequence": 4, + "type": "py307.ok", + "class": "py307.ok", + "local_activity": false, + "activity_type": "py307.ok", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBgAAAAAAAOA/AA==" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "running", + "attempt_count": 1, + "closed_at": null, + "attempt_id": "01M7TA2HSPDGABA249E1XKK6EM", + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities", + "started_at": "2026-11-19T16:26:12.662Z" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 3, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 3 + } + ], + "activity_attempt_id": "01M7TA2HSPDGABA249E1XKK6EM", + "attempt_number": 1, + "task": { + "id": "01M7TA2HA77XPRQ2H05A6SRV55", + "type": "activity", + "status": "leased", + "repair_count": 0, + "available_at": "2026-11-19T16:26:12.167Z", + "leased_at": "2026-11-19T16:26:12.662Z", + "lease_owner": "py-worker-9b247caa", + "lease_expires_at": "2026-11-19T16:31:12.662Z", + "attempt_count": 1, + "queue": "py307-activities", + "connection": "sqlite" + } + } + }, + { + "sequence": 13, + "event_type": "ActivityCompleted", + "timestamp": "2026-11-19T16:26:13.261000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA2HA66FRY17YJN6HC6S55", + "activity_type": "py307.ok", + "activity_class": "py307.ok", + "sequence": 4, + "activity": { + "id": "01M7TA2HA66FRY17YJN6HC6S55", + "idempotency_key": "01M7TA2HA66FRY17YJN6HC6S55", + "sequence": 4, + "type": "py307.ok", + "class": "py307.ok", + "local_activity": false, + "activity_type": "py307.ok", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBgAAAAAAAOA/AA==" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "completed", + "attempt_count": 1, + "closed_at": "2026-11-19T16:26:13.261Z", + "attempt_id": "01M7TA2HSPDGABA249E1XKK6EM", + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities", + "started_at": "2026-11-19T16:26:12.662Z" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 3, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 3 + } + ], + "activity_attempt_id": "01M7TA2HSPDGABA249E1XKK6EM", + "attempt_number": 1, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoEb2s=" + }, + "payload_codec": "avro", + "task": { + "id": "01M7TA2HA77XPRQ2H05A6SRV55", + "type": "activity", + "status": "leased", + "repair_count": 0, + "available_at": "2026-11-19T16:26:12.167Z", + "leased_at": "2026-11-19T16:26:12.662Z", + "lease_owner": "py-worker-9b247caa", + "lease_expires_at": "2026-11-19T16:31:12.724Z", + "attempt_count": 1, + "queue": "py307-activities", + "connection": "sqlite" + } + } + }, + { + "sequence": 14, + "event_type": "ActivityFailed", + "timestamp": "2026-11-19T16:26:14.004000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA2HA40T3VYAMYBJSGH18V", + "activity_type": "py307.fail", + "activity_class": "py307.fail", + "sequence": 1, + "activity": { + "id": "01M7TA2HA40T3VYAMYBJSGH18V", + "idempotency_key": "01M7TA2HA40T3VYAMYBJSGH18V", + "sequence": 1, + "type": "py307.fail", + "class": "py307.fail", + "local_activity": false, + "activity_type": "py307.fail", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBgAAAAAAAPg/AA==" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "failed", + "attempt_count": 1, + "closed_at": "2026-11-19T16:26:14.003Z", + "attempt_id": "01M7TA2HK1H2Y7GACGE77Z49TH", + "exception": { + "code": 0, + "type": "RuntimeError", + "message": "py307 failed after 1.5" + }, + "payload_codec": "avro", + "queue": "py307-activities", + "started_at": "2026-11-19T16:26:12.449Z" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 0 + } + ], + "activity_attempt_id": "01M7TA2HK1H2Y7GACGE77Z49TH", + "attempt_number": 1, + "failure_id": "01M7TA2K3KSSXE8NCV02N8BP2J", + "failure_category": "activity", + "non_retryable": false, + "exception_type": "RuntimeError", + "exception_class": "builtins.RuntimeError", + "message": "py307 failed after 1.5", + "code": 0, + "exception": { + "type": "RuntimeError", + "message": "py307 failed after 1.5", + "code": 0 + }, + "task": { + "id": "01M7TA2HA40T3VYAMYBJSGH18X", + "type": "activity", + "status": "leased", + "available_at": "2026-11-19T16:26:12.164Z", + "leased_at": "2026-11-19T16:26:12.448Z", + "lease_owner": "py-worker-9b247caa", + "lease_expires_at": "2026-11-19T16:31:12.471Z", + "attempt_count": 1, + "repair_count": 0, + "queue": "py307-activities", + "connection": "sqlite" + } + } + }, + { + "sequence": 15, + "event_type": "SideEffectRecorded", + "timestamp": "2026-11-19T16:26:15.406000Z", + "principal": null, + "payload": { + "sequence": 5, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNw4CVGR1cmFibGVfd29ya2Zsb3cucGFyYWxsZWxfZmFpbHVyZV9kZWNpc2lvbg4IInBhcmFsbGVsX2dyb3VwX2lkCi5wYXJhbGxlbC1hY3Rpdml0aWVzOjE6NDhwYXJhbGxlbF9ncm91cF9iYXNlX3NlcXVlbmNlBAImcGFyYWxsZWxfZ3JvdXBfc2l6ZQQIHm1lbWJlcl9zZXF1ZW5jZQQEAAA=" + }, + "task": { + "id": "01M7TA2HQ5FCX8G18Y9FJDHP25", + "type": "workflow", + "status": "leased", + "attempt_count": 1, + "repair_count": 0, + "queue": "py307-workflows", + "connection": "sqlite", + "available_at": "2026-11-19T16:26:12.581000Z", + "leased_at": "2026-11-19T16:26:15.354000Z", + "lease_expires_at": "2026-11-19T16:27:15.354000Z", + "last_dispatch_attempt_at": "2026-11-19T16:26:12.581000Z", + "last_dispatched_at": "2026-11-19T16:26:12.581000Z", + "lease_owner": "py-worker-4a68eb6b" + } + } + }, + { + "sequence": 16, + "event_type": "ActivityScheduled", + "timestamp": "2026-11-19T16:26:15.408000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA2MFF9GAY7D9X6VPGKEFX", + "activity_type": "py307.compensate", + "activity_class": "py307.compensate", + "sequence": 6, + "activity": { + "id": "01M7TA2MFF9GAY7D9X6VPGKEFX", + "idempotency_key": "01M7TA2MFF9GAY7D9X6VPGKEFX", + "sequence": 6, + "type": "py307.compensate", + "class": "py307.compensate", + "local_activity": false, + "activity_type": "py307.compensate", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCChhweTMwNy5mYWlsIzIA" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "pending", + "attempt_count": 0, + "closed_at": null, + "attempt_id": null, + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities" + }, + "task": { + "id": "01M7TA2HQ5FCX8G18Y9FJDHP25", + "type": "workflow", + "status": "leased", + "attempt_count": 1, + "repair_count": 0, + "queue": "py307-workflows", + "connection": "sqlite", + "available_at": "2026-11-19T16:26:12.581000Z", + "leased_at": "2026-11-19T16:26:15.354000Z", + "lease_expires_at": "2026-11-19T16:27:15.354000Z", + "last_dispatch_attempt_at": "2026-11-19T16:26:12.581000Z", + "last_dispatched_at": "2026-11-19T16:26:12.581000Z", + "lease_owner": "py-worker-4a68eb6b" + } + } + }, + { + "sequence": 17, + "event_type": "ActivityStarted", + "timestamp": "2026-11-19T16:26:15.459000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA2MFF9GAY7D9X6VPGKEFX", + "activity_type": "py307.compensate", + "activity_class": "py307.compensate", + "sequence": 6, + "activity": { + "id": "01M7TA2MFF9GAY7D9X6VPGKEFX", + "idempotency_key": "01M7TA2MFF9GAY7D9X6VPGKEFX", + "sequence": 6, + "type": "py307.compensate", + "class": "py307.compensate", + "local_activity": false, + "activity_type": "py307.compensate", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCChhweTMwNy5mYWlsIzIA" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "running", + "attempt_count": 1, + "closed_at": null, + "attempt_id": "01M7TA2MH3ZS1380G4XZQY3RJA", + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities", + "started_at": "2026-11-19T16:26:15.459Z" + }, + "activity_attempt_id": "01M7TA2MH3ZS1380G4XZQY3RJA", + "attempt_number": 1, + "task": { + "id": "01M7TA2MFG4KJAR1B08YR7ZMNW", + "type": "activity", + "status": "leased", + "repair_count": 0, + "available_at": "2026-11-19T16:26:15.408Z", + "leased_at": "2026-11-19T16:26:15.458Z", + "lease_owner": "py-worker-9b247caa", + "lease_expires_at": "2026-11-19T16:31:15.458Z", + "attempt_count": 1, + "queue": "py307-activities", + "connection": "sqlite" + } + } + }, + { + "sequence": 18, + "event_type": "ActivityCompleted", + "timestamp": "2026-11-19T16:26:15.520000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA2MFF9GAY7D9X6VPGKEFX", + "activity_type": "py307.compensate", + "activity_class": "py307.compensate", + "sequence": 6, + "activity": { + "id": "01M7TA2MFF9GAY7D9X6VPGKEFX", + "idempotency_key": "01M7TA2MFF9GAY7D9X6VPGKEFX", + "sequence": 6, + "type": "py307.compensate", + "class": "py307.compensate", + "local_activity": false, + "activity_type": "py307.compensate", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCChhweTMwNy5mYWlsIzIA" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "completed", + "attempt_count": 1, + "closed_at": "2026-11-19T16:26:15.520Z", + "attempt_id": "01M7TA2MH3ZS1380G4XZQY3RJA", + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities", + "started_at": "2026-11-19T16:26:15.459Z" + }, + "activity_attempt_id": "01M7TA2MH3ZS1380G4XZQY3RJA", + "attempt_number": 1, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwomdW5kb25lOnB5MzA3LmZhaWwjMg==" + }, + "payload_codec": "avro", + "task": { + "id": "01M7TA2MFG4KJAR1B08YR7ZMNW", + "type": "activity", + "status": "leased", + "repair_count": 0, + "available_at": "2026-11-19T16:26:15.408Z", + "leased_at": "2026-11-19T16:26:15.458Z", + "lease_owner": "py-worker-9b247caa", + "lease_expires_at": "2026-11-19T16:31:15.493Z", + "attempt_count": 1, + "queue": "py307-activities", + "connection": "sqlite" + } + } + }, + { + "sequence": 19, + "event_type": "WorkflowCompleted", + "timestamp": "2026-11-19T16:26:15.577000Z", + "principal": { + "type": "auth:token", + "id": "legacy-token", + "label": "Admin" + }, + "payload": { + "output": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwomdW5kb25lOnB5MzA3LmZhaWwjMg==" + }, + "payload_codec": "avro", + "task": { + "id": "01M7TA2MK03BFHYXNRNDTRSBEE", + "type": "workflow", + "status": "leased", + "attempt_count": 1, + "repair_count": 0, + "queue": "py307-workflows", + "connection": "sqlite", + "available_at": "2026-11-19T16:26:15.520000Z", + "leased_at": "2026-11-19T16:26:15.538000Z", + "lease_expires_at": "2026-11-19T16:27:15.538000Z", + "last_dispatch_attempt_at": "2026-11-19T16:26:15.520000Z", + "last_dispatched_at": "2026-11-19T16:26:15.520000Z", + "lease_owner": "py-worker-4a68eb6b" + }, + "command": { + "type": "complete_workflow", + "source": "worker_protocol", + "principal_type": "auth:token", + "principal_id": "legacy-token", + "principal_label": "Admin" + } + } + } + ], + "expected": { + "query_result": "py307.fail#2" + } +} diff --git a/tests/fixtures/replay_regressions/settled-group-released-worker-in-flight.json b/tests/fixtures/replay_regressions/settled-group-released-worker-in-flight.json new file mode 100644 index 00000000..aa40d4ea --- /dev/null +++ b/tests/fixtures/replay_regressions/settled-group-released-worker-in-flight.json @@ -0,0 +1,1112 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "settled-group-released-worker-in-flight", + "protocol_version": "1.20", + "bindings": [ + "python" + ], + "settled_group_first_recorded_failure": { + "issues": [ + "durable-workflow/sdk-python#307" + ], + "note": "The same 2.19.1 history up to the compensation being scheduled, as a newer worker would receive it mid-run. The group settled before the run moved on and there is no decision, so replay keeps the position rule, records nothing and waits for the compensation." + }, + "workflow": { + "type": "py307.caught", + "input": [], + "payload_codec": "avro" + }, + "history": [ + { + "sequence": 1, + "event_type": "StartAccepted", + "timestamp": "2026-11-19T16:26:03.345000Z", + "principal": { + "type": "auth:token", + "id": "legacy-token", + "label": "Admin" + }, + "payload": { + "workflow_instance_id": "py307-caught-1795105562", + "workflow_run_id": "01M7TA28PGCF0Y2NYMJ5SKAF2R", + "workflow_type": "py307.caught", + "workflow_class": "py307.caught", + "workflow_command_id": "01M7TA28PGCF0Y2NYMJ5SKAF2S", + "memo": null, + "business_key": null, + "visibility_labels": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01M7TA28PGCF0Y2NYMJ5SKAF2S", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01M7TA28PGCF0Y2NYMJ5SKAF2R", + "payload_codec": "avro", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "source": "control_plane", + "context": { + "principal": { + "type": "auth:token", + "id": "legacy-token", + "label": "Admin" + } + }, + "caller_label": "Standalone Server", + "principal_type": "auth:token", + "principal_id": "legacy-token", + "principal_label": "Admin", + "auth_status": "authorized", + "auth_method": "token", + "request_method": "POST", + "request_path": "/api/workflows", + "request_fingerprint": "sha256:3c0cfd10f3b47621a23bf48ca6d5c4c30eef5510c7f8c53bc915ba405e304c8e", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-11-19T16:26:03.344000Z", + "applied_at": "2026-11-19T16:26:03.344000Z" + } + } + }, + { + "sequence": 2, + "event_type": "WorkflowStarted", + "timestamp": "2026-11-19T16:26:03.346000Z", + "principal": { + "type": "auth:token", + "id": "legacy-token", + "label": "Admin" + }, + "payload": { + "workflow_instance_id": "py307-caught-1795105562", + "workflow_run_id": "01M7TA28PGCF0Y2NYMJ5SKAF2R", + "workflow_type": "py307.caught", + "workflow_class": "py307.caught", + "workflow_command_id": "01M7TA28PGCF0Y2NYMJ5SKAF2S", + "execution_timeout_seconds": 3600, + "run_timeout_seconds": 600, + "execution_deadline_at": "2026-11-19T17:26:03.344Z", + "run_deadline_at": "2026-11-19T16:36:03.344Z", + "workflow_definition_fingerprint": "sha256:b6ef61c873dd7992ecea935b6da5da46e58eeb3f59d778941a9238c62da0775b", + "workflow_definition_fingerprint_source": "worker", + "declared_signals": [], + "declared_signal_contracts": [], + "declared_queries": [ + "caught" + ], + "declared_query_contracts": [ + { + "name": "caught", + "parameters": [] + } + ], + "declared_updates": [], + "declared_update_contracts": [], + "declared_update_validators": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "external:py307.caught", + "command": { + "id": "01M7TA28PGCF0Y2NYMJ5SKAF2S", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01M7TA28PGCF0Y2NYMJ5SKAF2R", + "payload_codec": "avro", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "source": "control_plane", + "context": { + "principal": { + "type": "auth:token", + "id": "legacy-token", + "label": "Admin" + } + }, + "caller_label": "Standalone Server", + "principal_type": "auth:token", + "principal_id": "legacy-token", + "principal_label": "Admin", + "auth_status": "authorized", + "auth_method": "token", + "request_method": "POST", + "request_path": "/api/workflows", + "request_fingerprint": "sha256:3c0cfd10f3b47621a23bf48ca6d5c4c30eef5510c7f8c53bc915ba405e304c8e", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-11-19T16:26:03.344000Z", + "applied_at": "2026-11-19T16:26:03.344000Z" + } + } + }, + { + "sequence": 3, + "event_type": "ActivityScheduled", + "timestamp": "2026-11-19T16:26:03.414000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA28RNH2S976DGTHY5XQV3", + "activity_type": "py307.fail", + "activity_class": "py307.fail", + "sequence": 1, + "activity": { + "id": "01M7TA28RNH2S976DGTHY5XQV3", + "idempotency_key": "01M7TA28RNH2S976DGTHY5XQV3", + "sequence": 1, + "type": "py307.fail", + "class": "py307.fail", + "local_activity": false, + "activity_type": "py307.fail", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBgAAAAAAAPg/AA==" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "pending", + "attempt_count": 0, + "closed_at": null, + "attempt_id": null, + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 0 + } + ], + "task": { + "id": "01M7TA28PJ36XM1BKGW1EEBY7Z", + "type": "workflow", + "status": "leased", + "attempt_count": 1, + "repair_count": 0, + "queue": "py307-workflows", + "connection": "sqlite", + "available_at": "2026-11-19T16:26:03.346000Z", + "leased_at": "2026-11-19T16:26:03.363000Z", + "lease_expires_at": "2026-11-19T16:27:03.363000Z", + "last_dispatch_attempt_at": "2026-11-19T16:26:03.346000Z", + "last_dispatched_at": "2026-11-19T16:26:03.346000Z", + "lease_owner": "py-worker-7541e585" + } + } + }, + { + "sequence": 4, + "event_type": "ActivityScheduled", + "timestamp": "2026-11-19T16:26:03.418000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA28RTG30JYJK03KWMMAX0", + "activity_type": "py307.fail", + "activity_class": "py307.fail", + "sequence": 2, + "activity": { + "id": "01M7TA28RTG30JYJK03KWMMAX0", + "idempotency_key": "01M7TA28RTG30JYJK03KWMMAX0", + "sequence": 2, + "type": "py307.fail", + "class": "py307.fail", + "local_activity": false, + "activity_type": "py307.fail", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "pending", + "attempt_count": 0, + "closed_at": null, + "attempt_id": null, + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 1 + } + ], + "task": { + "id": "01M7TA28PJ36XM1BKGW1EEBY7Z", + "type": "workflow", + "status": "leased", + "attempt_count": 1, + "repair_count": 0, + "queue": "py307-workflows", + "connection": "sqlite", + "available_at": "2026-11-19T16:26:03.346000Z", + "leased_at": "2026-11-19T16:26:03.363000Z", + "lease_expires_at": "2026-11-19T16:27:03.363000Z", + "last_dispatch_attempt_at": "2026-11-19T16:26:03.346000Z", + "last_dispatched_at": "2026-11-19T16:26:03.346000Z", + "lease_owner": "py-worker-7541e585" + } + } + }, + { + "sequence": 5, + "event_type": "ActivityScheduled", + "timestamp": "2026-11-19T16:26:03.418000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA28RTG30JYJK03KWMMAX3", + "activity_type": "py307.ok", + "activity_class": "py307.ok", + "sequence": 3, + "activity": { + "id": "01M7TA28RTG30JYJK03KWMMAX3", + "idempotency_key": "01M7TA28RTG30JYJK03KWMMAX3", + "sequence": 3, + "type": "py307.ok", + "class": "py307.ok", + "local_activity": false, + "activity_type": "py307.ok", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "pending", + "attempt_count": 0, + "closed_at": null, + "attempt_id": null, + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 2, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 2 + } + ], + "task": { + "id": "01M7TA28PJ36XM1BKGW1EEBY7Z", + "type": "workflow", + "status": "leased", + "attempt_count": 1, + "repair_count": 0, + "queue": "py307-workflows", + "connection": "sqlite", + "available_at": "2026-11-19T16:26:03.346000Z", + "leased_at": "2026-11-19T16:26:03.363000Z", + "lease_expires_at": "2026-11-19T16:27:03.363000Z", + "last_dispatch_attempt_at": "2026-11-19T16:26:03.346000Z", + "last_dispatched_at": "2026-11-19T16:26:03.346000Z", + "lease_owner": "py-worker-7541e585" + } + } + }, + { + "sequence": 6, + "event_type": "ActivityScheduled", + "timestamp": "2026-11-19T16:26:03.419000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA28RV50NSTJRKGCX0ZSWZ", + "activity_type": "py307.ok", + "activity_class": "py307.ok", + "sequence": 4, + "activity": { + "id": "01M7TA28RV50NSTJRKGCX0ZSWZ", + "idempotency_key": "01M7TA28RV50NSTJRKGCX0ZSWZ", + "sequence": 4, + "type": "py307.ok", + "class": "py307.ok", + "local_activity": false, + "activity_type": "py307.ok", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBgAAAAAAAOA/AA==" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "pending", + "attempt_count": 0, + "closed_at": null, + "attempt_id": null, + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 3, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 3 + } + ], + "task": { + "id": "01M7TA28PJ36XM1BKGW1EEBY7Z", + "type": "workflow", + "status": "leased", + "attempt_count": 1, + "repair_count": 0, + "queue": "py307-workflows", + "connection": "sqlite", + "available_at": "2026-11-19T16:26:03.346000Z", + "leased_at": "2026-11-19T16:26:03.363000Z", + "lease_expires_at": "2026-11-19T16:27:03.363000Z", + "last_dispatch_attempt_at": "2026-11-19T16:26:03.346000Z", + "last_dispatched_at": "2026-11-19T16:26:03.346000Z", + "lease_owner": "py-worker-7541e585" + } + } + }, + { + "sequence": 7, + "event_type": "ActivityStarted", + "timestamp": "2026-11-19T16:26:03.732000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA28RNH2S976DGTHY5XQV3", + "activity_type": "py307.fail", + "activity_class": "py307.fail", + "sequence": 1, + "activity": { + "id": "01M7TA28RNH2S976DGTHY5XQV3", + "idempotency_key": "01M7TA28RNH2S976DGTHY5XQV3", + "sequence": 1, + "type": "py307.fail", + "class": "py307.fail", + "local_activity": false, + "activity_type": "py307.fail", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBgAAAAAAAPg/AA==" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "running", + "attempt_count": 1, + "closed_at": null, + "attempt_id": "01M7TA292MTQHDAPQXJFXWC6AR", + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities", + "started_at": "2026-11-19T16:26:03.732Z" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 0 + } + ], + "activity_attempt_id": "01M7TA292MTQHDAPQXJFXWC6AR", + "attempt_number": 1, + "task": { + "id": "01M7TA28RQWRA89H9PRJH37XN3", + "type": "activity", + "status": "leased", + "repair_count": 0, + "available_at": "2026-11-19T16:26:03.415Z", + "leased_at": "2026-11-19T16:26:03.731Z", + "lease_owner": "py-worker-1fa82ca5", + "lease_expires_at": "2026-11-19T16:31:03.731Z", + "attempt_count": 1, + "queue": "py307-activities", + "connection": "sqlite" + } + } + }, + { + "sequence": 8, + "event_type": "ActivityStarted", + "timestamp": "2026-11-19T16:26:03.776000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA28RTG30JYJK03KWMMAX0", + "activity_type": "py307.fail", + "activity_class": "py307.fail", + "sequence": 2, + "activity": { + "id": "01M7TA28RTG30JYJK03KWMMAX0", + "idempotency_key": "01M7TA28RTG30JYJK03KWMMAX0", + "sequence": 2, + "type": "py307.fail", + "class": "py307.fail", + "local_activity": false, + "activity_type": "py307.fail", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "running", + "attempt_count": 1, + "closed_at": null, + "attempt_id": "01M7TA2940W1AY6NJFYRRF6DPS", + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities", + "started_at": "2026-11-19T16:26:03.776Z" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 1 + } + ], + "activity_attempt_id": "01M7TA2940W1AY6NJFYRRF6DPS", + "attempt_number": 1, + "task": { + "id": "01M7TA28RTG30JYJK03KWMMAX2", + "type": "activity", + "status": "leased", + "repair_count": 0, + "available_at": "2026-11-19T16:26:03.418Z", + "leased_at": "2026-11-19T16:26:03.776Z", + "lease_owner": "py-worker-1fa82ca5", + "lease_expires_at": "2026-11-19T16:31:03.776Z", + "attempt_count": 1, + "queue": "py307-activities", + "connection": "sqlite" + } + } + }, + { + "sequence": 9, + "event_type": "ActivityFailed", + "timestamp": "2026-11-19T16:26:03.820000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA28RTG30JYJK03KWMMAX0", + "activity_type": "py307.fail", + "activity_class": "py307.fail", + "sequence": 2, + "activity": { + "id": "01M7TA28RTG30JYJK03KWMMAX0", + "idempotency_key": "01M7TA28RTG30JYJK03KWMMAX0", + "sequence": 2, + "type": "py307.fail", + "class": "py307.fail", + "local_activity": false, + "activity_type": "py307.fail", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "failed", + "attempt_count": 1, + "closed_at": "2026-11-19T16:26:03.820Z", + "attempt_id": "01M7TA2940W1AY6NJFYRRF6DPS", + "exception": { + "code": 0, + "type": "RuntimeError", + "message": "py307 failed after 0" + }, + "payload_codec": "avro", + "queue": "py307-activities", + "started_at": "2026-11-19T16:26:03.776Z" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 1 + } + ], + "activity_attempt_id": "01M7TA2940W1AY6NJFYRRF6DPS", + "attempt_number": 1, + "failure_id": "01M7TA295CHSV4G450F0756BQB", + "failure_category": "activity", + "non_retryable": false, + "exception_type": "RuntimeError", + "exception_class": "builtins.RuntimeError", + "message": "py307 failed after 0", + "code": 0, + "exception": { + "type": "RuntimeError", + "message": "py307 failed after 0", + "code": 0 + }, + "task": { + "id": "01M7TA28RTG30JYJK03KWMMAX2", + "type": "activity", + "status": "leased", + "available_at": "2026-11-19T16:26:03.418Z", + "leased_at": "2026-11-19T16:26:03.776Z", + "lease_owner": "py-worker-1fa82ca5", + "lease_expires_at": "2026-11-19T16:31:03.791Z", + "attempt_count": 1, + "repair_count": 0, + "queue": "py307-activities", + "connection": "sqlite" + } + } + }, + { + "sequence": 10, + "event_type": "ActivityStarted", + "timestamp": "2026-11-19T16:26:03.842000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA28RTG30JYJK03KWMMAX3", + "activity_type": "py307.ok", + "activity_class": "py307.ok", + "sequence": 3, + "activity": { + "id": "01M7TA28RTG30JYJK03KWMMAX3", + "idempotency_key": "01M7TA28RTG30JYJK03KWMMAX3", + "sequence": 3, + "type": "py307.ok", + "class": "py307.ok", + "local_activity": false, + "activity_type": "py307.ok", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "running", + "attempt_count": 1, + "closed_at": null, + "attempt_id": "01M7TA2962MSYN39HYTMH5HYDR", + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities", + "started_at": "2026-11-19T16:26:03.842Z" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 2, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 2 + } + ], + "activity_attempt_id": "01M7TA2962MSYN39HYTMH5HYDR", + "attempt_number": 1, + "task": { + "id": "01M7TA28RTG30JYJK03KWMMAX5", + "type": "activity", + "status": "leased", + "repair_count": 0, + "available_at": "2026-11-19T16:26:03.418Z", + "leased_at": "2026-11-19T16:26:03.841Z", + "lease_owner": "py-worker-1fa82ca5", + "lease_expires_at": "2026-11-19T16:31:03.841Z", + "attempt_count": 1, + "queue": "py307-activities", + "connection": "sqlite" + } + } + }, + { + "sequence": 11, + "event_type": "ActivityCompleted", + "timestamp": "2026-11-19T16:26:03.882000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA28RTG30JYJK03KWMMAX3", + "activity_type": "py307.ok", + "activity_class": "py307.ok", + "sequence": 3, + "activity": { + "id": "01M7TA28RTG30JYJK03KWMMAX3", + "idempotency_key": "01M7TA28RTG30JYJK03KWMMAX3", + "sequence": 3, + "type": "py307.ok", + "class": "py307.ok", + "local_activity": false, + "activity_type": "py307.ok", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "completed", + "attempt_count": 1, + "closed_at": "2026-11-19T16:26:03.882Z", + "attempt_id": "01M7TA2962MSYN39HYTMH5HYDR", + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities", + "started_at": "2026-11-19T16:26:03.842Z" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 2, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 2 + } + ], + "activity_attempt_id": "01M7TA2962MSYN39HYTMH5HYDR", + "attempt_number": 1, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoEb2s=" + }, + "payload_codec": "avro", + "task": { + "id": "01M7TA28RTG30JYJK03KWMMAX5", + "type": "activity", + "status": "leased", + "repair_count": 0, + "available_at": "2026-11-19T16:26:03.418Z", + "leased_at": "2026-11-19T16:26:03.841Z", + "lease_owner": "py-worker-1fa82ca5", + "lease_expires_at": "2026-11-19T16:31:03.861Z", + "attempt_count": 1, + "queue": "py307-activities", + "connection": "sqlite" + } + } + }, + { + "sequence": 12, + "event_type": "ActivityStarted", + "timestamp": "2026-11-19T16:26:03.903000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA28RV50NSTJRKGCX0ZSWZ", + "activity_type": "py307.ok", + "activity_class": "py307.ok", + "sequence": 4, + "activity": { + "id": "01M7TA28RV50NSTJRKGCX0ZSWZ", + "idempotency_key": "01M7TA28RV50NSTJRKGCX0ZSWZ", + "sequence": 4, + "type": "py307.ok", + "class": "py307.ok", + "local_activity": false, + "activity_type": "py307.ok", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBgAAAAAAAOA/AA==" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "running", + "attempt_count": 1, + "closed_at": null, + "attempt_id": "01M7TA297Z31QD8QN22Y56NG92", + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities", + "started_at": "2026-11-19T16:26:03.903Z" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 3, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 3 + } + ], + "activity_attempt_id": "01M7TA297Z31QD8QN22Y56NG92", + "attempt_number": 1, + "task": { + "id": "01M7TA28RV50NSTJRKGCX0ZSX1", + "type": "activity", + "status": "leased", + "repair_count": 0, + "available_at": "2026-11-19T16:26:03.419Z", + "leased_at": "2026-11-19T16:26:03.902Z", + "lease_owner": "py-worker-1fa82ca5", + "lease_expires_at": "2026-11-19T16:31:03.902Z", + "attempt_count": 1, + "queue": "py307-activities", + "connection": "sqlite" + } + } + }, + { + "sequence": 13, + "event_type": "ActivityCompleted", + "timestamp": "2026-11-19T16:26:04.501000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA28RV50NSTJRKGCX0ZSWZ", + "activity_type": "py307.ok", + "activity_class": "py307.ok", + "sequence": 4, + "activity": { + "id": "01M7TA28RV50NSTJRKGCX0ZSWZ", + "idempotency_key": "01M7TA28RV50NSTJRKGCX0ZSWZ", + "sequence": 4, + "type": "py307.ok", + "class": "py307.ok", + "local_activity": false, + "activity_type": "py307.ok", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBgAAAAAAAOA/AA==" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "completed", + "attempt_count": 1, + "closed_at": "2026-11-19T16:26:04.501Z", + "attempt_id": "01M7TA297Z31QD8QN22Y56NG92", + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities", + "started_at": "2026-11-19T16:26:03.903Z" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 3, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 3 + } + ], + "activity_attempt_id": "01M7TA297Z31QD8QN22Y56NG92", + "attempt_number": 1, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoEb2s=" + }, + "payload_codec": "avro", + "task": { + "id": "01M7TA28RV50NSTJRKGCX0ZSX1", + "type": "activity", + "status": "leased", + "repair_count": 0, + "available_at": "2026-11-19T16:26:03.419Z", + "leased_at": "2026-11-19T16:26:03.902Z", + "lease_owner": "py-worker-1fa82ca5", + "lease_expires_at": "2026-11-19T16:31:03.965Z", + "attempt_count": 1, + "queue": "py307-activities", + "connection": "sqlite" + } + } + }, + { + "sequence": 14, + "event_type": "ActivityFailed", + "timestamp": "2026-11-19T16:26:05.279000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA28RNH2S976DGTHY5XQV3", + "activity_type": "py307.fail", + "activity_class": "py307.fail", + "sequence": 1, + "activity": { + "id": "01M7TA28RNH2S976DGTHY5XQV3", + "idempotency_key": "01M7TA28RNH2S976DGTHY5XQV3", + "sequence": 1, + "type": "py307.fail", + "class": "py307.fail", + "local_activity": false, + "activity_type": "py307.fail", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBgAAAAAAAPg/AA==" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "failed", + "attempt_count": 1, + "closed_at": "2026-11-19T16:26:05.278Z", + "attempt_id": "01M7TA292MTQHDAPQXJFXWC6AR", + "exception": { + "code": 0, + "type": "RuntimeError", + "message": "py307 failed after 1.5" + }, + "payload_codec": "avro", + "queue": "py307-activities", + "started_at": "2026-11-19T16:26:03.732Z" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 0 + } + ], + "activity_attempt_id": "01M7TA292MTQHDAPQXJFXWC6AR", + "attempt_number": 1, + "failure_id": "01M7TA2AJY5XYXFMX8N37GDBN8", + "failure_category": "activity", + "non_retryable": false, + "exception_type": "RuntimeError", + "exception_class": "builtins.RuntimeError", + "message": "py307 failed after 1.5", + "code": 0, + "exception": { + "type": "RuntimeError", + "message": "py307 failed after 1.5", + "code": 0 + }, + "task": { + "id": "01M7TA28RQWRA89H9PRJH37XN3", + "type": "activity", + "status": "leased", + "available_at": "2026-11-19T16:26:03.415Z", + "leased_at": "2026-11-19T16:26:03.731Z", + "lease_owner": "py-worker-1fa82ca5", + "lease_expires_at": "2026-11-19T16:31:03.760Z", + "attempt_count": 1, + "repair_count": 0, + "queue": "py307-activities", + "connection": "sqlite" + } + } + }, + { + "sequence": 15, + "event_type": "ActivityScheduled", + "timestamp": "2026-11-19T16:26:06.659000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA2BY3Y2NKYZT41M8FRC4T", + "activity_type": "py307.compensate", + "activity_class": "py307.compensate", + "sequence": 5, + "activity": { + "id": "01M7TA2BY3Y2NKYZT41M8FRC4T", + "idempotency_key": "01M7TA2BY3Y2NKYZT41M8FRC4T", + "sequence": 5, + "type": "py307.compensate", + "class": "py307.compensate", + "local_activity": false, + "activity_type": "py307.compensate", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCChhweTMwNy5mYWlsIzEA" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "pending", + "attempt_count": 0, + "closed_at": null, + "attempt_id": null, + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities" + }, + "task": { + "id": "01M7TA295DM570FQ0HKAAAXZ7B", + "type": "workflow", + "status": "leased", + "attempt_count": 1, + "repair_count": 0, + "queue": "py307-workflows", + "connection": "sqlite", + "available_at": "2026-11-19T16:26:03.821000Z", + "leased_at": "2026-11-19T16:26:06.618000Z", + "lease_expires_at": "2026-11-19T16:27:06.618000Z", + "last_dispatch_attempt_at": "2026-11-19T16:26:03.821000Z", + "last_dispatched_at": "2026-11-19T16:26:03.821000Z", + "lease_owner": "py-worker-08341312" + } + } + } + ], + "expected": { + "command_sequence": [] + } +} diff --git a/tests/fixtures/replay_regressions/settled-group-released-worker-keeps-position-rule.json b/tests/fixtures/replay_regressions/settled-group-released-worker-keeps-position-rule.json new file mode 100644 index 00000000..7c8c9ff0 --- /dev/null +++ b/tests/fixtures/replay_regressions/settled-group-released-worker-keeps-position-rule.json @@ -0,0 +1,1276 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "settled-group-released-worker-keeps-position-rule", + "protocol_version": "1.20", + "bindings": [ + "python" + ], + "settled_group_first_recorded_failure": { + "issues": [ + "durable-workflow/sdk-python#307" + ], + "note": "Recorded by published durable-workflow 2.19.1 on native Server built from durable-workflow/server main 52e3ec00. Member 1 (sequence 2) failed first and member 0 (sequence 1) failed later; every member had settled before the workflow task ran. 2.19.1 raised member 0 by position, recorded no decision and scheduled the compensation at sequence 5. Replay keeps that choice." + }, + "workflow": { + "type": "py307.caught", + "input": [], + "payload_codec": "avro", + "query": { + "name": "caught", + "arguments": [] + } + }, + "history": [ + { + "sequence": 1, + "event_type": "StartAccepted", + "timestamp": "2026-11-19T16:26:03.345000Z", + "principal": { + "type": "auth:token", + "id": "legacy-token", + "label": "Admin" + }, + "payload": { + "workflow_instance_id": "py307-caught-1795105562", + "workflow_run_id": "01M7TA28PGCF0Y2NYMJ5SKAF2R", + "workflow_type": "py307.caught", + "workflow_class": "py307.caught", + "workflow_command_id": "01M7TA28PGCF0Y2NYMJ5SKAF2S", + "memo": null, + "business_key": null, + "visibility_labels": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01M7TA28PGCF0Y2NYMJ5SKAF2S", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01M7TA28PGCF0Y2NYMJ5SKAF2R", + "payload_codec": "avro", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "source": "control_plane", + "context": { + "principal": { + "type": "auth:token", + "id": "legacy-token", + "label": "Admin" + } + }, + "caller_label": "Standalone Server", + "principal_type": "auth:token", + "principal_id": "legacy-token", + "principal_label": "Admin", + "auth_status": "authorized", + "auth_method": "token", + "request_method": "POST", + "request_path": "/api/workflows", + "request_fingerprint": "sha256:3c0cfd10f3b47621a23bf48ca6d5c4c30eef5510c7f8c53bc915ba405e304c8e", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-11-19T16:26:03.344000Z", + "applied_at": "2026-11-19T16:26:03.344000Z" + } + } + }, + { + "sequence": 2, + "event_type": "WorkflowStarted", + "timestamp": "2026-11-19T16:26:03.346000Z", + "principal": { + "type": "auth:token", + "id": "legacy-token", + "label": "Admin" + }, + "payload": { + "workflow_instance_id": "py307-caught-1795105562", + "workflow_run_id": "01M7TA28PGCF0Y2NYMJ5SKAF2R", + "workflow_type": "py307.caught", + "workflow_class": "py307.caught", + "workflow_command_id": "01M7TA28PGCF0Y2NYMJ5SKAF2S", + "execution_timeout_seconds": 3600, + "run_timeout_seconds": 600, + "execution_deadline_at": "2026-11-19T17:26:03.344Z", + "run_deadline_at": "2026-11-19T16:36:03.344Z", + "workflow_definition_fingerprint": "sha256:b6ef61c873dd7992ecea935b6da5da46e58eeb3f59d778941a9238c62da0775b", + "workflow_definition_fingerprint_source": "worker", + "declared_signals": [], + "declared_signal_contracts": [], + "declared_queries": [ + "caught" + ], + "declared_query_contracts": [ + { + "name": "caught", + "parameters": [] + } + ], + "declared_updates": [], + "declared_update_contracts": [], + "declared_update_validators": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "external:py307.caught", + "command": { + "id": "01M7TA28PGCF0Y2NYMJ5SKAF2S", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01M7TA28PGCF0Y2NYMJ5SKAF2R", + "payload_codec": "avro", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "source": "control_plane", + "context": { + "principal": { + "type": "auth:token", + "id": "legacy-token", + "label": "Admin" + } + }, + "caller_label": "Standalone Server", + "principal_type": "auth:token", + "principal_id": "legacy-token", + "principal_label": "Admin", + "auth_status": "authorized", + "auth_method": "token", + "request_method": "POST", + "request_path": "/api/workflows", + "request_fingerprint": "sha256:3c0cfd10f3b47621a23bf48ca6d5c4c30eef5510c7f8c53bc915ba405e304c8e", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-11-19T16:26:03.344000Z", + "applied_at": "2026-11-19T16:26:03.344000Z" + } + } + }, + { + "sequence": 3, + "event_type": "ActivityScheduled", + "timestamp": "2026-11-19T16:26:03.414000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA28RNH2S976DGTHY5XQV3", + "activity_type": "py307.fail", + "activity_class": "py307.fail", + "sequence": 1, + "activity": { + "id": "01M7TA28RNH2S976DGTHY5XQV3", + "idempotency_key": "01M7TA28RNH2S976DGTHY5XQV3", + "sequence": 1, + "type": "py307.fail", + "class": "py307.fail", + "local_activity": false, + "activity_type": "py307.fail", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBgAAAAAAAPg/AA==" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "pending", + "attempt_count": 0, + "closed_at": null, + "attempt_id": null, + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 0 + } + ], + "task": { + "id": "01M7TA28PJ36XM1BKGW1EEBY7Z", + "type": "workflow", + "status": "leased", + "attempt_count": 1, + "repair_count": 0, + "queue": "py307-workflows", + "connection": "sqlite", + "available_at": "2026-11-19T16:26:03.346000Z", + "leased_at": "2026-11-19T16:26:03.363000Z", + "lease_expires_at": "2026-11-19T16:27:03.363000Z", + "last_dispatch_attempt_at": "2026-11-19T16:26:03.346000Z", + "last_dispatched_at": "2026-11-19T16:26:03.346000Z", + "lease_owner": "py-worker-7541e585" + } + } + }, + { + "sequence": 4, + "event_type": "ActivityScheduled", + "timestamp": "2026-11-19T16:26:03.418000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA28RTG30JYJK03KWMMAX0", + "activity_type": "py307.fail", + "activity_class": "py307.fail", + "sequence": 2, + "activity": { + "id": "01M7TA28RTG30JYJK03KWMMAX0", + "idempotency_key": "01M7TA28RTG30JYJK03KWMMAX0", + "sequence": 2, + "type": "py307.fail", + "class": "py307.fail", + "local_activity": false, + "activity_type": "py307.fail", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "pending", + "attempt_count": 0, + "closed_at": null, + "attempt_id": null, + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 1 + } + ], + "task": { + "id": "01M7TA28PJ36XM1BKGW1EEBY7Z", + "type": "workflow", + "status": "leased", + "attempt_count": 1, + "repair_count": 0, + "queue": "py307-workflows", + "connection": "sqlite", + "available_at": "2026-11-19T16:26:03.346000Z", + "leased_at": "2026-11-19T16:26:03.363000Z", + "lease_expires_at": "2026-11-19T16:27:03.363000Z", + "last_dispatch_attempt_at": "2026-11-19T16:26:03.346000Z", + "last_dispatched_at": "2026-11-19T16:26:03.346000Z", + "lease_owner": "py-worker-7541e585" + } + } + }, + { + "sequence": 5, + "event_type": "ActivityScheduled", + "timestamp": "2026-11-19T16:26:03.418000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA28RTG30JYJK03KWMMAX3", + "activity_type": "py307.ok", + "activity_class": "py307.ok", + "sequence": 3, + "activity": { + "id": "01M7TA28RTG30JYJK03KWMMAX3", + "idempotency_key": "01M7TA28RTG30JYJK03KWMMAX3", + "sequence": 3, + "type": "py307.ok", + "class": "py307.ok", + "local_activity": false, + "activity_type": "py307.ok", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "pending", + "attempt_count": 0, + "closed_at": null, + "attempt_id": null, + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 2, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 2 + } + ], + "task": { + "id": "01M7TA28PJ36XM1BKGW1EEBY7Z", + "type": "workflow", + "status": "leased", + "attempt_count": 1, + "repair_count": 0, + "queue": "py307-workflows", + "connection": "sqlite", + "available_at": "2026-11-19T16:26:03.346000Z", + "leased_at": "2026-11-19T16:26:03.363000Z", + "lease_expires_at": "2026-11-19T16:27:03.363000Z", + "last_dispatch_attempt_at": "2026-11-19T16:26:03.346000Z", + "last_dispatched_at": "2026-11-19T16:26:03.346000Z", + "lease_owner": "py-worker-7541e585" + } + } + }, + { + "sequence": 6, + "event_type": "ActivityScheduled", + "timestamp": "2026-11-19T16:26:03.419000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA28RV50NSTJRKGCX0ZSWZ", + "activity_type": "py307.ok", + "activity_class": "py307.ok", + "sequence": 4, + "activity": { + "id": "01M7TA28RV50NSTJRKGCX0ZSWZ", + "idempotency_key": "01M7TA28RV50NSTJRKGCX0ZSWZ", + "sequence": 4, + "type": "py307.ok", + "class": "py307.ok", + "local_activity": false, + "activity_type": "py307.ok", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBgAAAAAAAOA/AA==" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "pending", + "attempt_count": 0, + "closed_at": null, + "attempt_id": null, + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 3, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 3 + } + ], + "task": { + "id": "01M7TA28PJ36XM1BKGW1EEBY7Z", + "type": "workflow", + "status": "leased", + "attempt_count": 1, + "repair_count": 0, + "queue": "py307-workflows", + "connection": "sqlite", + "available_at": "2026-11-19T16:26:03.346000Z", + "leased_at": "2026-11-19T16:26:03.363000Z", + "lease_expires_at": "2026-11-19T16:27:03.363000Z", + "last_dispatch_attempt_at": "2026-11-19T16:26:03.346000Z", + "last_dispatched_at": "2026-11-19T16:26:03.346000Z", + "lease_owner": "py-worker-7541e585" + } + } + }, + { + "sequence": 7, + "event_type": "ActivityStarted", + "timestamp": "2026-11-19T16:26:03.732000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA28RNH2S976DGTHY5XQV3", + "activity_type": "py307.fail", + "activity_class": "py307.fail", + "sequence": 1, + "activity": { + "id": "01M7TA28RNH2S976DGTHY5XQV3", + "idempotency_key": "01M7TA28RNH2S976DGTHY5XQV3", + "sequence": 1, + "type": "py307.fail", + "class": "py307.fail", + "local_activity": false, + "activity_type": "py307.fail", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBgAAAAAAAPg/AA==" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "running", + "attempt_count": 1, + "closed_at": null, + "attempt_id": "01M7TA292MTQHDAPQXJFXWC6AR", + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities", + "started_at": "2026-11-19T16:26:03.732Z" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 0 + } + ], + "activity_attempt_id": "01M7TA292MTQHDAPQXJFXWC6AR", + "attempt_number": 1, + "task": { + "id": "01M7TA28RQWRA89H9PRJH37XN3", + "type": "activity", + "status": "leased", + "repair_count": 0, + "available_at": "2026-11-19T16:26:03.415Z", + "leased_at": "2026-11-19T16:26:03.731Z", + "lease_owner": "py-worker-1fa82ca5", + "lease_expires_at": "2026-11-19T16:31:03.731Z", + "attempt_count": 1, + "queue": "py307-activities", + "connection": "sqlite" + } + } + }, + { + "sequence": 8, + "event_type": "ActivityStarted", + "timestamp": "2026-11-19T16:26:03.776000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA28RTG30JYJK03KWMMAX0", + "activity_type": "py307.fail", + "activity_class": "py307.fail", + "sequence": 2, + "activity": { + "id": "01M7TA28RTG30JYJK03KWMMAX0", + "idempotency_key": "01M7TA28RTG30JYJK03KWMMAX0", + "sequence": 2, + "type": "py307.fail", + "class": "py307.fail", + "local_activity": false, + "activity_type": "py307.fail", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "running", + "attempt_count": 1, + "closed_at": null, + "attempt_id": "01M7TA2940W1AY6NJFYRRF6DPS", + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities", + "started_at": "2026-11-19T16:26:03.776Z" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 1 + } + ], + "activity_attempt_id": "01M7TA2940W1AY6NJFYRRF6DPS", + "attempt_number": 1, + "task": { + "id": "01M7TA28RTG30JYJK03KWMMAX2", + "type": "activity", + "status": "leased", + "repair_count": 0, + "available_at": "2026-11-19T16:26:03.418Z", + "leased_at": "2026-11-19T16:26:03.776Z", + "lease_owner": "py-worker-1fa82ca5", + "lease_expires_at": "2026-11-19T16:31:03.776Z", + "attempt_count": 1, + "queue": "py307-activities", + "connection": "sqlite" + } + } + }, + { + "sequence": 9, + "event_type": "ActivityFailed", + "timestamp": "2026-11-19T16:26:03.820000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA28RTG30JYJK03KWMMAX0", + "activity_type": "py307.fail", + "activity_class": "py307.fail", + "sequence": 2, + "activity": { + "id": "01M7TA28RTG30JYJK03KWMMAX0", + "idempotency_key": "01M7TA28RTG30JYJK03KWMMAX0", + "sequence": 2, + "type": "py307.fail", + "class": "py307.fail", + "local_activity": false, + "activity_type": "py307.fail", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "failed", + "attempt_count": 1, + "closed_at": "2026-11-19T16:26:03.820Z", + "attempt_id": "01M7TA2940W1AY6NJFYRRF6DPS", + "exception": { + "code": 0, + "type": "RuntimeError", + "message": "py307 failed after 0" + }, + "payload_codec": "avro", + "queue": "py307-activities", + "started_at": "2026-11-19T16:26:03.776Z" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 1 + } + ], + "activity_attempt_id": "01M7TA2940W1AY6NJFYRRF6DPS", + "attempt_number": 1, + "failure_id": "01M7TA295CHSV4G450F0756BQB", + "failure_category": "activity", + "non_retryable": false, + "exception_type": "RuntimeError", + "exception_class": "builtins.RuntimeError", + "message": "py307 failed after 0", + "code": 0, + "exception": { + "type": "RuntimeError", + "message": "py307 failed after 0", + "code": 0 + }, + "task": { + "id": "01M7TA28RTG30JYJK03KWMMAX2", + "type": "activity", + "status": "leased", + "available_at": "2026-11-19T16:26:03.418Z", + "leased_at": "2026-11-19T16:26:03.776Z", + "lease_owner": "py-worker-1fa82ca5", + "lease_expires_at": "2026-11-19T16:31:03.791Z", + "attempt_count": 1, + "repair_count": 0, + "queue": "py307-activities", + "connection": "sqlite" + } + } + }, + { + "sequence": 10, + "event_type": "ActivityStarted", + "timestamp": "2026-11-19T16:26:03.842000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA28RTG30JYJK03KWMMAX3", + "activity_type": "py307.ok", + "activity_class": "py307.ok", + "sequence": 3, + "activity": { + "id": "01M7TA28RTG30JYJK03KWMMAX3", + "idempotency_key": "01M7TA28RTG30JYJK03KWMMAX3", + "sequence": 3, + "type": "py307.ok", + "class": "py307.ok", + "local_activity": false, + "activity_type": "py307.ok", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "running", + "attempt_count": 1, + "closed_at": null, + "attempt_id": "01M7TA2962MSYN39HYTMH5HYDR", + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities", + "started_at": "2026-11-19T16:26:03.842Z" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 2, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 2 + } + ], + "activity_attempt_id": "01M7TA2962MSYN39HYTMH5HYDR", + "attempt_number": 1, + "task": { + "id": "01M7TA28RTG30JYJK03KWMMAX5", + "type": "activity", + "status": "leased", + "repair_count": 0, + "available_at": "2026-11-19T16:26:03.418Z", + "leased_at": "2026-11-19T16:26:03.841Z", + "lease_owner": "py-worker-1fa82ca5", + "lease_expires_at": "2026-11-19T16:31:03.841Z", + "attempt_count": 1, + "queue": "py307-activities", + "connection": "sqlite" + } + } + }, + { + "sequence": 11, + "event_type": "ActivityCompleted", + "timestamp": "2026-11-19T16:26:03.882000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA28RTG30JYJK03KWMMAX3", + "activity_type": "py307.ok", + "activity_class": "py307.ok", + "sequence": 3, + "activity": { + "id": "01M7TA28RTG30JYJK03KWMMAX3", + "idempotency_key": "01M7TA28RTG30JYJK03KWMMAX3", + "sequence": 3, + "type": "py307.ok", + "class": "py307.ok", + "local_activity": false, + "activity_type": "py307.ok", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "completed", + "attempt_count": 1, + "closed_at": "2026-11-19T16:26:03.882Z", + "attempt_id": "01M7TA2962MSYN39HYTMH5HYDR", + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities", + "started_at": "2026-11-19T16:26:03.842Z" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 2, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 2 + } + ], + "activity_attempt_id": "01M7TA2962MSYN39HYTMH5HYDR", + "attempt_number": 1, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoEb2s=" + }, + "payload_codec": "avro", + "task": { + "id": "01M7TA28RTG30JYJK03KWMMAX5", + "type": "activity", + "status": "leased", + "repair_count": 0, + "available_at": "2026-11-19T16:26:03.418Z", + "leased_at": "2026-11-19T16:26:03.841Z", + "lease_owner": "py-worker-1fa82ca5", + "lease_expires_at": "2026-11-19T16:31:03.861Z", + "attempt_count": 1, + "queue": "py307-activities", + "connection": "sqlite" + } + } + }, + { + "sequence": 12, + "event_type": "ActivityStarted", + "timestamp": "2026-11-19T16:26:03.903000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA28RV50NSTJRKGCX0ZSWZ", + "activity_type": "py307.ok", + "activity_class": "py307.ok", + "sequence": 4, + "activity": { + "id": "01M7TA28RV50NSTJRKGCX0ZSWZ", + "idempotency_key": "01M7TA28RV50NSTJRKGCX0ZSWZ", + "sequence": 4, + "type": "py307.ok", + "class": "py307.ok", + "local_activity": false, + "activity_type": "py307.ok", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBgAAAAAAAOA/AA==" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "running", + "attempt_count": 1, + "closed_at": null, + "attempt_id": "01M7TA297Z31QD8QN22Y56NG92", + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities", + "started_at": "2026-11-19T16:26:03.903Z" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 3, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 3 + } + ], + "activity_attempt_id": "01M7TA297Z31QD8QN22Y56NG92", + "attempt_number": 1, + "task": { + "id": "01M7TA28RV50NSTJRKGCX0ZSX1", + "type": "activity", + "status": "leased", + "repair_count": 0, + "available_at": "2026-11-19T16:26:03.419Z", + "leased_at": "2026-11-19T16:26:03.902Z", + "lease_owner": "py-worker-1fa82ca5", + "lease_expires_at": "2026-11-19T16:31:03.902Z", + "attempt_count": 1, + "queue": "py307-activities", + "connection": "sqlite" + } + } + }, + { + "sequence": 13, + "event_type": "ActivityCompleted", + "timestamp": "2026-11-19T16:26:04.501000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA28RV50NSTJRKGCX0ZSWZ", + "activity_type": "py307.ok", + "activity_class": "py307.ok", + "sequence": 4, + "activity": { + "id": "01M7TA28RV50NSTJRKGCX0ZSWZ", + "idempotency_key": "01M7TA28RV50NSTJRKGCX0ZSWZ", + "sequence": 4, + "type": "py307.ok", + "class": "py307.ok", + "local_activity": false, + "activity_type": "py307.ok", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBgAAAAAAAOA/AA==" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "completed", + "attempt_count": 1, + "closed_at": "2026-11-19T16:26:04.501Z", + "attempt_id": "01M7TA297Z31QD8QN22Y56NG92", + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities", + "started_at": "2026-11-19T16:26:03.903Z" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 3, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 3 + } + ], + "activity_attempt_id": "01M7TA297Z31QD8QN22Y56NG92", + "attempt_number": 1, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoEb2s=" + }, + "payload_codec": "avro", + "task": { + "id": "01M7TA28RV50NSTJRKGCX0ZSX1", + "type": "activity", + "status": "leased", + "repair_count": 0, + "available_at": "2026-11-19T16:26:03.419Z", + "leased_at": "2026-11-19T16:26:03.902Z", + "lease_owner": "py-worker-1fa82ca5", + "lease_expires_at": "2026-11-19T16:31:03.965Z", + "attempt_count": 1, + "queue": "py307-activities", + "connection": "sqlite" + } + } + }, + { + "sequence": 14, + "event_type": "ActivityFailed", + "timestamp": "2026-11-19T16:26:05.279000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA28RNH2S976DGTHY5XQV3", + "activity_type": "py307.fail", + "activity_class": "py307.fail", + "sequence": 1, + "activity": { + "id": "01M7TA28RNH2S976DGTHY5XQV3", + "idempotency_key": "01M7TA28RNH2S976DGTHY5XQV3", + "sequence": 1, + "type": "py307.fail", + "class": "py307.fail", + "local_activity": false, + "activity_type": "py307.fail", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBgAAAAAAAPg/AA==" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "failed", + "attempt_count": 1, + "closed_at": "2026-11-19T16:26:05.278Z", + "attempt_id": "01M7TA292MTQHDAPQXJFXWC6AR", + "exception": { + "code": 0, + "type": "RuntimeError", + "message": "py307 failed after 1.5" + }, + "payload_codec": "avro", + "queue": "py307-activities", + "started_at": "2026-11-19T16:26:03.732Z" + }, + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:4", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 4, + "parallel_group_index": 0 + } + ], + "activity_attempt_id": "01M7TA292MTQHDAPQXJFXWC6AR", + "attempt_number": 1, + "failure_id": "01M7TA2AJY5XYXFMX8N37GDBN8", + "failure_category": "activity", + "non_retryable": false, + "exception_type": "RuntimeError", + "exception_class": "builtins.RuntimeError", + "message": "py307 failed after 1.5", + "code": 0, + "exception": { + "type": "RuntimeError", + "message": "py307 failed after 1.5", + "code": 0 + }, + "task": { + "id": "01M7TA28RQWRA89H9PRJH37XN3", + "type": "activity", + "status": "leased", + "available_at": "2026-11-19T16:26:03.415Z", + "leased_at": "2026-11-19T16:26:03.731Z", + "lease_owner": "py-worker-1fa82ca5", + "lease_expires_at": "2026-11-19T16:31:03.760Z", + "attempt_count": 1, + "repair_count": 0, + "queue": "py307-activities", + "connection": "sqlite" + } + } + }, + { + "sequence": 15, + "event_type": "ActivityScheduled", + "timestamp": "2026-11-19T16:26:06.659000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA2BY3Y2NKYZT41M8FRC4T", + "activity_type": "py307.compensate", + "activity_class": "py307.compensate", + "sequence": 5, + "activity": { + "id": "01M7TA2BY3Y2NKYZT41M8FRC4T", + "idempotency_key": "01M7TA2BY3Y2NKYZT41M8FRC4T", + "sequence": 5, + "type": "py307.compensate", + "class": "py307.compensate", + "local_activity": false, + "activity_type": "py307.compensate", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCChhweTMwNy5mYWlsIzEA" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "pending", + "attempt_count": 0, + "closed_at": null, + "attempt_id": null, + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities" + }, + "task": { + "id": "01M7TA295DM570FQ0HKAAAXZ7B", + "type": "workflow", + "status": "leased", + "attempt_count": 1, + "repair_count": 0, + "queue": "py307-workflows", + "connection": "sqlite", + "available_at": "2026-11-19T16:26:03.821000Z", + "leased_at": "2026-11-19T16:26:06.618000Z", + "lease_expires_at": "2026-11-19T16:27:06.618000Z", + "last_dispatch_attempt_at": "2026-11-19T16:26:03.821000Z", + "last_dispatched_at": "2026-11-19T16:26:03.821000Z", + "lease_owner": "py-worker-08341312" + } + } + }, + { + "sequence": 16, + "event_type": "ActivityStarted", + "timestamp": "2026-11-19T16:26:06.704000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA2BY3Y2NKYZT41M8FRC4T", + "activity_type": "py307.compensate", + "activity_class": "py307.compensate", + "sequence": 5, + "activity": { + "id": "01M7TA2BY3Y2NKYZT41M8FRC4T", + "idempotency_key": "01M7TA2BY3Y2NKYZT41M8FRC4T", + "sequence": 5, + "type": "py307.compensate", + "class": "py307.compensate", + "local_activity": false, + "activity_type": "py307.compensate", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCChhweTMwNy5mYWlsIzEA" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "running", + "attempt_count": 1, + "closed_at": null, + "attempt_id": "01M7TA2BZGW6PVX5D4FQB359S6", + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities", + "started_at": "2026-11-19T16:26:06.704Z" + }, + "activity_attempt_id": "01M7TA2BZGW6PVX5D4FQB359S6", + "attempt_number": 1, + "task": { + "id": "01M7TA2BY3Y2NKYZT41M8FRC4W", + "type": "activity", + "status": "leased", + "repair_count": 0, + "available_at": "2026-11-19T16:26:06.659Z", + "leased_at": "2026-11-19T16:26:06.704Z", + "lease_owner": "py-worker-1fa82ca5", + "lease_expires_at": "2026-11-19T16:31:06.704Z", + "attempt_count": 1, + "queue": "py307-activities", + "connection": "sqlite" + } + } + }, + { + "sequence": 17, + "event_type": "ActivityCompleted", + "timestamp": "2026-11-19T16:26:06.753000Z", + "principal": null, + "payload": { + "activity_execution_id": "01M7TA2BY3Y2NKYZT41M8FRC4T", + "activity_type": "py307.compensate", + "activity_class": "py307.compensate", + "sequence": 5, + "activity": { + "id": "01M7TA2BY3Y2NKYZT41M8FRC4T", + "idempotency_key": "01M7TA2BY3Y2NKYZT41M8FRC4T", + "sequence": 5, + "type": "py307.compensate", + "class": "py307.compensate", + "local_activity": false, + "activity_type": "py307.compensate", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCChhweTMwNy5mYWlsIzEA" + }, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "non_retryable_error_types": [], + "start_to_close_timeout": 60, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null + }, + "status": "completed", + "attempt_count": 1, + "closed_at": "2026-11-19T16:26:06.753Z", + "attempt_id": "01M7TA2BZGW6PVX5D4FQB359S6", + "exception": null, + "payload_codec": "avro", + "queue": "py307-activities", + "started_at": "2026-11-19T16:26:06.704Z" + }, + "activity_attempt_id": "01M7TA2BZGW6PVX5D4FQB359S6", + "attempt_number": 1, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwomdW5kb25lOnB5MzA3LmZhaWwjMQ==" + }, + "payload_codec": "avro", + "task": { + "id": "01M7TA2BY3Y2NKYZT41M8FRC4W", + "type": "activity", + "status": "leased", + "repair_count": 0, + "available_at": "2026-11-19T16:26:06.659Z", + "leased_at": "2026-11-19T16:26:06.704Z", + "lease_owner": "py-worker-1fa82ca5", + "lease_expires_at": "2026-11-19T16:31:06.725Z", + "attempt_count": 1, + "queue": "py307-activities", + "connection": "sqlite" + } + } + }, + { + "sequence": 18, + "event_type": "WorkflowCompleted", + "timestamp": "2026-11-19T16:26:06.842000Z", + "principal": { + "type": "auth:token", + "id": "legacy-token", + "label": "Admin" + }, + "payload": { + "output": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwomdW5kb25lOnB5MzA3LmZhaWwjMQ==" + }, + "payload_codec": "avro", + "task": { + "id": "01M7TA2C12G53MD54RR2VZ70V6", + "type": "workflow", + "status": "leased", + "attempt_count": 1, + "repair_count": 0, + "queue": "py307-workflows", + "connection": "sqlite", + "available_at": "2026-11-19T16:26:06.754000Z", + "leased_at": "2026-11-19T16:26:06.770000Z", + "lease_expires_at": "2026-11-19T16:27:06.770000Z", + "last_dispatch_attempt_at": "2026-11-19T16:26:06.754000Z", + "last_dispatched_at": "2026-11-19T16:26:06.754000Z", + "lease_owner": "py-worker-08341312" + }, + "command": { + "type": "complete_workflow", + "source": "worker_protocol", + "principal_type": "auth:token", + "principal_id": "legacy-token", + "principal_label": "Admin" + } + } + } + ], + "expected": { + "query_result": "py307.fail#1" + } +} diff --git a/tests/test_business_replay_clock_parallel.py b/tests/test_business_replay_clock_parallel.py index cd0f6ce0..3821a1f1 100644 --- a/tests/test_business_replay_clock_parallel.py +++ b/tests/test_business_replay_clock_parallel.py @@ -13,7 +13,13 @@ from durable_workflow import serializer, workflow from durable_workflow.errors import ActivityFailed, NonDeterministicReplayError, TypedValueError from durable_workflow.replay_clock_math import canonical_milliseconds, from_canonical_milliseconds -from durable_workflow.workflow import WorkflowContext, query_state, replay +from durable_workflow.workflow import ( + CompleteWorkflow, + RecordSideEffect, + WorkflowContext, + query_state, + replay, +) from tests.business_clock_workflows import clock_text from tests.test_business_replay_clock import result @@ -74,12 +80,37 @@ def test_canonical_v1_authored_comparison_and_selected_failure_vectors(case: dic value = result(history(case["times"], order=(1, 0)), ParallelProbe) expected = datetime.fromisoformat(case["times"][case["success_member"]].replace("Z", "+00:00")) assert value["clock"] == expected.isoformat(timespec="microseconds") - failure = result(history(case["times"], failures={0, 1}, order=(1, 0)), ParallelProbe) + rows = history(case["times"], failures={0, 1}, order=(1, 0)) + failure = result(closed(rows), ParallelProbe) assert failure["failure"] == ("group.0", "failure-0") time = case["failure_time"] assert failure["clock"] == (datetime.fromisoformat(time.replace("Z", "+00:00")).isoformat( timespec="microseconds", ) if time is not None else None) + now = frontier_result(rows) + assert now["failure"] == ("group.1", "failure-1") + assert now["clock"] == canonical_text(case["times"][1]) + + +def closed(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + """The history as a released worker left it: the run closed after the group.""" + return [*rows, {"event_type": "WorkflowCompleted", "recorded_at": "2099-01-01T00:00:00Z", "payload": {}}] + + +def frontier_result(rows: list[dict[str, Any]]) -> Any: + """A task reaching a settled group whose first recorded failure is not the + lowest position raises that failure and records it (sdk-python#307).""" + outcome = replay(ParallelProbe, rows, [], payload_codec="avro") + try: + assert [type(command) for command in outcome.commands] == [RecordSideEffect, CompleteWorkflow] + return outcome.commands[1].result + finally: + outcome.close() + + +def canonical_text(time: str) -> str | None: + instant = from_canonical_milliseconds(canonical_milliseconds(datetime.fromisoformat(time.replace("Z", "+00:00")))) + return instant.isoformat(timespec="microseconds") if instant is not None else None def test_reconstruction_overflow_refuses_time_without_changing_the_selected_failure() -> None: @@ -195,9 +226,13 @@ def test_caught_failure_ignores_successful_sibling_time_and_reconstructs_canonic def test_multiple_failures_preserve_legacy_winner_and_refuse_a_different_canonical_winner( times: list[str], expected: str | None, ) -> None: - value = result(history(times, failures={0, 1}, order=(1, 0)), ParallelProbe) + rows = history(times, failures={0, 1}, order=(1, 0)) + value = result(closed(rows), ParallelProbe) assert value["failure"] == ("group.0", "failure-0") assert value["clock"] == expected + now = frontier_result(rows) + assert now["failure"] == ("group.1", "failure-1") + assert now["clock"] == canonical_text(times[1]) @pytest.mark.parametrize("order", list(permutations(range(3)))) diff --git a/tests/test_parallel_completion_order_replay.py b/tests/test_parallel_completion_order_replay.py index 7fd3f363..a3ea8370 100644 --- a/tests/test_parallel_completion_order_replay.py +++ b/tests/test_parallel_completion_order_replay.py @@ -66,6 +66,23 @@ def _completed_result(workflow_type: type, history: list[dict[str, Any]]) -> Any return command.result +def _failure_results(workflow_type: type, history: list[dict[str, Any]]) -> tuple[Any, Any]: + """The result a released worker recorded, and the one a task picks now. + + A history whose run already closed replays the failure released workers + raised: the lowest workflow position. A task reaching the settled group + for the first time raises the first recorded failure, as Workflow's + embedded engine does (sdk-python#307), recording the member it raised + when the two rules differ. + """ + closed = [*history, {"event_type": "WorkflowCompleted", "payload": {}}] + legacy = _completed_result(workflow_type, closed) + outcome = replay(workflow_type, history, [], payload_codec=serializer.AVRO_CODEC) + assert isinstance(outcome.commands[-1], CompleteWorkflow) + assert len(outcome.commands) in (1, 2) + return legacy, outcome.commands[-1].result + + @workflow.defn(name="parallel-completion-order-activities") class ParallelCompletionOrderActivities: def run(self, ctx: WorkflowContext): # type: ignore[no-untyped-def] @@ -357,13 +374,18 @@ def test_activity_failure_selection_follows_workflow_position(completion_order: ), ] - assert _completed_result( + legacy, now = _failure_results( ParallelCompletionOrderActivityFailures, _history_in_completion_order(events, completion_order), - ) == { + ) + assert legacy == { "activity_type": "activity-fails-first", "message": "first positional activity failure", } + first = 1 if completion_order.index(1) < completion_order.index(2) else 2 + assert now == ({"activity_type": "activity-fails-first", "message": "first positional activity failure"} + if first == 1 else + {"activity_type": "activity-fails-second", "message": "second positional activity failure"}) @pytest.mark.parametrize("completion_order", THREE_SIBLING_ORDERS) @@ -395,13 +417,18 @@ def test_child_failure_selection_follows_workflow_position(completion_order: tup ), ] - assert _completed_result( + legacy, now = _failure_results( ParallelCompletionOrderChildFailures, _history_in_completion_order(events, completion_order), - ) == { + ) + assert legacy == { "child_workflow_type": "child-fails-first", "message": "first positional child failure", } + first = 1 if completion_order.index(1) < completion_order.index(2) else 2 + assert now == ({"child_workflow_type": "child-fails-first", "message": "first positional child failure"} + if first == 1 else + {"child_workflow_type": "child-fails-second", "message": "second positional child failure"}) @pytest.mark.parametrize("completion_order", THREE_SIBLING_ORDERS) @@ -426,13 +453,18 @@ def test_mixed_failure_selection_follows_workflow_position(completion_order: tup _terminal_event("TimerFired", "mixed", 3, 2, timer_kind="sleep"), ] - assert _completed_result( + legacy, now = _failure_results( ParallelCompletionOrderMixedFailures, _history_in_completion_order(events, completion_order), - ) == { + ) + assert legacy == { "failure_type": "ChildWorkflowFailed", "message": "first positional mixed failure", } + first = 0 if completion_order.index(0) < completion_order.index(1) else 1 + assert now == ({"failure_type": "ChildWorkflowFailed", "message": "first positional mixed failure"} + if first == 0 else + {"failure_type": "ActivityFailed", "message": "second positional mixed failure"}) @pytest.mark.parametrize("completion_order", [(0, 1), (1, 0)]) @@ -451,10 +483,9 @@ def test_repeated_identical_failures_stay_bound_to_yielded_positions( for index in range(2) ] - assert ( - _completed_result( - ParallelCompletionOrderRepeatedFailures, - _history_in_completion_order(events, completion_order), - ) - == "position-0-failed" + legacy, now = _failure_results( + ParallelCompletionOrderRepeatedFailures, + _history_in_completion_order(events, completion_order), ) + assert legacy == "position-0-failed" + assert now == f"position-{completion_order[0]}-failed" diff --git a/tests/test_parallel_group_policy.py b/tests/test_parallel_group_policy.py index c8413d76..cc297b3d 100644 --- a/tests/test_parallel_group_policy.py +++ b/tests/test_parallel_group_policy.py @@ -62,15 +62,19 @@ def test_absent_policy_raises_the_first_failure_while_a_sibling_is_open() -> Non assert query_state(ParallelFailurePolicyWorkflow, history, [], "caught") == "fast-second" -def test_absent_policy_preserves_positional_failure_after_a_later_sibling_settles() -> None: +def test_absent_policy_selection_after_a_later_sibling_settles() -> None: + """Released workers raised the lower position once both members settled. + A closed history keeps that; a task reaching the settled group now raises + the first recorded failure and records it (sdk-python#307).""" history = json.loads(FIXTURE.read_text())["history"] first = dict(history[1]["payload"]) first["message"] = "lower position failed later" history.append({"event_type": "ActivityFailed", "recorded_at": "2026-01-01T00:00:03Z", "payload": first}) + closed = replay(ParallelFailurePolicyWorkflow, [*history, {"event_type": "WorkflowCompleted", "payload": {}}], []) + assert closed.commands == [CompleteWorkflow("slow-first")] outcome = replay(ParallelFailurePolicyWorkflow, history, []) - assert len(outcome.commands) == 1 - assert isinstance(outcome.commands[0], CompleteWorkflow) - assert outcome.commands[0].result == "slow-first" + assert [type(command) for command in outcome.commands] == [RecordSideEffect, CompleteWorkflow] + assert outcome.commands[1] == CompleteWorkflow("fast-second") def explicit_history(policy: str = "propagate") -> list[dict]: @@ -111,7 +115,8 @@ def test_enabling_draft_support_does_not_change_absent_policy_history() -> None: def test_enabling_draft_support_keeps_the_legacy_failure_clock_gap() -> None: fixture = json.loads(FIXTURE.with_name("business-clock-parallel-failure-selection-gap.json").read_text()) - outcome = draft_replay(fixture["history"], ParallelFailureBusinessClockWorkflow) + closed = [*fixture["history"], {"event_type": "WorkflowCompleted", "payload": {}}] + outcome = draft_replay(closed, ParallelFailureBusinessClockWorkflow) assert outcome.commands[0].result == fixture["expected"]["command_sequence"][0]["result"] diff --git a/tests/test_parallel_member_failure_open_siblings.py b/tests/test_parallel_member_failure_open_siblings.py index 484e30d5..c3cf2101 100644 --- a/tests/test_parallel_member_failure_open_siblings.py +++ b/tests/test_parallel_member_failure_open_siblings.py @@ -277,22 +277,87 @@ def test_a_late_sibling_never_takes_the_compensation_result(late_first: bool) -> assert outcome.commands == [CompleteWorkflow({"caught": "member-0", "compensated": "undone"})] -def test_a_group_that_settled_before_the_workflow_moved_on_keeps_positional_selection() -> None: - """A history recorded while the worker waited for every member: the - lower-position failure is the one raised, as before, and no decision is - recorded because the task saw every outcome.""" - history = [ +def _settled_out_of_order() -> list[dict[str, Any]]: + """Every member settled before a task reached the group. Member 2 failed + first; member 0, the lower position, failed later.""" + return [ *_scheduled(), _member("ActivityFailed", 2), _member("ActivityCompleted", 1), _member("ActivityFailed", 0), _member("ActivityCompleted", 3), ] - commands = _wire(history) - assert [(command["type"], command.get("activity_type")) for command in commands] == [ - ("schedule_activity", "compensate"), + + +def test_a_settled_group_raises_its_first_recorded_failure_and_records_it() -> None: + """sdk-python#307: Workflow's embedded engine raises the earliest recorded + failure, not the lowest position. Released workers raised member 0 here + and wrote no decision, so this task records the member it raised.""" + history = _settled_out_of_order() + outcome = replay(AllFailureOpenSiblings, history, [], payload_codec=serializer.AVRO_CODEC) + assert outcome.commands == [ + RecordSideEffect({PARALLEL_FAILURE_DECISION: { + "parallel_group_id": f"parallel-activities:1:{SIZE}", "parallel_group_base_sequence": 1, + "parallel_group_size": SIZE, "member_sequence": 3, + }}), + ScheduleActivity("compensate", ["member-2"]), + ] + assert query_state(AllFailureOpenSiblings, history, [], "caught") == "member-2" + uncaught = replay(AllFailureUncaught, history, [], payload_codec=serializer.AVRO_CODEC) + assert [type(command) for command in uncaught.commands] == [RecordSideEffect, FailWorkflow] + assert "member-2" in str(cast(FailWorkflow, uncaught.commands[1]).message) + + +def test_a_settled_group_replays_the_recorded_first_failure_at_every_later_task() -> None: + history = [ + *_settled_out_of_order(), + _decision(2), + _compensation("ActivityScheduled"), + _compensation("ActivityCompleted", "undone"), ] + for end in range(len(history) - 1, len(history) + 1): + expected = [] if end < len(history) else [ + CompleteWorkflow({"caught": "member-2", "compensated": "undone"}), + ] + outcome = replay(AllFailureOpenSiblings, history[:end], [], payload_codec=serializer.AVRO_CODEC) + assert outcome.commands == expected, end + assert query_state(AllFailureOpenSiblings, history[:end], [], "caught") == "member-2", end + + +@pytest.mark.parametrize("order", [(0, 2), (2,), (1, 3)]) +def test_a_settled_group_whose_first_failure_has_the_lowest_position_records_nothing( + order: tuple[int, ...], +) -> None: + """Both rules raise the same member, so the commands are exactly what + released workers sent.""" + history = [*_scheduled(), *(_member("ActivityFailed", index) for index in order), + *(_member("ActivityCompleted", index) for index in range(SIZE) if index not in order)] + first = order[0] + assert _wire(history) == _wire_scheduled_compensation(f"member-{first}") + assert query_state(AllFailureOpenSiblings, history, [], "caught") == f"member-{first}" + + +def _wire_scheduled_compensation(caught: str) -> list[dict[str, Any]]: + return commands_to_server_commands( + [ScheduleActivity("compensate", [caught])], "queue", payload_codec=serializer.AVRO_CODEC, + ) + + +def test_a_released_worker_history_of_a_settled_group_keeps_positional_selection() -> None: + """Recorded by 2.19.1 and earlier: the task saw every member settled, + raised member 0 by position and moved on with no decision. It replays + exactly as recorded.""" + history = [*_settled_out_of_order(), _compensation("ActivityScheduled", sequence=SIZE + 1)] + assert replay(AllFailureOpenSiblings, history, [], payload_codec=serializer.AVRO_CODEC).commands == [] assert query_state(AllFailureOpenSiblings, history, [], "caught") == "member-0" + history.append(_compensation("ActivityCompleted", "undone", sequence=SIZE + 1)) + assert replay(AllFailureOpenSiblings, history, [], payload_codec=serializer.AVRO_CODEC).commands == [ + CompleteWorkflow({"caught": "member-0", "compensated": "undone"}), + ] + closed = [*_settled_out_of_order(), {"event_type": "WorkflowFailed", "payload": {"message": "member-0 failed"}}] + outcome = replay(AllFailureUncaught, closed, [], payload_codec=serializer.AVRO_CODEC) + assert [type(command) for command in outcome.commands] == [FailWorkflow] + assert "member-0" in str(cast(FailWorkflow, outcome.commands[0]).message) def test_a_history_from_another_runtime_raises_its_first_recorded_failure() -> None: @@ -344,3 +409,43 @@ def run(self, ctx: WorkflowContext): # type: ignore[no-untyped-def] return (yield ctx.schedule_activity("py338.compensate", [failure.activity_type], start_to_close_timeout=60)) return "no failure" + + +# Reproducer for the native Server histories in +# tests/fixtures/replay_regressions/settled-group-*.json (sdk-python#307). +# Member 1 fails at once and member 0 fails later; the others complete. The +# workflow worker was stopped while they ran, so the woken task saw the whole +# group settled. Published 2.19.1 raised member 0 by position; this version +# raises member 1, whose failure was recorded first. +PY307_ACTIVITIES = "py307-activities" + + +def _py307_group(ctx: WorkflowContext) -> list[Any]: + once = ActivityRetryPolicy(max_attempts=1) + return [ + ctx.schedule_activity("py307.fail", [1.5], queue=PY307_ACTIVITIES, start_to_close_timeout=60, + retry_policy=once), + ctx.schedule_activity("py307.fail", [0], queue=PY307_ACTIVITIES, start_to_close_timeout=60, + retry_policy=once), + ctx.schedule_activity("py307.ok", [0], queue=PY307_ACTIVITIES, start_to_close_timeout=60), + ctx.schedule_activity("py307.ok", [0.5], queue=PY307_ACTIVITIES, start_to_close_timeout=60), + ] + + +@workflow.defn(name="py307.caught") +class NativeSettledGroupCaught: + def __init__(self) -> None: + self.caught: str | None = None + + @workflow.query("caught") + def caught_member(self) -> str | None: + return self.caught + + def run(self, ctx: WorkflowContext): # type: ignore[no-untyped-def] + try: + yield _py307_group(ctx) + except ActivityFailed as failure: + self.caught = f"{failure.activity_type}#{failure.step_sequence}" + return (yield ctx.schedule_activity("py307.compensate", [self.caught], queue=PY307_ACTIVITIES, + start_to_close_timeout=60)) + return "no failure" diff --git a/tests/test_replay_regression_corpus.py b/tests/test_replay_regression_corpus.py index 965fb1c1..1a8606e2 100644 --- a/tests/test_replay_regression_corpus.py +++ b/tests/test_replay_regression_corpus.py @@ -52,6 +52,7 @@ from tests.test_parallel_group_policy import ParallelFailurePolicyWorkflow from tests.test_parallel_member_failure_open_siblings import ( NativeCaughtMemberFailure, + NativeSettledGroupCaught, NativeUncaughtMemberFailure, ) from tests.test_patch_history import ( @@ -479,6 +480,7 @@ def run(self, ctx: WorkflowContext, message: str) -> None: SelectionLoserAtCloseWorkflow, AllFailureClosedWorkflow, NativeCaughtMemberFailure, + NativeSettledGroupCaught, NativeUncaughtMemberFailure, UpdateSignalConditionTimerWorkflow, WorkflowStreamAuthorWorkflow, @@ -502,13 +504,17 @@ def run(self, ctx: WorkflowContext, message: str) -> None: WORKFLOW_TYPES["tests.replay.parallel-failure-policy"] = ParallelFailurePolicyWorkflow -# Immutable evidence whose recorded expectation was the sdk-python#338 defect: -# replay sent an already-scheduled group again, which Server refuses. The files -# stay unchanged. Each one's history replays under the named fixture instead, -# which cannot use ``supersedes`` because the protocol version is the same. +# Immutable evidence whose recorded expectation no longer holds. The files stay +# unchanged. Each one's history replays under the named fixture instead, which +# cannot use ``supersedes`` because the protocol version is the same. +# - sdk-python#338: replay sent an already-scheduled group again, which Server +# refuses. +# - sdk-python#307: a task reaching a settled group raised the lowest-position +# failure instead of the first recorded one. DEFECT_EVIDENCE = { "legacy-parallel-failure-remains-pending": "absent-policy-failure-raises-with-open-sibling", "run-closed-cancellation-not-a-group-outcome": "run-closed-all-failure-keeps-the-handled-failure", + "business-clock-parallel-failure-selection-gap": "settled-group-raises-first-recorded-failure", } diff --git a/tests/test_run_closed_settlement_replay.py b/tests/test_run_closed_settlement_replay.py index 5be31c3f..4001220b 100644 --- a/tests/test_run_closed_settlement_replay.py +++ b/tests/test_run_closed_settlement_replay.py @@ -623,9 +623,13 @@ def test_cancellation_at_close_never_replaces_the_failure_the_workflow_handled() def test_ordinary_cancellation_is_still_an_activity_outcome() -> None: """Only ``cancellation_reason: run_closed`` is skipped.""" - history = _all_failure_history(cancelled_at_close=True)[:-1] - del history[-1]["payload"]["cancellation_reason"] + history = _all_failure_history(cancelled_at_close=True) + del history[-2]["payload"]["cancellation_reason"] + history[-1]["payload"]["output"] = serializer.envelope("caught: activity failed")["blob"] + # The run closed after both members settled, as a released worker that + # chose the lower position left it. A skipped cancellation would leave + # member 1 open and make replay raise "boom" instead. outcome = replay(AllFailureClosedWorkflow, history, []) assert len(outcome.commands) == 1 From 8e541ee69b54bb200f85a429d934c68e7318fec5 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:11:52 +0000 Subject: [PATCH 2/3] Gate first-recorded-failure selection on a per-run marker (#307) Review of the previous head showed that deciding from history alone is unsound. A released worker could catch a settled group's lowest-position failure and finish the task with no command, for example while awaiting a select() handle. A later task then saw no decision and nothing after the group, took it for a new frontier and chose another member. An update answered in between had already reported the first choice. Each run now records the version marker durable_workflow.parallel_failure_order once, directly before the first all-group that contains an activity or child workflow, when that group is first scheduled. A marked run raises the first failure in durable event order, whether or not the other members have settled, and needs no per-group decision because history is append-only. A run whose first such group was recorded without the marker keeps the released rule (position when settled, the #338 decision when raised early) for its whole life, on every task, query and update. A run with a pending cancellation request at that point records no marker. Native and PHP Server built from main accept the marker. Published 2.19.1 refuses a marked run as nondeterministic rather than choosing differently. Fixtures recorded by 2.19.1 (zero-command catch, update before and between this version's tasks, select() cancellation, settled group in flight) replay under the released rule. Fixtures recorded by this version replay under the new rule. The selection-gap fixture is kept as released behaviour. Three fixtures of a new run's first group emission are superseded because they now include the marker. Docs now say Python follows the contract's event order where embedded Workflow compares millisecond timestamps, then index (durable-workflow/.github#415). The select() base-sequence bug found in review is #350. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 7 +- docs/sdk-reference.md | 38 +- src/durable_workflow/workflow.py | 105 +- tests/failure_order.py | 35 + ...s-run-once-after-failure-order-marker.json | 109 ++ ...s-run-once-after-failure-order-marker.json | 107 ++ .../failure-order-new-run-update-query.json | 1098 ++++++++++++ .../failure-order-new-run-update.json | 1098 ++++++++++++ ...run-zero-command-catch-complete-query.json | 1327 ++++++++++++++ ...r-new-run-zero-command-catch-complete.json | 1327 ++++++++++++++ ...rder-new-run-zero-command-catch-query.json | 1045 +++++++++++ ...lure-order-new-run-zero-command-catch.json | 1041 +++++++++++ ...re-order-released-select-cancel-query.json | 1012 +++++++++++ .../failure-order-released-select-cancel.json | 1019 +++++++++++ ...leased-update-before-new-worker-query.json | 882 ++++++++++ ...der-released-update-before-new-worker.json | 886 ++++++++++ ...r-released-update-between-tasks-query.json | 1523 +++++++++++++++++ ...e-order-released-update-between-tasks.json | 1523 +++++++++++++++++ ...sed-zero-command-catch-complete-query.json | 1292 ++++++++++++++ ...-released-zero-command-catch-complete.json | 1292 ++++++++++++++ ...der-released-zero-command-catch-query.json | 1107 ++++++++++++ ...ure-order-released-zero-command-catch.json | 1111 ++++++++++++ ...a-emission-after-failure-order-marker.json | 93 + ...led-group-first-task-records-decision.json | 1047 ----------- ...ew-run-raises-first-recorded-failure.json} | 730 ++++---- ...d-group-raises-first-recorded-failure.json | 144 -- tests/test_activity_cancellation_policies.py | 9 +- tests/test_business_replay_clock_parallel.py | 41 +- tests/test_cancellation_scope_authoring.py | 7 +- tests/test_child_workflow_policies.py | 10 +- tests/test_completion_size_limit.py | 7 +- .../test_parallel_completion_order_replay.py | 55 +- tests/test_parallel_failure_order.py | 396 +++++ tests/test_parallel_group_policy.py | 15 +- ...t_parallel_member_failure_open_siblings.py | 85 +- tests/test_parallel_saga_authoring.py | 18 +- tests/test_replay.py | 33 +- tests/test_replay_regression_corpus.py | 35 +- tests/test_run_closed_settlement_replay.py | 8 +- tests/test_worker.py | 3 +- 40 files changed, 19910 insertions(+), 1810 deletions(-) create mode 100644 tests/failure_order.py create mode 100644 tests/fixtures/replay_regressions/completion-history-limit-fails-run-once-after-failure-order-marker.json create mode 100644 tests/fixtures/replay_regressions/completion-size-limit-fails-run-once-after-failure-order-marker.json create mode 100644 tests/fixtures/replay_regressions/failure-order-new-run-update-query.json create mode 100644 tests/fixtures/replay_regressions/failure-order-new-run-update.json create mode 100644 tests/fixtures/replay_regressions/failure-order-new-run-zero-command-catch-complete-query.json create mode 100644 tests/fixtures/replay_regressions/failure-order-new-run-zero-command-catch-complete.json create mode 100644 tests/fixtures/replay_regressions/failure-order-new-run-zero-command-catch-query.json create mode 100644 tests/fixtures/replay_regressions/failure-order-new-run-zero-command-catch.json create mode 100644 tests/fixtures/replay_regressions/failure-order-released-select-cancel-query.json create mode 100644 tests/fixtures/replay_regressions/failure-order-released-select-cancel.json create mode 100644 tests/fixtures/replay_regressions/failure-order-released-update-before-new-worker-query.json create mode 100644 tests/fixtures/replay_regressions/failure-order-released-update-before-new-worker.json create mode 100644 tests/fixtures/replay_regressions/failure-order-released-update-between-tasks-query.json create mode 100644 tests/fixtures/replay_regressions/failure-order-released-update-between-tasks.json create mode 100644 tests/fixtures/replay_regressions/failure-order-released-zero-command-catch-complete-query.json create mode 100644 tests/fixtures/replay_regressions/failure-order-released-zero-command-catch-complete.json create mode 100644 tests/fixtures/replay_regressions/failure-order-released-zero-command-catch-query.json create mode 100644 tests/fixtures/replay_regressions/failure-order-released-zero-command-catch.json create mode 100644 tests/fixtures/replay_regressions/parallel-group-metadata-emission-after-failure-order-marker.json delete mode 100644 tests/fixtures/replay_regressions/settled-group-first-task-records-decision.json rename tests/fixtures/replay_regressions/{settled-group-records-first-recorded-failure.json => settled-group-new-run-raises-first-recorded-failure.json} (65%) delete mode 100644 tests/fixtures/replay_regressions/settled-group-raises-first-recorded-failure.json create mode 100644 tests/test_parallel_failure_order.py diff --git a/CHANGELOG.md b/CHANGELOG.md index f37224b7..147a7c0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,9 +15,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- **Behavior change:** when several members of an `all()` group (`yield [...]`) have failed by the time the workflow resumes, the failure recorded first in history is now raised, even if every member has already settled. Workers up to 2.19.1 raised the failure with the lowest input position in that case, so the exception a workflow catches could depend on whether its task ran before or after the other members finished. This completes the fail-fast default: the first recorded member failure is raised as soon as it is recorded, the other members keep running and are not cancelled, and their outcomes are ignored, as in Workflow's embedded engine. Groups with one failure, or whose first failure also has the lowest position, are unaffected. (#307) - - **Replay safety:** the new choice is gated by the recorded decision from #338. A task records which member it raised (a side effect keyed `durable_workflow.parallel_failure_decision`, at the sequence right after the group) only when the first recorded failure is not the lowest-position one, so its commands are identical to 2.19.1 whenever both rules agree. Replay raises the member a decision names. A settled group with no decision, after which the run closed or did further durable work, is a history from a released worker and keeps the lowest-position rule, including its business-clock behavior. Released workers could not leave such a group without one or the other, since a workflow task that raises a failure either closes the run or blocks on a durable command. New regression fixtures replay a native Server history recorded by published 2.19.1 under the old rule and one recorded by this version under the new rule. The `business-clock-parallel-failure-selection-gap` fixture, whose expectation was the old rule at an unrecorded frontier, stays unchanged as evidence and its history now replays under `settled-group-raises-first-recorded-failure`. - - **Who is affected:** workflows that catch an `all()` failure and act on which member failed, when two or more members can fail. Code after a group that records a decision starts one workflow sequence later. The explicit `propagate`, `cancel_remaining` and `settle` policies remain tracked in #307 and durable-workflow/.github#232. +- **Behavior change:** in a new run, an `all()` group (`yield [...]`) with several failed members raises the failure recorded first in history, even if every member had settled by the time the workflow resumed. Workers up to 2.19.1 raised the lowest-position failure in that case, so the exception a workflow caught depended on whether its task ran before or after the other members finished. The first recorded failure is still raised as soon as it is recorded; the other members keep running, are not cancelled, and their outcomes are ignored. Ties follow durable event order, as the shared parallel-group contract specifies. Workflow's embedded engine compares millisecond timestamps and then member index, so failures recorded in the same millisecond can be chosen differently (durable-workflow/.github#415). (#307) + - **Replay safety:** the rule is recorded once per run. When a run first schedules a group that contains an activity or child workflow, the worker records the version marker `durable_workflow.parallel_failure_order` (version 1) directly before it, so that group and everything after it start one workflow sequence later. A run whose first such group was recorded without the marker keeps the earlier rule for its whole life, on every workflow task, query and update. This holds even when a task recorded no command after catching the failure, or another worker answered an update in between. That run's settled groups raise the lowest-position failure, and a failure raised while siblings are open still records the #338 decision. A run with a cancellation request when it reaches its first group records no marker and keeps the earlier rule. Marked runs need no per-group decision, because the first failure a task sees in append-only history is the first every later replay sees. + - **Upgrade:** published 2.19.1 and earlier fail a marked run's workflow task as nondeterministic at the marker. They do not choose a different failure, and the task is retried, so finish rolling out every worker on a task queue before relying on the new rule. Native and PHP Server built from server main accept the marker. + - **Regression evidence:** fixtures recorded by published 2.19.1 on a Server built from main replay under the earlier rule. They cover a settled group in flight, a catch followed by a task that recorded no command, an update answered by 2.19.1 before or between this version's tasks, and a select() cancellation. Fixtures recorded by this version replay under the new rule. `business-clock-parallel-failure-selection-gap` is unchanged and still passes, as released behaviour. Three fixtures recorded the commands for a new run's first group: `parallel-group-metadata-emission` (superseded at protocol 1.20) and the two completion-limit fixtures (whose inputs now replay under `*-after-failure-order-marker`). These now include the marker. ## [2.19.1] - 2026-11-18 diff --git a/docs/sdk-reference.md b/docs/sdk-reference.md index ea150f08..11101b51 100644 --- a/docs/sdk-reference.md +++ b/docs/sdk-reference.md @@ -524,20 +524,30 @@ profile, (shipping, _) = results A failed activity or child is thrown at the list-yield point as soon as it is recorded. The other members keep running and are not cancelled; their later outcomes do not change the thrown failure or reach the workflow's next -operations. When several members have failed by the time the workflow -resumes, the one whose failure was recorded first in history is thrown, even -if every member has already settled, as in Workflow's embedded engine. A -group whose members are already scheduled is never scheduled again on a later -wake, such as a signal. When a failure is raised while other members are open, -or when a settled group's first recorded failure is not the one with the -lowest input position, the worker records which member it raised as a side -effect directly after the group, so every later replay raises the same one. -Code after such a group therefore starts one workflow sequence later. - -Workers up to 2.19.1 threw the failure with the lowest input position once -every member had settled. Histories they recorded keep that choice on replay: -a settled group with no recorded decision, after which the run closed or did -further durable work, still throws the lowest-position failure. +operations. A group whose members are already scheduled is never scheduled +again on a later wake, such as a signal. + +When several members have failed, a run started on this version throws the +failure recorded first in history, in durable event order, even if every +member had already settled when the workflow resumed. The run records this +choice once, as the version marker `durable_workflow.parallel_failure_order` +directly before its first group that contains an activity or child workflow, +so code after that point starts one workflow sequence later. Workflow's +embedded engine also throws the earliest failure, but compares millisecond +timestamps and then member index, so two failures recorded in the same +millisecond can be chosen differently +([durable-workflow/.github#415](https://github.com/durable-workflow/.github/issues/415)). + +A run whose first such group was recorded without the marker, by an earlier +worker, keeps that worker's rule for its whole life, on every task, query and +update: if every member had settled when the workflow resumed, the failure +with the lowest input position is thrown. When such a run raises a failure +while other members are open, the worker records which member it raised as a +side effect directly after the group, so every later replay raises the same +one. A run that has a cancellation request when it reaches its first group +also keeps this rule. Workers up to 2.19.1 cannot replay a run that recorded +the marker; they fail its workflow task as nondeterministic, so finish +upgrading every worker on a task queue before relying on the new rule. ## Saga compensation diff --git a/src/durable_workflow/workflow.py b/src/durable_workflow/workflow.py index 272d2407..7ce03ae3 100644 --- a/src/durable_workflow/workflow.py +++ b/src/durable_workflow/workflow.py @@ -4265,6 +4265,11 @@ def _history_event_type(event: Mapping[str, Any]) -> str | None: # absent-policy all-group it raised while other members were open # (sdk-python#338). Replay raises exactly that member. PARALLEL_FAILURE_DECISION = "durable_workflow.parallel_failure_decision" +# The version marker a run records, directly before its first all-group, when +# that group is first scheduled by a worker that raises the first recorded +# member failure (sdk-python#307). A run without it keeps the failure +# selection of the workers that ran it before, for its whole life. +PARALLEL_FAILURE_ORDER_MARKER = "durable_workflow.parallel_failure_order" def _settled_at_close(event: Any) -> bool: @@ -6709,6 +6714,11 @@ def _resume_business(function: Callable[[Any], Any], value: Any) -> Any: # Its terminal position proves whether a different-process duplicate belongs. terminal_condition_chain_current: Mapping[str, Any] | None = None version_decisions: dict[str, int] = {} + # Whether this run raises an all-group's first recorded member failure + # (sdk-python#307). Decided once, at the run's first ordinary all-group: + # True when the group is preceded by PARALLEL_FAILURE_ORDER_MARKER or is + # scheduled now, False when it was recorded without the marker. + recorded_failure_order: bool | None = None def _version_error(detail: str, step: _RecordedStep | None = None) -> NonDeterministicReplayError: return NonDeterministicReplayError( @@ -6795,6 +6805,64 @@ def _resolve_version(command: RecordVersionMarker) -> Any: version_decisions.setdefault(change_id, version) return _version_marker_result(command, version) + def _ordinary_all_group(command: Any) -> bool: + """A remote all-group whose failure selection the run's marker decides.""" + if not isinstance(command, list) or "propagate" in parallel_group_policies: + return False + try: + leaves = _parallel_leaves(command) + except (TypeError, ValueError): + return False + if any(isinstance(leaf, WaitCondition | RecordLocalActivity) for leaf in leaves) or ( + prepare_local_activities and _contains_local_activity(command) + ): + return False + # A group of timers alone cannot fail, so it does not decide the rule. + return any(isinstance(leaf, ScheduleActivity | StartChildWorkflow) for leaf in leaves) + + def _resolve_failure_order() -> bool: + """Read or record the run's failure-order marker at its first all-group. + + A run whose first all-group was recorded without the marker was + scheduled by an earlier worker, and keeps that worker's failure + selection for its whole life, on every later task, query and update. + A run reaching its first all-group now records the marker directly + before scheduling it. No all-group of the run has produced an outcome + before that point, so no earlier task can have chosen a failure under + the other rule. A run with a cancellation request that may replace the + group records no marker and keeps the earlier rule. + """ + nonlocal result_cursor, authored_sequence + marker = RecordVersionMarker( + change_id=PARALLEL_FAILURE_ORDER_MARKER, version=1, min_supported=1, max_supported=1, + ) + # The marker is the next authored step; an open select() loser or + # condition wait recorded earlier can still be unconsumed. + step = recorded_steps[result_cursor] if result_cursor < len(recorded_steps) else None + if (step is not None and step.shape == "version marker" + and step.details.get("change_id") == PARALLEL_FAILURE_ORDER_MARKER): + _assert_step_matches(marker, step) + if _recorded_version(step) != 1: + raise _version_error("Unsupported parallel failure order marker version.", step) + result_cursor += 1 + authored_sequence = max(authored_sequence, step.workflow_sequence + 1) + return True + step = _next_unconsumed_recorded_step() + closed = any(_history_event_type(event) in { + "WorkflowCompleted", "WorkflowFailed", "WorkflowCancelled", "WorkflowTerminated", + "WorkflowContinuedAsNew", "WorkflowTimedOut", + } for event in events) + interrupting = cancellation.request is not None or any( + sequence not in consumed_scope_deliveries for sequence in committed_scopes.deliveries + ) + if step is None and not closed and not interrupting: + pending.append(marker) + authored_sequence += 1 + return True + # Scheduled now without the marker, under a cancellation request that + # may replace it, or recorded without it: the earlier rule. + return False + def _parallel_leaf_kind(command: Any) -> str: if isinstance(command, ScheduleActivity | RecordLocalActivity): return "activity" @@ -7820,22 +7888,12 @@ def _absent_policy_failure_decision(commands: list[Any]) -> tuple[bool, int | No task therefore records its decision as a side effect directly after the group, and replay raises exactly the member it names. - When every member has already settled by the time a task first reaches - the group, that task also raises the first recorded failure, as - Workflow's embedded engine does (sdk-python#307). It records the - decision only when that failure is not also the lowest-position one, - the member released workers chose; otherwise both rules raise the same - member and the task's commands are unchanged. - A history without a decision replays as it always has when every member settled before the workflow did any further durable work: the workflow waited for the whole group and the failure is chosen by member - position. Released workers wrote no decision for a settled group, and a - task that raises a failure always either closes the run or records - durable work after the group before it can stop, so such a history - always shows that it moved on. A history without a decision that moved - on while a member was open came from another runtime, such as - Workflow's embedded engine, and raises its first recorded failure. + position. A history without a decision that moved on while a member was + open came from another runtime, such as Workflow's embedded engine, and + raises its first recorded failure. """ marked = _parallel_failure_marker(commands) if marked is not None: @@ -7846,25 +7904,18 @@ def _absent_policy_failure_decision(commands: list[Any]) -> tuple[bool, int | No if base <= recorded_steps[index].workflow_sequence < end] if not any(isinstance(resolved_results[index], ActivityFailed | ChildWorkflowFailed) for index in slots): return False, None, False + if recorded_failure_order: + # History is append-only, so the first recorded failure a task + # sees is the first one every later replay sees: no decision is + # needed, whether or not the other members have settled. + return True, None, False moved_on = [index for sequence, index in first_command_index_by_sequence.items() if sequence >= end] if authored_close_index is not None: moved_on.append(authored_close_index) if len({recorded_steps[index].workflow_sequence for index in slots}) != len(commands): return True, None, not moved_on settled = [recorded_steps[index].history_index for index in slots] - if any(index is None for index in settled): - return False, None, False - if not moved_on and len(slots) == len(commands): - # This task is the first to act on the settled group. Raise the - # first recorded failure; record it only where the legacy - # position rule would choose another member (sdk-python#307). - failures = [index for index in slots - if isinstance(resolved_results[index], ActivityFailed | ChildWorkflowFailed)] - first_recorded = min(failures, key=lambda index: cast(int, recorded_steps[index].history_index)) - lowest_position = min(failures, key=lambda index: recorded_steps[index].workflow_sequence) - differs = first_recorded != lowest_position - return differs, None, differs - if not moved_on: + if any(index is None for index in settled) or not moved_on: return False, None, False return min(moved_on) < max(cast(list[int], settled)), None, False @@ -7947,6 +7998,8 @@ def _advance_selection_clock(base: int, size: int, failure: BaseException | None first = False _apply_due_receivers() _validate_local_activity_policies(cmd) + if recorded_failure_order is None and _ordinary_all_group(cmd): + recorded_failure_order = _resolve_failure_order() if isinstance(cmd, RecordVersionMarker): next_value = _resolve_version(cmd) continue diff --git a/tests/failure_order.py b/tests/failure_order.py new file mode 100644 index 00000000..8058f0fd --- /dev/null +++ b/tests/failure_order.py @@ -0,0 +1,35 @@ +"""Helpers for the run's parallel failure-order marker (sdk-python#307). + +A run records ``PARALLEL_FAILURE_ORDER_MARKER`` directly before the first +all-group that can fail, when that group is first scheduled. Tests about other +behavior that build a history from a fresh run's commands drop it, which +replays as a run an earlier worker started. +""" + +from __future__ import annotations + +from typing import Any + +from durable_workflow.workflow import PARALLEL_FAILURE_ORDER_MARKER, RecordVersionMarker + +FAILURE_ORDER_MARKER = RecordVersionMarker( + change_id=PARALLEL_FAILURE_ORDER_MARKER, version=1, min_supported=1, max_supported=1, +) + + +def is_failure_order_marker(command: Any) -> bool: + if isinstance(command, RecordVersionMarker): + return command.change_id == PARALLEL_FAILURE_ORDER_MARKER + return (isinstance(command, dict) and command.get("type") == "record_version_marker" + and command.get("change_id") == PARALLEL_FAILURE_ORDER_MARKER) + + +def without_failure_order_marker(commands: list[Any]) -> list[Any]: + return [command for command in commands if not is_failure_order_marker(command)] + + +def failure_order_marker_event(sequence: int = 1, **extra: Any) -> dict[str, Any]: + return {"event_type": "VersionMarkerRecorded", **extra, "payload": { + "sequence": sequence, "change_id": PARALLEL_FAILURE_ORDER_MARKER, "version": 1, + "min_supported": 1, "max_supported": 1, + }} diff --git a/tests/fixtures/replay_regressions/completion-history-limit-fails-run-once-after-failure-order-marker.json b/tests/fixtures/replay_regressions/completion-history-limit-fails-run-once-after-failure-order-marker.json new file mode 100644 index 00000000..45812194 --- /dev/null +++ b/tests/fixtures/replay_regressions/completion-history-limit-fails-run-once-after-failure-order-marker.json @@ -0,0 +1,109 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "python-completion-history-limit-fails-run-once-after-failure-order-marker", + "protocol_version": "1.20", + "bindings": [ + "python" + ], + "parallel_failure_order": { + "issues": [ + "durable-workflow/sdk-python#307" + ], + "note": "The fan-out of python-completion-history-limit-fails-run-once. Its first completion now also carries the run's parallel failure-order version marker before the all-group. The refused completion is still never resent and the run still fails once." + }, + "workflow": { + "type": "sdk.completion-size-limit", + "input": [ + { + "activities": 4, + "argument_bytes": 2048, + "limit": 4096, + "refusal": "history_transaction_limit_exceeded", + "limit_kind": "history_transaction_bytes" + } + ] + }, + "command_sequence": [ + { + "type": "workflow_task_request", + "route": "complete", + "command_types": [ + "record_version_marker", + "schedule_activity", + "schedule_activity", + "schedule_activity", + "schedule_activity" + ], + "non_retryable": [ + false, + false, + false, + false, + false + ], + "structural_limit": [ + false, + false, + false, + false, + false + ] + }, + { + "type": "workflow_task_request", + "route": "complete", + "command_types": [ + "fail_workflow" + ], + "non_retryable": [ + true + ], + "structural_limit": [ + true + ] + } + ], + "expected": { + "command_sequence": [ + { + "type": "workflow_task_request", + "route": "complete", + "command_types": [ + "record_version_marker", + "schedule_activity", + "schedule_activity", + "schedule_activity", + "schedule_activity" + ], + "non_retryable": [ + false, + false, + false, + false, + false + ], + "structural_limit": [ + false, + false, + false, + false, + false + ] + }, + { + "type": "workflow_task_request", + "route": "complete", + "command_types": [ + "fail_workflow" + ], + "non_retryable": [ + true + ], + "structural_limit": [ + true + ] + } + ] + } +} diff --git a/tests/fixtures/replay_regressions/completion-size-limit-fails-run-once-after-failure-order-marker.json b/tests/fixtures/replay_regressions/completion-size-limit-fails-run-once-after-failure-order-marker.json new file mode 100644 index 00000000..bd4e446c --- /dev/null +++ b/tests/fixtures/replay_regressions/completion-size-limit-fails-run-once-after-failure-order-marker.json @@ -0,0 +1,107 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "python-completion-size-limit-fails-run-once-after-failure-order-marker", + "protocol_version": "1.20", + "bindings": [ + "python" + ], + "parallel_failure_order": { + "issues": [ + "durable-workflow/sdk-python#307" + ], + "note": "The fan-out of python-completion-size-limit-fails-run-once. Its first completion now also carries the run's parallel failure-order version marker before the all-group. The refused completion is still never resent and the run still fails once." + }, + "workflow": { + "type": "sdk.completion-size-limit", + "input": [ + { + "activities": 4, + "argument_bytes": 2048, + "limit": 4096 + } + ] + }, + "command_sequence": [ + { + "type": "workflow_task_request", + "route": "complete", + "command_types": [ + "record_version_marker", + "schedule_activity", + "schedule_activity", + "schedule_activity", + "schedule_activity" + ], + "non_retryable": [ + false, + false, + false, + false, + false + ], + "structural_limit": [ + false, + false, + false, + false, + false + ] + }, + { + "type": "workflow_task_request", + "route": "complete", + "command_types": [ + "fail_workflow" + ], + "non_retryable": [ + true + ], + "structural_limit": [ + true + ] + } + ], + "expected": { + "command_sequence": [ + { + "type": "workflow_task_request", + "route": "complete", + "command_types": [ + "record_version_marker", + "schedule_activity", + "schedule_activity", + "schedule_activity", + "schedule_activity" + ], + "non_retryable": [ + false, + false, + false, + false, + false + ], + "structural_limit": [ + false, + false, + false, + false, + false + ] + }, + { + "type": "workflow_task_request", + "route": "complete", + "command_types": [ + "fail_workflow" + ], + "non_retryable": [ + true + ], + "structural_limit": [ + true + ] + } + ] + } +} diff --git a/tests/fixtures/replay_regressions/failure-order-new-run-update-query.json b/tests/fixtures/replay_regressions/failure-order-new-run-update-query.json new file mode 100644 index 00000000..9809e20a --- /dev/null +++ b/tests/fixtures/replay_regressions/failure-order-new-run-update-query.json @@ -0,0 +1,1098 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "failure-order-new-run-update-query", + "protocol_version": "1.20", + "bindings": [ + "python" + ], + "parallel_failure_order": { + "issues": [ + "durable-workflow/sdk-python#307" + ], + "note": "Recorded by this version on a Server built from durable-workflow/server main 52e3ec00 (docker-compose.test.yml). The run recorded the marker at sequence 1. After m1 then m0 failed, an update was answered with {compensating: fail-m1} and the workflow compensated fail-m1. This is the history of its last task." + }, + "workflow": { + "type": "rv348_plain", + "input": [], + "payload_codec": "avro", + "run_id": "01m7tfn1drg737gpdqqq521f99", + "workflow_id": "rv307-rv348_plain-021643", + "query": { + "name": "caught", + "arguments": [] + } + }, + "history": [ + { + "id": "01m7tfn1dyeq9qmk4mqccjd4k3", + "sequence": 1, + "event_type": "StartAccepted", + "namespace": "default", + "payload": { + "memo": null, + "command": { + "id": "01m7tfn1dt9s08nvqqk4kh61am", + "type": "start", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "outcome": "started_new", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "sequence": 1, + "applied_at": "2026-11-19T18:03:41.369328Z", + "accepted_at": "2026-11-19T18:03:41.369324Z", + "auth_method": "token", + "auth_status": "authorized", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfn1drg737gpdqqq521f99", + "request_route_name": "generated::FqJuJMl97iiXlIDz", + "request_fingerprint": "sha256:6f781847735e7bee891c0cee63c564fe6ff8c5bdabac5982864acea8cb2c003c" + }, + "outcome": "started_new", + "business_key": null, + "workflow_type": "rv348_plain", + "workflow_class": "rv348_plain", + "workflow_run_id": "01m7tfn1drg737gpdqqq521f99", + "search_attributes": null, + "visibility_labels": null, + "workflow_command_id": "01m7tfn1dt9s08nvqqk4kh61am", + "workflow_instance_id": "rv307-rv348_plain-021643" + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfn1dt9s08nvqqk4kh61am", + "recorded_at": "2026-11-19T18:03:41.373188Z" + }, + { + "id": "01m7tfn1e1g7qswx5qqcjvx475", + "sequence": 2, + "event_type": "WorkflowStarted", + "namespace": "default", + "payload": { + "command": { + "id": "01m7tfn1dt9s08nvqqk4kh61am", + "type": "start", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "outcome": "started_new", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "sequence": 1, + "applied_at": "2026-11-19T18:03:41.369328Z", + "accepted_at": "2026-11-19T18:03:41.369324Z", + "auth_method": "token", + "auth_status": "authorized", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfn1drg737gpdqqq521f99", + "request_route_name": "generated::FqJuJMl97iiXlIDz", + "request_fingerprint": "sha256:6f781847735e7bee891c0cee63c564fe6ff8c5bdabac5982864acea8cb2c003c" + }, + "workflow_type": "rv348_plain", + "workflow_class": "rv348_plain", + "run_deadline_at": "2026-11-19T18:13:41+00:00", + "workflow_run_id": "01m7tfn1drg737gpdqqq521f99", + "declared_queries": [ + "caught" + ], + "declared_signals": [], + "declared_updates": [ + "report" + ], + "declared_entry_mode": "canonical", + "run_timeout_seconds": 600, + "workflow_command_id": "01m7tfn1dt9s08nvqqk4kh61am", + "workflow_instance_id": "rv307-rv348_plain-021643", + "declared_entry_method": "handle", + "execution_deadline_at": "2026-11-19T19:03:41+00:00", + "declared_query_contracts": [ + { + "name": "caught", + "parameters": [] + } + ], + "declared_signal_contracts": [], + "declared_update_contracts": [ + { + "name": "report", + "parameters": [ + { + "name": "_", + "type": null, + "default": null, + "position": 0, + "required": false, + "variadic": true, + "allows_null": true, + "default_available": false + } + ] + } + ], + "execution_timeout_seconds": 3600, + "declared_update_validators": [], + "declared_entry_declaring_class": "external:rv348_plain", + "workflow_definition_fingerprint": "sha256:4cca49f53721ff6d13fd7cd42526e63e2497bbd302876fde0ff10eb9748ac44b", + "workflow_definition_fingerprint_source": "worker" + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfn1dt9s08nvqqk4kh61am", + "recorded_at": "2026-11-19T18:03:41.376531Z" + }, + { + "id": "01m7tfn1x3khh6vmn18q1ry5n5", + "sequence": 3, + "event_type": "VersionMarkerRecorded", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfn1e2tveddgcezafrdzz7", + "type": "workflow", + "queue": "rv307-3c487c", + "status": "leased", + "leased_at": "2026-11-19T18:03:41.431271Z", + "connection": "mysql", + "lease_owner": "rv307-w-b9f827", + "available_at": "2026-11-19T18:03:41.378538Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:51.431219Z", + "last_dispatched_at": "2026-11-19T18:03:41.381151Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:41.381151Z" + }, + "version": 1, + "sequence": 1, + "change_id": "durable_workflow.parallel_failure_order", + "max_supported": 1, + "min_supported": 1 + }, + "workflow_task_id": "01m7tfn1e2tveddgcezafrdzz7", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:41.858774Z" + }, + { + "id": "01m7tfn1x7e983nhj0at552gg3", + "sequence": 4, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfn1e2tveddgcezafrdzz7", + "type": "workflow", + "queue": "rv307-3c487c", + "status": "leased", + "leased_at": "2026-11-19T18:03:41.431271Z", + "connection": "mysql", + "lease_owner": "rv307-w-b9f827", + "available_at": "2026-11-19T18:03:41.378538Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:51.431219Z", + "last_dispatched_at": "2026-11-19T18:03:41.381151Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:41.381151Z" + }, + "activity": { + "id": "01m7tfn1x5cgsn2v049s817edb", + "type": "m0", + "class": "m0", + "queue": "rv307-3c487c", + "status": "pending", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "created_at": "2026-11-19T18:03:41.861897Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfn1x5cgsn2v049s817edb" + }, + "sequence": 2, + "activity_type": "m0", + "activity_class": "m0", + "parallel_group_id": "parallel-activities:2:2", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:2:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfn1x5cgsn2v049s817edb", + "parallel_group_base_sequence": 2 + }, + "workflow_task_id": "01m7tfn1e2tveddgcezafrdzz7", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:41.862859Z" + }, + { + "id": "01m7tfn1xbvgjmxp87jjfzswez", + "sequence": 5, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfn1e2tveddgcezafrdzz7", + "type": "workflow", + "queue": "rv307-3c487c", + "status": "leased", + "leased_at": "2026-11-19T18:03:41.431271Z", + "connection": "mysql", + "lease_owner": "rv307-w-b9f827", + "available_at": "2026-11-19T18:03:41.378538Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:51.431219Z", + "last_dispatched_at": "2026-11-19T18:03:41.381151Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:41.381151Z" + }, + "activity": { + "id": "01m7tfn1xa8kz80n8x8gmt3yvj", + "type": "m1", + "class": "m1", + "queue": "rv307-3c487c", + "status": "pending", + "sequence": 3, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "created_at": "2026-11-19T18:03:41.866480Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfn1xa8kz80n8x8gmt3yvj" + }, + "sequence": 3, + "activity_type": "m1", + "activity_class": "m1", + "parallel_group_id": "parallel-activities:2:2", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:2:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfn1xa8kz80n8x8gmt3yvj", + "parallel_group_base_sequence": 2 + }, + "workflow_task_id": "01m7tfn1e2tveddgcezafrdzz7", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:41.867275Z" + }, + { + "id": "01m7tfn1z39dm0af76pn8tx18g", + "sequence": 6, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfn1x8ydcp1n4ewg2n80v8", + "type": "activity", + "queue": "rv307-3c487c", + "status": "leased", + "leased_at": "2026-11-19T18:03:41.919452Z", + "connection": "mysql", + "lease_owner": "rv307-w-b9f827", + "available_at": "2026-11-19T18:03:41.864680Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:41.919532Z" + }, + "activity": { + "id": "01m7tfn1x5cgsn2v049s817edb", + "type": "m0", + "class": "m0", + "queue": "rv307-3c487c", + "status": "running", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "attempt_id": "01M7TFN1YZJRN3E25EV1D72SE7", + "created_at": "2026-11-19T18:03:41.861897Z", + "started_at": "2026-11-19T18:03:41.919452Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfn1x5cgsn2v049s817edb" + }, + "sequence": 2, + "activity_type": "m0", + "activity_class": "m0", + "attempt_number": 1, + "parallel_group_id": "parallel-activities:2:2", + "activity_attempt_id": "01M7TFN1YZJRN3E25EV1D72SE7", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:2:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfn1x5cgsn2v049s817edb", + "parallel_group_base_sequence": 2 + }, + "workflow_task_id": "01m7tfn1x8ydcp1n4ewg2n80v8", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:41.922970Z" + }, + { + "id": "01m7tfn22f9ff4nhbr2c2rgxm3", + "sequence": 7, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfn1xc73933aec88y8fke7", + "type": "activity", + "queue": "rv307-3c487c", + "status": "leased", + "leased_at": "2026-11-19T18:03:42.027082Z", + "connection": "mysql", + "lease_owner": "rv307-w-b9f827", + "available_at": "2026-11-19T18:03:41.868856Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:42.027169Z" + }, + "activity": { + "id": "01m7tfn1xa8kz80n8x8gmt3yvj", + "type": "m1", + "class": "m1", + "queue": "rv307-3c487c", + "status": "running", + "sequence": 3, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "attempt_id": "01M7TFN22BAVCPPS6NVD4GBRPM", + "created_at": "2026-11-19T18:03:41.866480Z", + "started_at": "2026-11-19T18:03:42.027082Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfn1xa8kz80n8x8gmt3yvj" + }, + "sequence": 3, + "activity_type": "m1", + "activity_class": "m1", + "attempt_number": 1, + "parallel_group_id": "parallel-activities:2:2", + "activity_attempt_id": "01M7TFN22BAVCPPS6NVD4GBRPM", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:2:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfn1xa8kz80n8x8gmt3yvj", + "parallel_group_base_sequence": 2 + }, + "workflow_task_id": "01m7tfn1xc73933aec88y8fke7", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:42.030648Z" + }, + { + "id": "01m7tfn25v10z3xhkzhd5g0wa4", + "sequence": 8, + "event_type": "ActivityFailed", + "namespace": "default", + "payload": { + "code": 0, + "task": { + "id": "01m7tfn1xc73933aec88y8fke7", + "type": "activity", + "queue": "rv307-3c487c", + "status": "leased", + "leased_at": "2026-11-19T18:03:42.027082Z", + "connection": "mysql", + "lease_owner": "rv307-w-b9f827", + "available_at": "2026-11-19T18:03:41.868856Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:42.076765Z" + }, + "message": "fail-m1", + "activity": { + "id": "01m7tfn1xa8kz80n8x8gmt3yvj", + "type": "m1", + "class": "m1", + "queue": "rv307-3c487c", + "status": "failed", + "sequence": 3, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "closed_at": "2026-11-19T18:03:42.138140Z", + "exception": { + "code": 0, + "type": null, + "message": "fail-m1", + "non_retryable": true + }, + "attempt_id": "01M7TFN22BAVCPPS6NVD4GBRPM", + "created_at": "2026-11-19T18:03:41.866480Z", + "started_at": "2026-11-19T18:03:42.027082Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfn1xa8kz80n8x8gmt3yvj" + }, + "sequence": 3, + "exception": { + "type": null, + "message": "fail-m1", + "code": 0, + "non_retryable": true + }, + "failure_id": "01m7tfn25scqz17xmq6hrb3zdr", + "activity_type": "m1", + "non_retryable": true, + "activity_class": "m1", + "attempt_number": 1, + "exception_type": null, + "exception_class": "RuntimeException", + "failure_category": "activity", + "parallel_group_id": "parallel-activities:2:2", + "activity_attempt_id": "01M7TFN22BAVCPPS6NVD4GBRPM", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:2:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfn1xa8kz80n8x8gmt3yvj", + "parallel_group_base_sequence": 2 + }, + "workflow_task_id": "01m7tfn1xc73933aec88y8fke7", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:42.139356Z" + }, + { + "id": "01m7tfn3fdekb6tkfjjxcs5ybc", + "sequence": 9, + "event_type": "ActivityFailed", + "namespace": "default", + "payload": { + "code": 0, + "task": { + "id": "01m7tfn1x8ydcp1n4ewg2n80v8", + "type": "activity", + "queue": "rv307-3c487c", + "status": "leased", + "leased_at": "2026-11-19T18:03:41.919452Z", + "connection": "mysql", + "lease_owner": "rv307-w-b9f827", + "available_at": "2026-11-19T18:03:41.864680Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:41.973109Z" + }, + "message": "fail-m0", + "activity": { + "id": "01m7tfn1x5cgsn2v049s817edb", + "type": "m0", + "class": "m0", + "queue": "rv307-3c487c", + "status": "failed", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "closed_at": "2026-11-19T18:03:43.467344Z", + "exception": { + "code": 0, + "type": null, + "message": "fail-m0", + "non_retryable": true + }, + "attempt_id": "01M7TFN1YZJRN3E25EV1D72SE7", + "created_at": "2026-11-19T18:03:41.861897Z", + "started_at": "2026-11-19T18:03:41.919452Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfn1x5cgsn2v049s817edb" + }, + "sequence": 2, + "exception": { + "type": null, + "message": "fail-m0", + "code": 0, + "non_retryable": true + }, + "failure_id": "01m7tfn3fa0nvybhksfsyk98xx", + "activity_type": "m0", + "non_retryable": true, + "activity_class": "m0", + "attempt_number": 1, + "exception_type": null, + "exception_class": "RuntimeException", + "failure_category": "activity", + "parallel_group_id": "parallel-activities:2:2", + "activity_attempt_id": "01M7TFN1YZJRN3E25EV1D72SE7", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:2:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfn1x5cgsn2v049s817edb", + "parallel_group_base_sequence": 2 + }, + "workflow_task_id": "01m7tfn1x8ydcp1n4ewg2n80v8", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:43.468603Z" + }, + { + "id": "01m7tfn42bnvtyz1c7geg6kzvg", + "sequence": 10, + "event_type": "UpdateAccepted", + "namespace": "default", + "payload": { + "command": { + "id": "01m7tfn424cxrk1nfwx2sn4mz3", + "type": "update", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNw4GCG5hbWUKDHJlcG9ydBJhcmd1bWVudHMMACJ2YWxpZGF0aW9uX2Vycm9ycwwAAA==" + }, + "sequence": 2, + "request_id": "sdk-python-update-928a030fbcde4de3a147d7549ac58407", + "accepted_at": "2026-11-19T18:03:44.065158Z", + "auth_method": "token", + "auth_status": "authorized", + "target_name": "report", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows/rv307-rv348_plain-021643/update/report", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfn1drg737gpdqqq521f99", + "message_sequence": 1, + "request_route_name": "generated::YF2YckSNfLzTOm1b", + "request_fingerprint": "sha256:b3ab45962900874a3816d6d428ad736697abec828ed736a7a1f54370266f35b9" + }, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "update_id": "01m7tfn425egqb0gwrxr2vry0f", + "request_id": "sdk-python-update-928a030fbcde4de3a147d7549ac58407", + "update_name": "report", + "ordering_state": "ready", + "workflow_run_id": "01m7tfn1drg737gpdqqq521f99", + "workflow_command_id": "01m7tfn424cxrk1nfwx2sn4mz3", + "workflow_instance_id": "rv307-rv348_plain-021643", + "queued_behind_command_id": null, + "queued_behind_command_type": null, + "queued_behind_command_sequence": null + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfn424cxrk1nfwx2sn4mz3", + "recorded_at": "2026-11-19T18:03:44.073042Z" + }, + { + "id": "01m7tfn4dbd6xggqf1pam9zeq7", + "sequence": 11, + "event_type": "UpdateApplied", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfn262t4er6jqsrrkc0tph", + "type": "workflow", + "queue": "rv307-3c487c", + "status": "leased", + "leased_at": "2026-11-19T18:03:44.196073Z", + "connection": "mysql", + "lease_owner": "rv307-w-b9f827", + "available_at": "2026-11-19T18:03:42.145784Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:54.195943Z", + "last_dispatched_at": "2026-11-19T18:03:42.195663Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:42.195663Z" + }, + "command": { + "id": "01m7tfn424cxrk1nfwx2sn4mz3", + "type": "update", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNw4GCG5hbWUKDHJlcG9ydBJhcmd1bWVudHMMACJ2YWxpZGF0aW9uX2Vycm9ycwwAAA==" + }, + "sequence": 2, + "request_id": "sdk-python-update-928a030fbcde4de3a147d7549ac58407", + "accepted_at": "2026-11-19T18:03:44.065158Z", + "auth_method": "token", + "auth_status": "authorized", + "target_name": "report", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows/rv307-rv348_plain-021643/update/report", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfn1drg737gpdqqq521f99", + "message_sequence": 1, + "request_route_name": "generated::YF2YckSNfLzTOm1b", + "request_fingerprint": "sha256:b3ab45962900874a3816d6d428ad736697abec828ed736a7a1f54370266f35b9" + }, + "sequence": 4, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "update_id": "01m7tfn425egqb0gwrxr2vry0f", + "update_name": "report", + "workflow_run_id": "01m7tfn1drg737gpdqqq521f99", + "workflow_command_id": "01m7tfn424cxrk1nfwx2sn4mz3", + "workflow_instance_id": "rv307-rv348_plain-021643" + }, + "workflow_task_id": "01m7tfn262t4er6jqsrrkc0tph", + "workflow_command_id": "01m7tfn424cxrk1nfwx2sn4mz3", + "recorded_at": "2026-11-19T18:03:44.426943Z" + }, + { + "id": "01m7tfn4dfff9acyj7phdaf0qr", + "sequence": 12, + "event_type": "UpdateCompleted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfn262t4er6jqsrrkc0tph", + "type": "workflow", + "queue": "rv307-3c487c", + "status": "leased", + "leased_at": "2026-11-19T18:03:44.196073Z", + "connection": "mysql", + "lease_owner": "rv307-w-b9f827", + "available_at": "2026-11-19T18:03:42.145784Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:54.195943Z", + "last_dispatched_at": "2026-11-19T18:03:42.195663Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:42.195663Z" + }, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNw4CGGNvbXBlbnNhdGluZwoOZmFpbC1tMQA=" + }, + "command": { + "id": "01m7tfn424cxrk1nfwx2sn4mz3", + "type": "update", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNw4GCG5hbWUKDHJlcG9ydBJhcmd1bWVudHMMACJ2YWxpZGF0aW9uX2Vycm9ycwwAAA==" + }, + "sequence": 2, + "request_id": "sdk-python-update-928a030fbcde4de3a147d7549ac58407", + "accepted_at": "2026-11-19T18:03:44.065158Z", + "auth_method": "token", + "auth_status": "authorized", + "target_name": "report", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows/rv307-rv348_plain-021643/update/report", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfn1drg737gpdqqq521f99", + "message_sequence": 1, + "request_route_name": "generated::YF2YckSNfLzTOm1b", + "request_fingerprint": "sha256:b3ab45962900874a3816d6d428ad736697abec828ed736a7a1f54370266f35b9" + }, + "sequence": 4, + "update_id": "01m7tfn425egqb0gwrxr2vry0f", + "update_name": "report", + "workflow_run_id": "01m7tfn1drg737gpdqqq521f99", + "workflow_command_id": "01m7tfn424cxrk1nfwx2sn4mz3", + "workflow_instance_id": "rv307-rv348_plain-021643" + }, + "workflow_task_id": "01m7tfn262t4er6jqsrrkc0tph", + "workflow_command_id": "01m7tfn424cxrk1nfwx2sn4mz3", + "recorded_at": "2026-11-19T18:03:44.430300Z" + }, + { + "id": "01m7tfn4dkxb7vmv12z8w7f9h0", + "sequence": 13, + "event_type": "MessageCursorAdvanced", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfn262t4er6jqsrrkc0tph", + "type": "workflow", + "queue": "rv307-3c487c", + "status": "leased", + "leased_at": "2026-11-19T18:03:44.196073Z", + "connection": "mysql", + "lease_owner": "rv307-w-b9f827", + "available_at": "2026-11-19T18:03:42.145784Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:54.195943Z", + "last_dispatched_at": "2026-11-19T18:03:42.195663Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:42.195663Z" + }, + "stream_key": "instance:rv307-rv348_plain-021643", + "new_position": 1, + "previous_position": 0 + }, + "workflow_task_id": "01m7tfn262t4er6jqsrrkc0tph", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:44.435034Z" + }, + { + "id": "01m7tfn4vda2w5frne75t80a97", + "sequence": 14, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfn3fm9p9mtajg5jf0dza3", + "type": "workflow", + "queue": "rv307-3c487c", + "status": "leased", + "leased_at": "2026-11-19T18:03:44.496300Z", + "connection": "mysql", + "lease_owner": "rv307-w-b9f827", + "available_at": "2026-11-19T18:03:43.475887Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:54.496269Z", + "last_dispatched_at": "2026-11-19T18:03:43.505829Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:43.505829Z" + }, + "activity": { + "id": "01m7tfn4vbn5sf1h1yn8n4hhrq", + "type": "compensate", + "class": "compensate", + "queue": "rv307-3c487c", + "status": "pending", + "sequence": 4, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCCg5mYWlsLW0xAA==" + }, + "created_at": "2026-11-19T18:03:44.875794Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfn4vbn5sf1h1yn8n4hhrq" + }, + "sequence": 4, + "activity_type": "compensate", + "activity_class": "compensate", + "activity_execution_id": "01m7tfn4vbn5sf1h1yn8n4hhrq" + }, + "workflow_task_id": "01m7tfn3fm9p9mtajg5jf0dza3", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:44.876814Z" + }, + { + "id": "01m7tfn6z0t2wp7y3x5bf5pkjg", + "sequence": 15, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfn4vfecsge06pr5gtfe7w", + "type": "activity", + "queue": "rv307-3c487c", + "status": "leased", + "leased_at": "2026-11-19T18:03:47.034837Z", + "connection": "mysql", + "lease_owner": "rv307-w-b9f827", + "available_at": "2026-11-19T18:03:44.879382Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:47.034929Z" + }, + "activity": { + "id": "01m7tfn4vbn5sf1h1yn8n4hhrq", + "type": "compensate", + "class": "compensate", + "queue": "rv307-3c487c", + "status": "running", + "sequence": 4, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCCg5mYWlsLW0xAA==" + }, + "attempt_id": "01M7TFN6YTGS8HKSTH82905RXS", + "created_at": "2026-11-19T18:03:44.875794Z", + "started_at": "2026-11-19T18:03:47.034837Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfn4vbn5sf1h1yn8n4hhrq" + }, + "sequence": 4, + "activity_type": "compensate", + "activity_class": "compensate", + "attempt_number": 1, + "activity_attempt_id": "01M7TFN6YTGS8HKSTH82905RXS", + "activity_execution_id": "01m7tfn4vbn5sf1h1yn8n4hhrq" + }, + "workflow_task_id": "01m7tfn4vfecsge06pr5gtfe7w", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:47.039650Z" + }, + { + "id": "01m7tfn71trh1rg3tvxrxdzy6p", + "sequence": 16, + "event_type": "ActivityCompleted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfn4vfecsge06pr5gtfe7w", + "type": "activity", + "queue": "rv307-3c487c", + "status": "leased", + "leased_at": "2026-11-19T18:03:47.034837Z", + "connection": "mysql", + "lease_owner": "rv307-w-b9f827", + "available_at": "2026-11-19T18:03:44.879382Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:47.102673Z" + }, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoMdW5kb25l" + }, + "activity": { + "id": "01m7tfn4vbn5sf1h1yn8n4hhrq", + "type": "compensate", + "class": "compensate", + "queue": "rv307-3c487c", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoMdW5kb25l" + }, + "status": "completed", + "sequence": 4, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCCg5mYWlsLW0xAA==" + }, + "closed_at": "2026-11-19T18:03:47.128561Z", + "attempt_id": "01M7TFN6YTGS8HKSTH82905RXS", + "created_at": "2026-11-19T18:03:44.875794Z", + "started_at": "2026-11-19T18:03:47.034837Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfn4vbn5sf1h1yn8n4hhrq" + }, + "sequence": 4, + "activity_type": "compensate", + "payload_codec": "avro", + "activity_class": "compensate", + "attempt_number": 1, + "activity_attempt_id": "01M7TFN6YTGS8HKSTH82905RXS", + "activity_execution_id": "01m7tfn4vbn5sf1h1yn8n4hhrq" + }, + "workflow_task_id": "01m7tfn4vfecsge06pr5gtfe7w", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:47.129768Z" + } + ], + "expected": { + "query_result": "fail-m1" + } +} diff --git a/tests/fixtures/replay_regressions/failure-order-new-run-update.json b/tests/fixtures/replay_regressions/failure-order-new-run-update.json new file mode 100644 index 00000000..20f0517d --- /dev/null +++ b/tests/fixtures/replay_regressions/failure-order-new-run-update.json @@ -0,0 +1,1098 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "failure-order-new-run-update", + "protocol_version": "1.20", + "bindings": [ + "python" + ], + "parallel_failure_order": { + "issues": [ + "durable-workflow/sdk-python#307" + ], + "note": "Recorded by this version on a Server built from durable-workflow/server main 52e3ec00 (docker-compose.test.yml). The run recorded the marker at sequence 1. After m1 then m0 failed, an update was answered with {compensating: fail-m1} and the workflow compensated fail-m1. This is the history of its last task." + }, + "workflow": { + "type": "rv348_plain", + "input": [], + "payload_codec": "avro", + "run_id": "01m7tfn1drg737gpdqqq521f99", + "workflow_id": "rv307-rv348_plain-021643" + }, + "history": [ + { + "id": "01m7tfn1dyeq9qmk4mqccjd4k3", + "sequence": 1, + "event_type": "StartAccepted", + "namespace": "default", + "payload": { + "memo": null, + "command": { + "id": "01m7tfn1dt9s08nvqqk4kh61am", + "type": "start", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "outcome": "started_new", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "sequence": 1, + "applied_at": "2026-11-19T18:03:41.369328Z", + "accepted_at": "2026-11-19T18:03:41.369324Z", + "auth_method": "token", + "auth_status": "authorized", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfn1drg737gpdqqq521f99", + "request_route_name": "generated::FqJuJMl97iiXlIDz", + "request_fingerprint": "sha256:6f781847735e7bee891c0cee63c564fe6ff8c5bdabac5982864acea8cb2c003c" + }, + "outcome": "started_new", + "business_key": null, + "workflow_type": "rv348_plain", + "workflow_class": "rv348_plain", + "workflow_run_id": "01m7tfn1drg737gpdqqq521f99", + "search_attributes": null, + "visibility_labels": null, + "workflow_command_id": "01m7tfn1dt9s08nvqqk4kh61am", + "workflow_instance_id": "rv307-rv348_plain-021643" + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfn1dt9s08nvqqk4kh61am", + "recorded_at": "2026-11-19T18:03:41.373188Z" + }, + { + "id": "01m7tfn1e1g7qswx5qqcjvx475", + "sequence": 2, + "event_type": "WorkflowStarted", + "namespace": "default", + "payload": { + "command": { + "id": "01m7tfn1dt9s08nvqqk4kh61am", + "type": "start", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "outcome": "started_new", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "sequence": 1, + "applied_at": "2026-11-19T18:03:41.369328Z", + "accepted_at": "2026-11-19T18:03:41.369324Z", + "auth_method": "token", + "auth_status": "authorized", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfn1drg737gpdqqq521f99", + "request_route_name": "generated::FqJuJMl97iiXlIDz", + "request_fingerprint": "sha256:6f781847735e7bee891c0cee63c564fe6ff8c5bdabac5982864acea8cb2c003c" + }, + "workflow_type": "rv348_plain", + "workflow_class": "rv348_plain", + "run_deadline_at": "2026-11-19T18:13:41+00:00", + "workflow_run_id": "01m7tfn1drg737gpdqqq521f99", + "declared_queries": [ + "caught" + ], + "declared_signals": [], + "declared_updates": [ + "report" + ], + "declared_entry_mode": "canonical", + "run_timeout_seconds": 600, + "workflow_command_id": "01m7tfn1dt9s08nvqqk4kh61am", + "workflow_instance_id": "rv307-rv348_plain-021643", + "declared_entry_method": "handle", + "execution_deadline_at": "2026-11-19T19:03:41+00:00", + "declared_query_contracts": [ + { + "name": "caught", + "parameters": [] + } + ], + "declared_signal_contracts": [], + "declared_update_contracts": [ + { + "name": "report", + "parameters": [ + { + "name": "_", + "type": null, + "default": null, + "position": 0, + "required": false, + "variadic": true, + "allows_null": true, + "default_available": false + } + ] + } + ], + "execution_timeout_seconds": 3600, + "declared_update_validators": [], + "declared_entry_declaring_class": "external:rv348_plain", + "workflow_definition_fingerprint": "sha256:4cca49f53721ff6d13fd7cd42526e63e2497bbd302876fde0ff10eb9748ac44b", + "workflow_definition_fingerprint_source": "worker" + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfn1dt9s08nvqqk4kh61am", + "recorded_at": "2026-11-19T18:03:41.376531Z" + }, + { + "id": "01m7tfn1x3khh6vmn18q1ry5n5", + "sequence": 3, + "event_type": "VersionMarkerRecorded", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfn1e2tveddgcezafrdzz7", + "type": "workflow", + "queue": "rv307-3c487c", + "status": "leased", + "leased_at": "2026-11-19T18:03:41.431271Z", + "connection": "mysql", + "lease_owner": "rv307-w-b9f827", + "available_at": "2026-11-19T18:03:41.378538Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:51.431219Z", + "last_dispatched_at": "2026-11-19T18:03:41.381151Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:41.381151Z" + }, + "version": 1, + "sequence": 1, + "change_id": "durable_workflow.parallel_failure_order", + "max_supported": 1, + "min_supported": 1 + }, + "workflow_task_id": "01m7tfn1e2tveddgcezafrdzz7", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:41.858774Z" + }, + { + "id": "01m7tfn1x7e983nhj0at552gg3", + "sequence": 4, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfn1e2tveddgcezafrdzz7", + "type": "workflow", + "queue": "rv307-3c487c", + "status": "leased", + "leased_at": "2026-11-19T18:03:41.431271Z", + "connection": "mysql", + "lease_owner": "rv307-w-b9f827", + "available_at": "2026-11-19T18:03:41.378538Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:51.431219Z", + "last_dispatched_at": "2026-11-19T18:03:41.381151Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:41.381151Z" + }, + "activity": { + "id": "01m7tfn1x5cgsn2v049s817edb", + "type": "m0", + "class": "m0", + "queue": "rv307-3c487c", + "status": "pending", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "created_at": "2026-11-19T18:03:41.861897Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfn1x5cgsn2v049s817edb" + }, + "sequence": 2, + "activity_type": "m0", + "activity_class": "m0", + "parallel_group_id": "parallel-activities:2:2", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:2:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfn1x5cgsn2v049s817edb", + "parallel_group_base_sequence": 2 + }, + "workflow_task_id": "01m7tfn1e2tveddgcezafrdzz7", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:41.862859Z" + }, + { + "id": "01m7tfn1xbvgjmxp87jjfzswez", + "sequence": 5, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfn1e2tveddgcezafrdzz7", + "type": "workflow", + "queue": "rv307-3c487c", + "status": "leased", + "leased_at": "2026-11-19T18:03:41.431271Z", + "connection": "mysql", + "lease_owner": "rv307-w-b9f827", + "available_at": "2026-11-19T18:03:41.378538Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:51.431219Z", + "last_dispatched_at": "2026-11-19T18:03:41.381151Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:41.381151Z" + }, + "activity": { + "id": "01m7tfn1xa8kz80n8x8gmt3yvj", + "type": "m1", + "class": "m1", + "queue": "rv307-3c487c", + "status": "pending", + "sequence": 3, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "created_at": "2026-11-19T18:03:41.866480Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfn1xa8kz80n8x8gmt3yvj" + }, + "sequence": 3, + "activity_type": "m1", + "activity_class": "m1", + "parallel_group_id": "parallel-activities:2:2", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:2:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfn1xa8kz80n8x8gmt3yvj", + "parallel_group_base_sequence": 2 + }, + "workflow_task_id": "01m7tfn1e2tveddgcezafrdzz7", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:41.867275Z" + }, + { + "id": "01m7tfn1z39dm0af76pn8tx18g", + "sequence": 6, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfn1x8ydcp1n4ewg2n80v8", + "type": "activity", + "queue": "rv307-3c487c", + "status": "leased", + "leased_at": "2026-11-19T18:03:41.919452Z", + "connection": "mysql", + "lease_owner": "rv307-w-b9f827", + "available_at": "2026-11-19T18:03:41.864680Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:41.919532Z" + }, + "activity": { + "id": "01m7tfn1x5cgsn2v049s817edb", + "type": "m0", + "class": "m0", + "queue": "rv307-3c487c", + "status": "running", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "attempt_id": "01M7TFN1YZJRN3E25EV1D72SE7", + "created_at": "2026-11-19T18:03:41.861897Z", + "started_at": "2026-11-19T18:03:41.919452Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfn1x5cgsn2v049s817edb" + }, + "sequence": 2, + "activity_type": "m0", + "activity_class": "m0", + "attempt_number": 1, + "parallel_group_id": "parallel-activities:2:2", + "activity_attempt_id": "01M7TFN1YZJRN3E25EV1D72SE7", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:2:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfn1x5cgsn2v049s817edb", + "parallel_group_base_sequence": 2 + }, + "workflow_task_id": "01m7tfn1x8ydcp1n4ewg2n80v8", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:41.922970Z" + }, + { + "id": "01m7tfn22f9ff4nhbr2c2rgxm3", + "sequence": 7, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfn1xc73933aec88y8fke7", + "type": "activity", + "queue": "rv307-3c487c", + "status": "leased", + "leased_at": "2026-11-19T18:03:42.027082Z", + "connection": "mysql", + "lease_owner": "rv307-w-b9f827", + "available_at": "2026-11-19T18:03:41.868856Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:42.027169Z" + }, + "activity": { + "id": "01m7tfn1xa8kz80n8x8gmt3yvj", + "type": "m1", + "class": "m1", + "queue": "rv307-3c487c", + "status": "running", + "sequence": 3, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "attempt_id": "01M7TFN22BAVCPPS6NVD4GBRPM", + "created_at": "2026-11-19T18:03:41.866480Z", + "started_at": "2026-11-19T18:03:42.027082Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfn1xa8kz80n8x8gmt3yvj" + }, + "sequence": 3, + "activity_type": "m1", + "activity_class": "m1", + "attempt_number": 1, + "parallel_group_id": "parallel-activities:2:2", + "activity_attempt_id": "01M7TFN22BAVCPPS6NVD4GBRPM", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:2:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfn1xa8kz80n8x8gmt3yvj", + "parallel_group_base_sequence": 2 + }, + "workflow_task_id": "01m7tfn1xc73933aec88y8fke7", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:42.030648Z" + }, + { + "id": "01m7tfn25v10z3xhkzhd5g0wa4", + "sequence": 8, + "event_type": "ActivityFailed", + "namespace": "default", + "payload": { + "code": 0, + "task": { + "id": "01m7tfn1xc73933aec88y8fke7", + "type": "activity", + "queue": "rv307-3c487c", + "status": "leased", + "leased_at": "2026-11-19T18:03:42.027082Z", + "connection": "mysql", + "lease_owner": "rv307-w-b9f827", + "available_at": "2026-11-19T18:03:41.868856Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:42.076765Z" + }, + "message": "fail-m1", + "activity": { + "id": "01m7tfn1xa8kz80n8x8gmt3yvj", + "type": "m1", + "class": "m1", + "queue": "rv307-3c487c", + "status": "failed", + "sequence": 3, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "closed_at": "2026-11-19T18:03:42.138140Z", + "exception": { + "code": 0, + "type": null, + "message": "fail-m1", + "non_retryable": true + }, + "attempt_id": "01M7TFN22BAVCPPS6NVD4GBRPM", + "created_at": "2026-11-19T18:03:41.866480Z", + "started_at": "2026-11-19T18:03:42.027082Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfn1xa8kz80n8x8gmt3yvj" + }, + "sequence": 3, + "exception": { + "type": null, + "message": "fail-m1", + "code": 0, + "non_retryable": true + }, + "failure_id": "01m7tfn25scqz17xmq6hrb3zdr", + "activity_type": "m1", + "non_retryable": true, + "activity_class": "m1", + "attempt_number": 1, + "exception_type": null, + "exception_class": "RuntimeException", + "failure_category": "activity", + "parallel_group_id": "parallel-activities:2:2", + "activity_attempt_id": "01M7TFN22BAVCPPS6NVD4GBRPM", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:2:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfn1xa8kz80n8x8gmt3yvj", + "parallel_group_base_sequence": 2 + }, + "workflow_task_id": "01m7tfn1xc73933aec88y8fke7", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:42.139356Z" + }, + { + "id": "01m7tfn3fdekb6tkfjjxcs5ybc", + "sequence": 9, + "event_type": "ActivityFailed", + "namespace": "default", + "payload": { + "code": 0, + "task": { + "id": "01m7tfn1x8ydcp1n4ewg2n80v8", + "type": "activity", + "queue": "rv307-3c487c", + "status": "leased", + "leased_at": "2026-11-19T18:03:41.919452Z", + "connection": "mysql", + "lease_owner": "rv307-w-b9f827", + "available_at": "2026-11-19T18:03:41.864680Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:41.973109Z" + }, + "message": "fail-m0", + "activity": { + "id": "01m7tfn1x5cgsn2v049s817edb", + "type": "m0", + "class": "m0", + "queue": "rv307-3c487c", + "status": "failed", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "closed_at": "2026-11-19T18:03:43.467344Z", + "exception": { + "code": 0, + "type": null, + "message": "fail-m0", + "non_retryable": true + }, + "attempt_id": "01M7TFN1YZJRN3E25EV1D72SE7", + "created_at": "2026-11-19T18:03:41.861897Z", + "started_at": "2026-11-19T18:03:41.919452Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfn1x5cgsn2v049s817edb" + }, + "sequence": 2, + "exception": { + "type": null, + "message": "fail-m0", + "code": 0, + "non_retryable": true + }, + "failure_id": "01m7tfn3fa0nvybhksfsyk98xx", + "activity_type": "m0", + "non_retryable": true, + "activity_class": "m0", + "attempt_number": 1, + "exception_type": null, + "exception_class": "RuntimeException", + "failure_category": "activity", + "parallel_group_id": "parallel-activities:2:2", + "activity_attempt_id": "01M7TFN1YZJRN3E25EV1D72SE7", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:2:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfn1x5cgsn2v049s817edb", + "parallel_group_base_sequence": 2 + }, + "workflow_task_id": "01m7tfn1x8ydcp1n4ewg2n80v8", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:43.468603Z" + }, + { + "id": "01m7tfn42bnvtyz1c7geg6kzvg", + "sequence": 10, + "event_type": "UpdateAccepted", + "namespace": "default", + "payload": { + "command": { + "id": "01m7tfn424cxrk1nfwx2sn4mz3", + "type": "update", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNw4GCG5hbWUKDHJlcG9ydBJhcmd1bWVudHMMACJ2YWxpZGF0aW9uX2Vycm9ycwwAAA==" + }, + "sequence": 2, + "request_id": "sdk-python-update-928a030fbcde4de3a147d7549ac58407", + "accepted_at": "2026-11-19T18:03:44.065158Z", + "auth_method": "token", + "auth_status": "authorized", + "target_name": "report", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows/rv307-rv348_plain-021643/update/report", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfn1drg737gpdqqq521f99", + "message_sequence": 1, + "request_route_name": "generated::YF2YckSNfLzTOm1b", + "request_fingerprint": "sha256:b3ab45962900874a3816d6d428ad736697abec828ed736a7a1f54370266f35b9" + }, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "update_id": "01m7tfn425egqb0gwrxr2vry0f", + "request_id": "sdk-python-update-928a030fbcde4de3a147d7549ac58407", + "update_name": "report", + "ordering_state": "ready", + "workflow_run_id": "01m7tfn1drg737gpdqqq521f99", + "workflow_command_id": "01m7tfn424cxrk1nfwx2sn4mz3", + "workflow_instance_id": "rv307-rv348_plain-021643", + "queued_behind_command_id": null, + "queued_behind_command_type": null, + "queued_behind_command_sequence": null + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfn424cxrk1nfwx2sn4mz3", + "recorded_at": "2026-11-19T18:03:44.073042Z" + }, + { + "id": "01m7tfn4dbd6xggqf1pam9zeq7", + "sequence": 11, + "event_type": "UpdateApplied", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfn262t4er6jqsrrkc0tph", + "type": "workflow", + "queue": "rv307-3c487c", + "status": "leased", + "leased_at": "2026-11-19T18:03:44.196073Z", + "connection": "mysql", + "lease_owner": "rv307-w-b9f827", + "available_at": "2026-11-19T18:03:42.145784Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:54.195943Z", + "last_dispatched_at": "2026-11-19T18:03:42.195663Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:42.195663Z" + }, + "command": { + "id": "01m7tfn424cxrk1nfwx2sn4mz3", + "type": "update", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNw4GCG5hbWUKDHJlcG9ydBJhcmd1bWVudHMMACJ2YWxpZGF0aW9uX2Vycm9ycwwAAA==" + }, + "sequence": 2, + "request_id": "sdk-python-update-928a030fbcde4de3a147d7549ac58407", + "accepted_at": "2026-11-19T18:03:44.065158Z", + "auth_method": "token", + "auth_status": "authorized", + "target_name": "report", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows/rv307-rv348_plain-021643/update/report", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfn1drg737gpdqqq521f99", + "message_sequence": 1, + "request_route_name": "generated::YF2YckSNfLzTOm1b", + "request_fingerprint": "sha256:b3ab45962900874a3816d6d428ad736697abec828ed736a7a1f54370266f35b9" + }, + "sequence": 4, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "update_id": "01m7tfn425egqb0gwrxr2vry0f", + "update_name": "report", + "workflow_run_id": "01m7tfn1drg737gpdqqq521f99", + "workflow_command_id": "01m7tfn424cxrk1nfwx2sn4mz3", + "workflow_instance_id": "rv307-rv348_plain-021643" + }, + "workflow_task_id": "01m7tfn262t4er6jqsrrkc0tph", + "workflow_command_id": "01m7tfn424cxrk1nfwx2sn4mz3", + "recorded_at": "2026-11-19T18:03:44.426943Z" + }, + { + "id": "01m7tfn4dfff9acyj7phdaf0qr", + "sequence": 12, + "event_type": "UpdateCompleted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfn262t4er6jqsrrkc0tph", + "type": "workflow", + "queue": "rv307-3c487c", + "status": "leased", + "leased_at": "2026-11-19T18:03:44.196073Z", + "connection": "mysql", + "lease_owner": "rv307-w-b9f827", + "available_at": "2026-11-19T18:03:42.145784Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:54.195943Z", + "last_dispatched_at": "2026-11-19T18:03:42.195663Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:42.195663Z" + }, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNw4CGGNvbXBlbnNhdGluZwoOZmFpbC1tMQA=" + }, + "command": { + "id": "01m7tfn424cxrk1nfwx2sn4mz3", + "type": "update", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNw4GCG5hbWUKDHJlcG9ydBJhcmd1bWVudHMMACJ2YWxpZGF0aW9uX2Vycm9ycwwAAA==" + }, + "sequence": 2, + "request_id": "sdk-python-update-928a030fbcde4de3a147d7549ac58407", + "accepted_at": "2026-11-19T18:03:44.065158Z", + "auth_method": "token", + "auth_status": "authorized", + "target_name": "report", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows/rv307-rv348_plain-021643/update/report", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfn1drg737gpdqqq521f99", + "message_sequence": 1, + "request_route_name": "generated::YF2YckSNfLzTOm1b", + "request_fingerprint": "sha256:b3ab45962900874a3816d6d428ad736697abec828ed736a7a1f54370266f35b9" + }, + "sequence": 4, + "update_id": "01m7tfn425egqb0gwrxr2vry0f", + "update_name": "report", + "workflow_run_id": "01m7tfn1drg737gpdqqq521f99", + "workflow_command_id": "01m7tfn424cxrk1nfwx2sn4mz3", + "workflow_instance_id": "rv307-rv348_plain-021643" + }, + "workflow_task_id": "01m7tfn262t4er6jqsrrkc0tph", + "workflow_command_id": "01m7tfn424cxrk1nfwx2sn4mz3", + "recorded_at": "2026-11-19T18:03:44.430300Z" + }, + { + "id": "01m7tfn4dkxb7vmv12z8w7f9h0", + "sequence": 13, + "event_type": "MessageCursorAdvanced", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfn262t4er6jqsrrkc0tph", + "type": "workflow", + "queue": "rv307-3c487c", + "status": "leased", + "leased_at": "2026-11-19T18:03:44.196073Z", + "connection": "mysql", + "lease_owner": "rv307-w-b9f827", + "available_at": "2026-11-19T18:03:42.145784Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:54.195943Z", + "last_dispatched_at": "2026-11-19T18:03:42.195663Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:42.195663Z" + }, + "stream_key": "instance:rv307-rv348_plain-021643", + "new_position": 1, + "previous_position": 0 + }, + "workflow_task_id": "01m7tfn262t4er6jqsrrkc0tph", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:44.435034Z" + }, + { + "id": "01m7tfn4vda2w5frne75t80a97", + "sequence": 14, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfn3fm9p9mtajg5jf0dza3", + "type": "workflow", + "queue": "rv307-3c487c", + "status": "leased", + "leased_at": "2026-11-19T18:03:44.496300Z", + "connection": "mysql", + "lease_owner": "rv307-w-b9f827", + "available_at": "2026-11-19T18:03:43.475887Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:54.496269Z", + "last_dispatched_at": "2026-11-19T18:03:43.505829Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:43.505829Z" + }, + "activity": { + "id": "01m7tfn4vbn5sf1h1yn8n4hhrq", + "type": "compensate", + "class": "compensate", + "queue": "rv307-3c487c", + "status": "pending", + "sequence": 4, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCCg5mYWlsLW0xAA==" + }, + "created_at": "2026-11-19T18:03:44.875794Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfn4vbn5sf1h1yn8n4hhrq" + }, + "sequence": 4, + "activity_type": "compensate", + "activity_class": "compensate", + "activity_execution_id": "01m7tfn4vbn5sf1h1yn8n4hhrq" + }, + "workflow_task_id": "01m7tfn3fm9p9mtajg5jf0dza3", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:44.876814Z" + }, + { + "id": "01m7tfn6z0t2wp7y3x5bf5pkjg", + "sequence": 15, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfn4vfecsge06pr5gtfe7w", + "type": "activity", + "queue": "rv307-3c487c", + "status": "leased", + "leased_at": "2026-11-19T18:03:47.034837Z", + "connection": "mysql", + "lease_owner": "rv307-w-b9f827", + "available_at": "2026-11-19T18:03:44.879382Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:47.034929Z" + }, + "activity": { + "id": "01m7tfn4vbn5sf1h1yn8n4hhrq", + "type": "compensate", + "class": "compensate", + "queue": "rv307-3c487c", + "status": "running", + "sequence": 4, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCCg5mYWlsLW0xAA==" + }, + "attempt_id": "01M7TFN6YTGS8HKSTH82905RXS", + "created_at": "2026-11-19T18:03:44.875794Z", + "started_at": "2026-11-19T18:03:47.034837Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfn4vbn5sf1h1yn8n4hhrq" + }, + "sequence": 4, + "activity_type": "compensate", + "activity_class": "compensate", + "attempt_number": 1, + "activity_attempt_id": "01M7TFN6YTGS8HKSTH82905RXS", + "activity_execution_id": "01m7tfn4vbn5sf1h1yn8n4hhrq" + }, + "workflow_task_id": "01m7tfn4vfecsge06pr5gtfe7w", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:47.039650Z" + }, + { + "id": "01m7tfn71trh1rg3tvxrxdzy6p", + "sequence": 16, + "event_type": "ActivityCompleted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfn4vfecsge06pr5gtfe7w", + "type": "activity", + "queue": "rv307-3c487c", + "status": "leased", + "leased_at": "2026-11-19T18:03:47.034837Z", + "connection": "mysql", + "lease_owner": "rv307-w-b9f827", + "available_at": "2026-11-19T18:03:44.879382Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:47.102673Z" + }, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoMdW5kb25l" + }, + "activity": { + "id": "01m7tfn4vbn5sf1h1yn8n4hhrq", + "type": "compensate", + "class": "compensate", + "queue": "rv307-3c487c", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoMdW5kb25l" + }, + "status": "completed", + "sequence": 4, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCCg5mYWlsLW0xAA==" + }, + "closed_at": "2026-11-19T18:03:47.128561Z", + "attempt_id": "01M7TFN6YTGS8HKSTH82905RXS", + "created_at": "2026-11-19T18:03:44.875794Z", + "started_at": "2026-11-19T18:03:47.034837Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfn4vbn5sf1h1yn8n4hhrq" + }, + "sequence": 4, + "activity_type": "compensate", + "payload_codec": "avro", + "activity_class": "compensate", + "attempt_number": 1, + "activity_attempt_id": "01M7TFN6YTGS8HKSTH82905RXS", + "activity_execution_id": "01m7tfn4vbn5sf1h1yn8n4hhrq" + }, + "workflow_task_id": "01m7tfn4vfecsge06pr5gtfe7w", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:47.129768Z" + } + ], + "expected": { + "command_sequence": [ + { + "type": "complete_workflow" + } + ] + } +} diff --git a/tests/fixtures/replay_regressions/failure-order-new-run-zero-command-catch-complete-query.json b/tests/fixtures/replay_regressions/failure-order-new-run-zero-command-catch-complete-query.json new file mode 100644 index 00000000..f8f99a79 --- /dev/null +++ b/tests/fixtures/replay_regressions/failure-order-new-run-zero-command-catch-complete-query.json @@ -0,0 +1,1327 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "failure-order-new-run-zero-command-catch-complete-query", + "protocol_version": "1.20", + "bindings": [ + "python" + ], + "parallel_failure_order": { + "issues": [ + "durable-workflow/sdk-python#307" + ], + "note": "The history the last task of failure-order-new-run-zero-command-catch received on a Server built from durable-workflow/server main 52e3ec00 (docker-compose.test.yml). Replay completes it with fail-m1. Published 2.19.1 refuses this history as nondeterministic at the marker." + }, + "workflow": { + "type": "rv348_handle", + "input": [], + "payload_codec": "avro", + "run_id": "01m7tfmajd2mzmq8bjygdsf5an", + "workflow_id": "rv307-rv348_handle-1c2847", + "query": { + "name": "caught", + "arguments": [] + } + }, + "history": [ + { + "id": "01m7tfmajjgp5q69b444d8cbzz", + "sequence": 1, + "event_type": "StartAccepted", + "namespace": "default", + "payload": { + "memo": null, + "command": { + "id": "01m7tfmajf35nsz3zz911ttetj", + "type": "start", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "outcome": "started_new", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "sequence": 1, + "applied_at": "2026-11-19T18:03:17.966013Z", + "accepted_at": "2026-11-19T18:03:17.966010Z", + "auth_method": "token", + "auth_status": "authorized", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfmajd2mzmq8bjygdsf5an", + "request_route_name": "generated::FqJuJMl97iiXlIDz", + "request_fingerprint": "sha256:f47aee00c4c048c39f0c6e7bba8ebada52b7e85b45eeb39b592043178306f9e4" + }, + "outcome": "started_new", + "business_key": null, + "workflow_type": "rv348_handle", + "workflow_class": "rv348_handle", + "workflow_run_id": "01m7tfmajd2mzmq8bjygdsf5an", + "search_attributes": null, + "visibility_labels": null, + "workflow_command_id": "01m7tfmajf35nsz3zz911ttetj", + "workflow_instance_id": "rv307-rv348_handle-1c2847" + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfmajf35nsz3zz911ttetj", + "recorded_at": "2026-11-19T18:03:17.969955Z" + }, + { + "id": "01m7tfmajrzvkk20crm5f4ptrz", + "sequence": 2, + "event_type": "WorkflowStarted", + "namespace": "default", + "payload": { + "command": { + "id": "01m7tfmajf35nsz3zz911ttetj", + "type": "start", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "outcome": "started_new", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "sequence": 1, + "applied_at": "2026-11-19T18:03:17.966013Z", + "accepted_at": "2026-11-19T18:03:17.966010Z", + "auth_method": "token", + "auth_status": "authorized", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfmajd2mzmq8bjygdsf5an", + "request_route_name": "generated::FqJuJMl97iiXlIDz", + "request_fingerprint": "sha256:f47aee00c4c048c39f0c6e7bba8ebada52b7e85b45eeb39b592043178306f9e4" + }, + "workflow_type": "rv348_handle", + "workflow_class": "rv348_handle", + "run_deadline_at": "2026-11-19T18:13:17+00:00", + "workflow_run_id": "01m7tfmajd2mzmq8bjygdsf5an", + "declared_queries": [ + "caught" + ], + "declared_signals": [], + "declared_updates": [ + "report" + ], + "declared_entry_mode": "canonical", + "run_timeout_seconds": 600, + "workflow_command_id": "01m7tfmajf35nsz3zz911ttetj", + "workflow_instance_id": "rv307-rv348_handle-1c2847", + "declared_entry_method": "handle", + "execution_deadline_at": "2026-11-19T19:03:17+00:00", + "declared_query_contracts": [ + { + "name": "caught", + "parameters": [] + } + ], + "declared_signal_contracts": [], + "declared_update_contracts": [ + { + "name": "report", + "parameters": [ + { + "name": "_", + "type": null, + "default": null, + "position": 0, + "required": false, + "variadic": true, + "allows_null": true, + "default_available": false + } + ] + } + ], + "execution_timeout_seconds": 3600, + "declared_update_validators": [], + "declared_entry_declaring_class": "external:rv348_handle", + "workflow_definition_fingerprint": "sha256:69ed762d44804878ebb703a2ad385119f6f776d0dae561c15f8f84e1707a704a", + "workflow_definition_fingerprint_source": "worker" + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfmajf35nsz3zz911ttetj", + "recorded_at": "2026-11-19T18:03:17.974885Z" + }, + { + "id": "01m7tfmb2j0k3f4w32jxb48e1w", + "sequence": 3, + "event_type": "TimerScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmajt78z8y51sj6e91dwk", + "type": "workflow", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:18.040583Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:17.978566Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:28.040524Z", + "last_dispatched_at": "2026-11-19T18:03:17.982237Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:17.982237Z" + }, + "fire_at": "2026-11-19T18:03:19.478674Z", + "sequence": 1, + "timer_id": "01m7tfmb2ec1j7gtczamgd4nyw", + "delay_seconds": 1, + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + }, + "workflow_task_id": "01m7tfmajt78z8y51sj6e91dwk", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:18.480737Z" + }, + { + "id": "01m7tfmb2vs6syc1x683pty7ew", + "sequence": 4, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmajt78z8y51sj6e91dwk", + "type": "workflow", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:18.040583Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:17.978566Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:28.040524Z", + "last_dispatched_at": "2026-11-19T18:03:17.982237Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:17.982237Z" + }, + "activity": { + "id": "01m7tfmb2r4yykrxz5ms6mxh7e", + "type": "slow", + "class": "slow", + "queue": "rv307-95d208", + "status": "pending", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "created_at": "2026-11-19T18:03:18.488686Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmb2r4yykrxz5ms6mxh7e" + }, + "sequence": 2, + "activity_type": "slow", + "activity_class": "slow", + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "activity_execution_id": "01m7tfmb2r4yykrxz5ms6mxh7e", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + }, + "workflow_task_id": "01m7tfmajt78z8y51sj6e91dwk", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:18.490538Z" + }, + { + "id": "01m7tfmd1558qccnq7tp4gx0rr", + "sequence": 5, + "event_type": "TimerFired", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmb2ncmszyfrjdst60e59", + "type": "timer", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:20.466569Z", + "connection": "mysql", + "lease_owner": "01m7tfmb2ncmszyfrjdst60e59", + "available_at": "2026-11-19T18:03:19.478674Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:30.466572Z" + }, + "fire_at": "2026-11-19T18:03:19.478674Z", + "fired_at": "2026-11-19T18:03:20.484504Z", + "sequence": 1, + "timer_id": "01m7tfmb2ec1j7gtczamgd4nyw", + "delay_seconds": 1, + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + }, + "workflow_task_id": "01m7tfmb2ncmszyfrjdst60e59", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:20.485608Z" + }, + { + "id": "01m7tfmd1921d4bkmatrk6ebdc", + "sequence": 6, + "event_type": "SelectionResolved", + "namespace": "default", + "payload": { + "outcome": "completed", + "member_key": "t", + "member_size": 1, + "member_index": 0, + "operation_kind": "timer", + "operation_identity": "01m7tfmb2ec1j7gtczamgd4nyw", + "selection_group_id": "select-calls:1:2", + "resolution_event_id": "01m7tfmd1558qccnq7tp4gx0rr", + "member_base_sequence": 1, + "selection_group_size": 2, + "resolution_event_type": "TimerFired", + "selection_group_base_sequence": 1 + }, + "workflow_task_id": null, + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:20.489579Z" + }, + { + "id": "01m7tfme1bwdba5r29kxxd6cf2", + "sequence": 7, + "event_type": "TimerScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmd1cj5fqtp8jwx6ct2mw", + "type": "workflow", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:21.093175Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:20.491921Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:31.093133Z", + "last_dispatched_at": "2026-11-19T18:03:20.510053Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:20.510053Z" + }, + "fire_at": "2026-11-19T18:03:22.512049Z", + "sequence": 3, + "timer_id": "01m7tfme183tr41x27sy93ae5t", + "delay_seconds": 1 + }, + "workflow_task_id": "01m7tfmd1cj5fqtp8jwx6ct2mw", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:21.514136Z" + }, + { + "id": "01m7tfmg2f61p1djskt19m30dx", + "sequence": 8, + "event_type": "TimerFired", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfme1ef1bzv4s1q937682x", + "type": "timer", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:23.566024Z", + "connection": "mysql", + "lease_owner": "01m7tfme1ef1bzv4s1q937682x", + "available_at": "2026-11-19T18:03:22.512049Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:33.566032Z" + }, + "fire_at": "2026-11-19T18:03:22.512049Z", + "fired_at": "2026-11-19T18:03:23.597085Z", + "sequence": 3, + "timer_id": "01m7tfme183tr41x27sy93ae5t", + "delay_seconds": 1 + }, + "workflow_task_id": "01m7tfme1ef1bzv4s1q937682x", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:23.599095Z" + }, + { + "id": "01m7tfmh0ewhpdffmw6f53xw1t", + "sequence": 9, + "event_type": "VersionMarkerRecorded", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmg2k7zfpt95rnxpnttbd", + "type": "workflow", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:24.111580Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:23.603148Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:34.111540Z", + "last_dispatched_at": "2026-11-19T18:03:23.630339Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:23.630339Z" + }, + "version": 1, + "sequence": 4, + "change_id": "durable_workflow.parallel_failure_order", + "max_supported": 1, + "min_supported": 1 + }, + "workflow_task_id": "01m7tfmg2k7zfpt95rnxpnttbd", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:24.556837Z" + }, + { + "id": "01m7tfmh0p07a6z70f3arbd67m", + "sequence": 10, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmg2k7zfpt95rnxpnttbd", + "type": "workflow", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:24.111580Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:23.603148Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:34.111540Z", + "last_dispatched_at": "2026-11-19T18:03:23.630339Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:23.630339Z" + }, + "activity": { + "id": "01m7tfmh0k7bth61zv6erffdnr", + "type": "m0", + "class": "m0", + "queue": "rv307-95d208", + "status": "pending", + "sequence": 5, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "created_at": "2026-11-19T18:03:24.563929Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmh0k7bth61zv6erffdnr" + }, + "sequence": 5, + "activity_type": "m0", + "activity_class": "m0", + "parallel_group_id": "parallel-activities:5:2", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:5:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 5 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfmh0k7bth61zv6erffdnr", + "parallel_group_base_sequence": 5 + }, + "workflow_task_id": "01m7tfmg2k7zfpt95rnxpnttbd", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:24.565789Z" + }, + { + "id": "01m7tfmh0vyz2awxy1qfqtytnp", + "sequence": 11, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmg2k7zfpt95rnxpnttbd", + "type": "workflow", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:24.111580Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:23.603148Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:34.111540Z", + "last_dispatched_at": "2026-11-19T18:03:23.630339Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:23.630339Z" + }, + "activity": { + "id": "01m7tfmh0s97v0b4ekhqj06q1z", + "type": "m1", + "class": "m1", + "queue": "rv307-95d208", + "status": "pending", + "sequence": 6, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "created_at": "2026-11-19T18:03:24.569756Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmh0s97v0b4ekhqj06q1z" + }, + "sequence": 6, + "activity_type": "m1", + "activity_class": "m1", + "parallel_group_id": "parallel-activities:5:2", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:5:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 5 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfmh0s97v0b4ekhqj06q1z", + "parallel_group_base_sequence": 5 + }, + "workflow_task_id": "01m7tfmg2k7zfpt95rnxpnttbd", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:24.570640Z" + }, + { + "id": "01m7tfmh3hhn1f2nskrftt3p0h", + "sequence": 12, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmb2y6dwd7js9zzj1d7bw", + "type": "activity", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:24.651495Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:18.494356Z", + "repair_count": 1, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:24.651591Z", + "last_dispatched_at": "2026-11-19T18:03:24.486915Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:24.486915Z" + }, + "activity": { + "id": "01m7tfmb2r4yykrxz5ms6mxh7e", + "type": "slow", + "class": "slow", + "queue": "rv307-95d208", + "status": "running", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "attempt_id": "01M7TFMH3BKEDQ2AT39KAVBR6V", + "created_at": "2026-11-19T18:03:18.488686Z", + "started_at": "2026-11-19T18:03:24.651495Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmb2r4yykrxz5ms6mxh7e" + }, + "sequence": 2, + "activity_type": "slow", + "activity_class": "slow", + "attempt_number": 1, + "parallel_group_id": "select-calls:1:2", + "activity_attempt_id": "01M7TFMH3BKEDQ2AT39KAVBR6V", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "activity_execution_id": "01m7tfmb2r4yykrxz5ms6mxh7e", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + }, + "workflow_task_id": "01m7tfmb2y6dwd7js9zzj1d7bw", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:24.656289Z" + }, + { + "id": "01m7tfmh7q122cv6edc7013fpb", + "sequence": 13, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmh0rce8ey59n2b3gsmae", + "type": "activity", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:24.785950Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:24.567858Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:24.786065Z" + }, + "activity": { + "id": "01m7tfmh0k7bth61zv6erffdnr", + "type": "m0", + "class": "m0", + "queue": "rv307-95d208", + "status": "running", + "sequence": 5, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "attempt_id": "01M7TFMH7JE9SDVJ7VR807EVPT", + "created_at": "2026-11-19T18:03:24.563929Z", + "started_at": "2026-11-19T18:03:24.785950Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmh0k7bth61zv6erffdnr" + }, + "sequence": 5, + "activity_type": "m0", + "activity_class": "m0", + "attempt_number": 1, + "parallel_group_id": "parallel-activities:5:2", + "activity_attempt_id": "01M7TFMH7JE9SDVJ7VR807EVPT", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:5:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 5 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfmh0k7bth61zv6erffdnr", + "parallel_group_base_sequence": 5 + }, + "workflow_task_id": "01m7tfmh0rce8ey59n2b3gsmae", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:24.790941Z" + }, + { + "id": "01m7tfmhasmzqs85em7rpmsq11", + "sequence": 14, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmh0w83cfz7kg6z4jxzqm", + "type": "activity", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:24.885211Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:24.572340Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:24.885300Z" + }, + "activity": { + "id": "01m7tfmh0s97v0b4ekhqj06q1z", + "type": "m1", + "class": "m1", + "queue": "rv307-95d208", + "status": "running", + "sequence": 6, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "attempt_id": "01M7TFMHAN3Y9AK69R60K108T9", + "created_at": "2026-11-19T18:03:24.569756Z", + "started_at": "2026-11-19T18:03:24.885211Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmh0s97v0b4ekhqj06q1z" + }, + "sequence": 6, + "activity_type": "m1", + "activity_class": "m1", + "attempt_number": 1, + "parallel_group_id": "parallel-activities:5:2", + "activity_attempt_id": "01M7TFMHAN3Y9AK69R60K108T9", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:5:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 5 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfmh0s97v0b4ekhqj06q1z", + "parallel_group_base_sequence": 5 + }, + "workflow_task_id": "01m7tfmh0w83cfz7kg6z4jxzqm", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:24.889189Z" + }, + { + "id": "01m7tfmheabrw4zq6gdmnyw0q6", + "sequence": 15, + "event_type": "ActivityFailed", + "namespace": "default", + "payload": { + "code": 0, + "task": { + "id": "01m7tfmh0w83cfz7kg6z4jxzqm", + "type": "activity", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:24.885211Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:24.572340Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:24.958755Z" + }, + "message": "fail-m1", + "activity": { + "id": "01m7tfmh0s97v0b4ekhqj06q1z", + "type": "m1", + "class": "m1", + "queue": "rv307-95d208", + "status": "failed", + "sequence": 6, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "closed_at": "2026-11-19T18:03:25.000007Z", + "exception": { + "code": 0, + "type": null, + "message": "fail-m1", + "non_retryable": true + }, + "attempt_id": "01M7TFMHAN3Y9AK69R60K108T9", + "created_at": "2026-11-19T18:03:24.569756Z", + "started_at": "2026-11-19T18:03:24.885211Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmh0s97v0b4ekhqj06q1z" + }, + "sequence": 6, + "exception": { + "type": null, + "message": "fail-m1", + "code": 0, + "non_retryable": true + }, + "failure_id": "01m7tfmhe7jx4mjg3xfm1dt4t8", + "activity_type": "m1", + "non_retryable": true, + "activity_class": "m1", + "attempt_number": 1, + "exception_type": null, + "exception_class": "RuntimeException", + "failure_category": "activity", + "parallel_group_id": "parallel-activities:5:2", + "activity_attempt_id": "01M7TFMHAN3Y9AK69R60K108T9", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:5:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 5 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfmh0s97v0b4ekhqj06q1z", + "parallel_group_base_sequence": 5 + }, + "workflow_task_id": "01m7tfmh0w83cfz7kg6z4jxzqm", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:25.001498Z" + }, + { + "id": "01m7tfmjq52v4y8q4sw055kg8r", + "sequence": 16, + "event_type": "ActivityFailed", + "namespace": "default", + "payload": { + "code": 0, + "task": { + "id": "01m7tfmh0rce8ey59n2b3gsmae", + "type": "activity", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:24.785950Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:24.567858Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:24.848965Z" + }, + "message": "fail-m0", + "activity": { + "id": "01m7tfmh0k7bth61zv6erffdnr", + "type": "m0", + "class": "m0", + "queue": "rv307-95d208", + "status": "failed", + "sequence": 5, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "closed_at": "2026-11-19T18:03:26.307727Z", + "exception": { + "code": 0, + "type": null, + "message": "fail-m0", + "non_retryable": true + }, + "attempt_id": "01M7TFMH7JE9SDVJ7VR807EVPT", + "created_at": "2026-11-19T18:03:24.563929Z", + "started_at": "2026-11-19T18:03:24.785950Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmh0k7bth61zv6erffdnr" + }, + "sequence": 5, + "exception": { + "type": null, + "message": "fail-m0", + "code": 0, + "non_retryable": true + }, + "failure_id": "01m7tfmjq3pnbgqfk7vy8pfqjq", + "activity_type": "m0", + "non_retryable": true, + "activity_class": "m0", + "attempt_number": 1, + "exception_type": null, + "exception_class": "RuntimeException", + "failure_category": "activity", + "parallel_group_id": "parallel-activities:5:2", + "activity_attempt_id": "01M7TFMH7JE9SDVJ7VR807EVPT", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:5:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 5 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfmh0k7bth61zv6erffdnr", + "parallel_group_base_sequence": 5 + }, + "workflow_task_id": "01m7tfmh0rce8ey59n2b3gsmae", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:26.309124Z" + }, + { + "id": "01m7tfmkspws6jrb7zgqgkb58a", + "sequence": 17, + "event_type": "ActivityCompleted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmb2y6dwd7js9zzj1d7bw", + "type": "activity", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:24.651495Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:18.494356Z", + "repair_count": 1, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:24.745799Z", + "last_dispatched_at": "2026-11-19T18:03:24.486915Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:24.486915Z" + }, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoEb2s=" + }, + "activity": { + "id": "01m7tfmb2r4yykrxz5ms6mxh7e", + "type": "slow", + "class": "slow", + "queue": "rv307-95d208", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoEb2s=" + }, + "status": "completed", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "closed_at": "2026-11-19T18:03:27.411433Z", + "attempt_id": "01M7TFMH3BKEDQ2AT39KAVBR6V", + "created_at": "2026-11-19T18:03:18.488686Z", + "started_at": "2026-11-19T18:03:24.651495Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmb2r4yykrxz5ms6mxh7e" + }, + "sequence": 2, + "activity_type": "slow", + "payload_codec": "avro", + "activity_class": "slow", + "attempt_number": 1, + "parallel_group_id": "select-calls:1:2", + "activity_attempt_id": "01M7TFMH3BKEDQ2AT39KAVBR6V", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "activity_execution_id": "01m7tfmb2r4yykrxz5ms6mxh7e", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + }, + "workflow_task_id": "01m7tfmb2y6dwd7js9zzj1d7bw", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:27.413691Z" + }, + { + "id": "01m7tfmmbpcn9ynjszjj5ype78", + "sequence": 18, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmjqb26py8pavrcy62ewh", + "type": "workflow", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:27.545637Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:26.314879Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:37.545600Z", + "last_dispatched_at": "2026-11-19T18:03:26.344774Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:26.344774Z" + }, + "activity": { + "id": "01m7tfmmbkgrbna0pf56qrg0rq", + "type": "compensate", + "class": "compensate", + "queue": "rv307-95d208", + "status": "pending", + "sequence": 7, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCCg5mYWlsLW0xAA==" + }, + "created_at": "2026-11-19T18:03:27.987490Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmmbkgrbna0pf56qrg0rq" + }, + "sequence": 7, + "activity_type": "compensate", + "activity_class": "compensate", + "activity_execution_id": "01m7tfmmbkgrbna0pf56qrg0rq" + }, + "workflow_task_id": "01m7tfmjqb26py8pavrcy62ewh", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:27.989472Z" + }, + { + "id": "01m7tfmphv8902yrdvqezkq2pv", + "sequence": 19, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmmbt42zy7fb4awkmq8pw", + "type": "activity", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:30.216603Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:27.993985Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:30.216970Z" + }, + "activity": { + "id": "01m7tfmmbkgrbna0pf56qrg0rq", + "type": "compensate", + "class": "compensate", + "queue": "rv307-95d208", + "status": "running", + "sequence": 7, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCCg5mYWlsLW0xAA==" + }, + "attempt_id": "01M7TFMPH8CWTANT27NX8VWT2Q", + "created_at": "2026-11-19T18:03:27.987490Z", + "started_at": "2026-11-19T18:03:30.216603Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmmbkgrbna0pf56qrg0rq" + }, + "sequence": 7, + "activity_type": "compensate", + "activity_class": "compensate", + "attempt_number": 1, + "activity_attempt_id": "01M7TFMPH8CWTANT27NX8VWT2Q", + "activity_execution_id": "01m7tfmmbkgrbna0pf56qrg0rq" + }, + "workflow_task_id": "01m7tfmmbt42zy7fb4awkmq8pw", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:30.233123Z" + }, + { + "id": "01m7tfmpn3yy98fedb7nq4xm38", + "sequence": 20, + "event_type": "ActivityCompleted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmmbt42zy7fb4awkmq8pw", + "type": "activity", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:30.216603Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:27.993985Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:30.306166Z" + }, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoMdW5kb25l" + }, + "activity": { + "id": "01m7tfmmbkgrbna0pf56qrg0rq", + "type": "compensate", + "class": "compensate", + "queue": "rv307-95d208", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoMdW5kb25l" + }, + "status": "completed", + "sequence": 7, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCCg5mYWlsLW0xAA==" + }, + "closed_at": "2026-11-19T18:03:30.337850Z", + "attempt_id": "01M7TFMPH8CWTANT27NX8VWT2Q", + "created_at": "2026-11-19T18:03:27.987490Z", + "started_at": "2026-11-19T18:03:30.216603Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmmbkgrbna0pf56qrg0rq" + }, + "sequence": 7, + "activity_type": "compensate", + "payload_codec": "avro", + "activity_class": "compensate", + "attempt_number": 1, + "activity_attempt_id": "01M7TFMPH8CWTANT27NX8VWT2Q", + "activity_execution_id": "01m7tfmmbkgrbna0pf56qrg0rq" + }, + "workflow_task_id": "01m7tfmmbt42zy7fb4awkmq8pw", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:30.339364Z" + } + ], + "expected": { + "query_result": "fail-m1" + } +} diff --git a/tests/fixtures/replay_regressions/failure-order-new-run-zero-command-catch-complete.json b/tests/fixtures/replay_regressions/failure-order-new-run-zero-command-catch-complete.json new file mode 100644 index 00000000..7bd67711 --- /dev/null +++ b/tests/fixtures/replay_regressions/failure-order-new-run-zero-command-catch-complete.json @@ -0,0 +1,1327 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "failure-order-new-run-zero-command-catch-complete", + "protocol_version": "1.20", + "bindings": [ + "python" + ], + "parallel_failure_order": { + "issues": [ + "durable-workflow/sdk-python#307" + ], + "note": "The history the last task of failure-order-new-run-zero-command-catch received on a Server built from durable-workflow/server main 52e3ec00 (docker-compose.test.yml). Replay completes it with fail-m1. Published 2.19.1 refuses this history as nondeterministic at the marker." + }, + "workflow": { + "type": "rv348_handle", + "input": [], + "payload_codec": "avro", + "run_id": "01m7tfmajd2mzmq8bjygdsf5an", + "workflow_id": "rv307-rv348_handle-1c2847" + }, + "history": [ + { + "id": "01m7tfmajjgp5q69b444d8cbzz", + "sequence": 1, + "event_type": "StartAccepted", + "namespace": "default", + "payload": { + "memo": null, + "command": { + "id": "01m7tfmajf35nsz3zz911ttetj", + "type": "start", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "outcome": "started_new", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "sequence": 1, + "applied_at": "2026-11-19T18:03:17.966013Z", + "accepted_at": "2026-11-19T18:03:17.966010Z", + "auth_method": "token", + "auth_status": "authorized", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfmajd2mzmq8bjygdsf5an", + "request_route_name": "generated::FqJuJMl97iiXlIDz", + "request_fingerprint": "sha256:f47aee00c4c048c39f0c6e7bba8ebada52b7e85b45eeb39b592043178306f9e4" + }, + "outcome": "started_new", + "business_key": null, + "workflow_type": "rv348_handle", + "workflow_class": "rv348_handle", + "workflow_run_id": "01m7tfmajd2mzmq8bjygdsf5an", + "search_attributes": null, + "visibility_labels": null, + "workflow_command_id": "01m7tfmajf35nsz3zz911ttetj", + "workflow_instance_id": "rv307-rv348_handle-1c2847" + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfmajf35nsz3zz911ttetj", + "recorded_at": "2026-11-19T18:03:17.969955Z" + }, + { + "id": "01m7tfmajrzvkk20crm5f4ptrz", + "sequence": 2, + "event_type": "WorkflowStarted", + "namespace": "default", + "payload": { + "command": { + "id": "01m7tfmajf35nsz3zz911ttetj", + "type": "start", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "outcome": "started_new", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "sequence": 1, + "applied_at": "2026-11-19T18:03:17.966013Z", + "accepted_at": "2026-11-19T18:03:17.966010Z", + "auth_method": "token", + "auth_status": "authorized", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfmajd2mzmq8bjygdsf5an", + "request_route_name": "generated::FqJuJMl97iiXlIDz", + "request_fingerprint": "sha256:f47aee00c4c048c39f0c6e7bba8ebada52b7e85b45eeb39b592043178306f9e4" + }, + "workflow_type": "rv348_handle", + "workflow_class": "rv348_handle", + "run_deadline_at": "2026-11-19T18:13:17+00:00", + "workflow_run_id": "01m7tfmajd2mzmq8bjygdsf5an", + "declared_queries": [ + "caught" + ], + "declared_signals": [], + "declared_updates": [ + "report" + ], + "declared_entry_mode": "canonical", + "run_timeout_seconds": 600, + "workflow_command_id": "01m7tfmajf35nsz3zz911ttetj", + "workflow_instance_id": "rv307-rv348_handle-1c2847", + "declared_entry_method": "handle", + "execution_deadline_at": "2026-11-19T19:03:17+00:00", + "declared_query_contracts": [ + { + "name": "caught", + "parameters": [] + } + ], + "declared_signal_contracts": [], + "declared_update_contracts": [ + { + "name": "report", + "parameters": [ + { + "name": "_", + "type": null, + "default": null, + "position": 0, + "required": false, + "variadic": true, + "allows_null": true, + "default_available": false + } + ] + } + ], + "execution_timeout_seconds": 3600, + "declared_update_validators": [], + "declared_entry_declaring_class": "external:rv348_handle", + "workflow_definition_fingerprint": "sha256:69ed762d44804878ebb703a2ad385119f6f776d0dae561c15f8f84e1707a704a", + "workflow_definition_fingerprint_source": "worker" + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfmajf35nsz3zz911ttetj", + "recorded_at": "2026-11-19T18:03:17.974885Z" + }, + { + "id": "01m7tfmb2j0k3f4w32jxb48e1w", + "sequence": 3, + "event_type": "TimerScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmajt78z8y51sj6e91dwk", + "type": "workflow", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:18.040583Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:17.978566Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:28.040524Z", + "last_dispatched_at": "2026-11-19T18:03:17.982237Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:17.982237Z" + }, + "fire_at": "2026-11-19T18:03:19.478674Z", + "sequence": 1, + "timer_id": "01m7tfmb2ec1j7gtczamgd4nyw", + "delay_seconds": 1, + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + }, + "workflow_task_id": "01m7tfmajt78z8y51sj6e91dwk", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:18.480737Z" + }, + { + "id": "01m7tfmb2vs6syc1x683pty7ew", + "sequence": 4, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmajt78z8y51sj6e91dwk", + "type": "workflow", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:18.040583Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:17.978566Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:28.040524Z", + "last_dispatched_at": "2026-11-19T18:03:17.982237Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:17.982237Z" + }, + "activity": { + "id": "01m7tfmb2r4yykrxz5ms6mxh7e", + "type": "slow", + "class": "slow", + "queue": "rv307-95d208", + "status": "pending", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "created_at": "2026-11-19T18:03:18.488686Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmb2r4yykrxz5ms6mxh7e" + }, + "sequence": 2, + "activity_type": "slow", + "activity_class": "slow", + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "activity_execution_id": "01m7tfmb2r4yykrxz5ms6mxh7e", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + }, + "workflow_task_id": "01m7tfmajt78z8y51sj6e91dwk", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:18.490538Z" + }, + { + "id": "01m7tfmd1558qccnq7tp4gx0rr", + "sequence": 5, + "event_type": "TimerFired", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmb2ncmszyfrjdst60e59", + "type": "timer", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:20.466569Z", + "connection": "mysql", + "lease_owner": "01m7tfmb2ncmszyfrjdst60e59", + "available_at": "2026-11-19T18:03:19.478674Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:30.466572Z" + }, + "fire_at": "2026-11-19T18:03:19.478674Z", + "fired_at": "2026-11-19T18:03:20.484504Z", + "sequence": 1, + "timer_id": "01m7tfmb2ec1j7gtczamgd4nyw", + "delay_seconds": 1, + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + }, + "workflow_task_id": "01m7tfmb2ncmszyfrjdst60e59", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:20.485608Z" + }, + { + "id": "01m7tfmd1921d4bkmatrk6ebdc", + "sequence": 6, + "event_type": "SelectionResolved", + "namespace": "default", + "payload": { + "outcome": "completed", + "member_key": "t", + "member_size": 1, + "member_index": 0, + "operation_kind": "timer", + "operation_identity": "01m7tfmb2ec1j7gtczamgd4nyw", + "selection_group_id": "select-calls:1:2", + "resolution_event_id": "01m7tfmd1558qccnq7tp4gx0rr", + "member_base_sequence": 1, + "selection_group_size": 2, + "resolution_event_type": "TimerFired", + "selection_group_base_sequence": 1 + }, + "workflow_task_id": null, + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:20.489579Z" + }, + { + "id": "01m7tfme1bwdba5r29kxxd6cf2", + "sequence": 7, + "event_type": "TimerScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmd1cj5fqtp8jwx6ct2mw", + "type": "workflow", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:21.093175Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:20.491921Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:31.093133Z", + "last_dispatched_at": "2026-11-19T18:03:20.510053Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:20.510053Z" + }, + "fire_at": "2026-11-19T18:03:22.512049Z", + "sequence": 3, + "timer_id": "01m7tfme183tr41x27sy93ae5t", + "delay_seconds": 1 + }, + "workflow_task_id": "01m7tfmd1cj5fqtp8jwx6ct2mw", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:21.514136Z" + }, + { + "id": "01m7tfmg2f61p1djskt19m30dx", + "sequence": 8, + "event_type": "TimerFired", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfme1ef1bzv4s1q937682x", + "type": "timer", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:23.566024Z", + "connection": "mysql", + "lease_owner": "01m7tfme1ef1bzv4s1q937682x", + "available_at": "2026-11-19T18:03:22.512049Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:33.566032Z" + }, + "fire_at": "2026-11-19T18:03:22.512049Z", + "fired_at": "2026-11-19T18:03:23.597085Z", + "sequence": 3, + "timer_id": "01m7tfme183tr41x27sy93ae5t", + "delay_seconds": 1 + }, + "workflow_task_id": "01m7tfme1ef1bzv4s1q937682x", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:23.599095Z" + }, + { + "id": "01m7tfmh0ewhpdffmw6f53xw1t", + "sequence": 9, + "event_type": "VersionMarkerRecorded", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmg2k7zfpt95rnxpnttbd", + "type": "workflow", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:24.111580Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:23.603148Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:34.111540Z", + "last_dispatched_at": "2026-11-19T18:03:23.630339Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:23.630339Z" + }, + "version": 1, + "sequence": 4, + "change_id": "durable_workflow.parallel_failure_order", + "max_supported": 1, + "min_supported": 1 + }, + "workflow_task_id": "01m7tfmg2k7zfpt95rnxpnttbd", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:24.556837Z" + }, + { + "id": "01m7tfmh0p07a6z70f3arbd67m", + "sequence": 10, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmg2k7zfpt95rnxpnttbd", + "type": "workflow", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:24.111580Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:23.603148Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:34.111540Z", + "last_dispatched_at": "2026-11-19T18:03:23.630339Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:23.630339Z" + }, + "activity": { + "id": "01m7tfmh0k7bth61zv6erffdnr", + "type": "m0", + "class": "m0", + "queue": "rv307-95d208", + "status": "pending", + "sequence": 5, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "created_at": "2026-11-19T18:03:24.563929Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmh0k7bth61zv6erffdnr" + }, + "sequence": 5, + "activity_type": "m0", + "activity_class": "m0", + "parallel_group_id": "parallel-activities:5:2", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:5:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 5 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfmh0k7bth61zv6erffdnr", + "parallel_group_base_sequence": 5 + }, + "workflow_task_id": "01m7tfmg2k7zfpt95rnxpnttbd", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:24.565789Z" + }, + { + "id": "01m7tfmh0vyz2awxy1qfqtytnp", + "sequence": 11, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmg2k7zfpt95rnxpnttbd", + "type": "workflow", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:24.111580Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:23.603148Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:34.111540Z", + "last_dispatched_at": "2026-11-19T18:03:23.630339Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:23.630339Z" + }, + "activity": { + "id": "01m7tfmh0s97v0b4ekhqj06q1z", + "type": "m1", + "class": "m1", + "queue": "rv307-95d208", + "status": "pending", + "sequence": 6, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "created_at": "2026-11-19T18:03:24.569756Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmh0s97v0b4ekhqj06q1z" + }, + "sequence": 6, + "activity_type": "m1", + "activity_class": "m1", + "parallel_group_id": "parallel-activities:5:2", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:5:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 5 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfmh0s97v0b4ekhqj06q1z", + "parallel_group_base_sequence": 5 + }, + "workflow_task_id": "01m7tfmg2k7zfpt95rnxpnttbd", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:24.570640Z" + }, + { + "id": "01m7tfmh3hhn1f2nskrftt3p0h", + "sequence": 12, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmb2y6dwd7js9zzj1d7bw", + "type": "activity", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:24.651495Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:18.494356Z", + "repair_count": 1, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:24.651591Z", + "last_dispatched_at": "2026-11-19T18:03:24.486915Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:24.486915Z" + }, + "activity": { + "id": "01m7tfmb2r4yykrxz5ms6mxh7e", + "type": "slow", + "class": "slow", + "queue": "rv307-95d208", + "status": "running", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "attempt_id": "01M7TFMH3BKEDQ2AT39KAVBR6V", + "created_at": "2026-11-19T18:03:18.488686Z", + "started_at": "2026-11-19T18:03:24.651495Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmb2r4yykrxz5ms6mxh7e" + }, + "sequence": 2, + "activity_type": "slow", + "activity_class": "slow", + "attempt_number": 1, + "parallel_group_id": "select-calls:1:2", + "activity_attempt_id": "01M7TFMH3BKEDQ2AT39KAVBR6V", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "activity_execution_id": "01m7tfmb2r4yykrxz5ms6mxh7e", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + }, + "workflow_task_id": "01m7tfmb2y6dwd7js9zzj1d7bw", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:24.656289Z" + }, + { + "id": "01m7tfmh7q122cv6edc7013fpb", + "sequence": 13, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmh0rce8ey59n2b3gsmae", + "type": "activity", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:24.785950Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:24.567858Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:24.786065Z" + }, + "activity": { + "id": "01m7tfmh0k7bth61zv6erffdnr", + "type": "m0", + "class": "m0", + "queue": "rv307-95d208", + "status": "running", + "sequence": 5, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "attempt_id": "01M7TFMH7JE9SDVJ7VR807EVPT", + "created_at": "2026-11-19T18:03:24.563929Z", + "started_at": "2026-11-19T18:03:24.785950Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmh0k7bth61zv6erffdnr" + }, + "sequence": 5, + "activity_type": "m0", + "activity_class": "m0", + "attempt_number": 1, + "parallel_group_id": "parallel-activities:5:2", + "activity_attempt_id": "01M7TFMH7JE9SDVJ7VR807EVPT", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:5:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 5 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfmh0k7bth61zv6erffdnr", + "parallel_group_base_sequence": 5 + }, + "workflow_task_id": "01m7tfmh0rce8ey59n2b3gsmae", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:24.790941Z" + }, + { + "id": "01m7tfmhasmzqs85em7rpmsq11", + "sequence": 14, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmh0w83cfz7kg6z4jxzqm", + "type": "activity", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:24.885211Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:24.572340Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:24.885300Z" + }, + "activity": { + "id": "01m7tfmh0s97v0b4ekhqj06q1z", + "type": "m1", + "class": "m1", + "queue": "rv307-95d208", + "status": "running", + "sequence": 6, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "attempt_id": "01M7TFMHAN3Y9AK69R60K108T9", + "created_at": "2026-11-19T18:03:24.569756Z", + "started_at": "2026-11-19T18:03:24.885211Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmh0s97v0b4ekhqj06q1z" + }, + "sequence": 6, + "activity_type": "m1", + "activity_class": "m1", + "attempt_number": 1, + "parallel_group_id": "parallel-activities:5:2", + "activity_attempt_id": "01M7TFMHAN3Y9AK69R60K108T9", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:5:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 5 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfmh0s97v0b4ekhqj06q1z", + "parallel_group_base_sequence": 5 + }, + "workflow_task_id": "01m7tfmh0w83cfz7kg6z4jxzqm", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:24.889189Z" + }, + { + "id": "01m7tfmheabrw4zq6gdmnyw0q6", + "sequence": 15, + "event_type": "ActivityFailed", + "namespace": "default", + "payload": { + "code": 0, + "task": { + "id": "01m7tfmh0w83cfz7kg6z4jxzqm", + "type": "activity", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:24.885211Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:24.572340Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:24.958755Z" + }, + "message": "fail-m1", + "activity": { + "id": "01m7tfmh0s97v0b4ekhqj06q1z", + "type": "m1", + "class": "m1", + "queue": "rv307-95d208", + "status": "failed", + "sequence": 6, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "closed_at": "2026-11-19T18:03:25.000007Z", + "exception": { + "code": 0, + "type": null, + "message": "fail-m1", + "non_retryable": true + }, + "attempt_id": "01M7TFMHAN3Y9AK69R60K108T9", + "created_at": "2026-11-19T18:03:24.569756Z", + "started_at": "2026-11-19T18:03:24.885211Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmh0s97v0b4ekhqj06q1z" + }, + "sequence": 6, + "exception": { + "type": null, + "message": "fail-m1", + "code": 0, + "non_retryable": true + }, + "failure_id": "01m7tfmhe7jx4mjg3xfm1dt4t8", + "activity_type": "m1", + "non_retryable": true, + "activity_class": "m1", + "attempt_number": 1, + "exception_type": null, + "exception_class": "RuntimeException", + "failure_category": "activity", + "parallel_group_id": "parallel-activities:5:2", + "activity_attempt_id": "01M7TFMHAN3Y9AK69R60K108T9", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:5:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 5 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfmh0s97v0b4ekhqj06q1z", + "parallel_group_base_sequence": 5 + }, + "workflow_task_id": "01m7tfmh0w83cfz7kg6z4jxzqm", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:25.001498Z" + }, + { + "id": "01m7tfmjq52v4y8q4sw055kg8r", + "sequence": 16, + "event_type": "ActivityFailed", + "namespace": "default", + "payload": { + "code": 0, + "task": { + "id": "01m7tfmh0rce8ey59n2b3gsmae", + "type": "activity", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:24.785950Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:24.567858Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:24.848965Z" + }, + "message": "fail-m0", + "activity": { + "id": "01m7tfmh0k7bth61zv6erffdnr", + "type": "m0", + "class": "m0", + "queue": "rv307-95d208", + "status": "failed", + "sequence": 5, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "closed_at": "2026-11-19T18:03:26.307727Z", + "exception": { + "code": 0, + "type": null, + "message": "fail-m0", + "non_retryable": true + }, + "attempt_id": "01M7TFMH7JE9SDVJ7VR807EVPT", + "created_at": "2026-11-19T18:03:24.563929Z", + "started_at": "2026-11-19T18:03:24.785950Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmh0k7bth61zv6erffdnr" + }, + "sequence": 5, + "exception": { + "type": null, + "message": "fail-m0", + "code": 0, + "non_retryable": true + }, + "failure_id": "01m7tfmjq3pnbgqfk7vy8pfqjq", + "activity_type": "m0", + "non_retryable": true, + "activity_class": "m0", + "attempt_number": 1, + "exception_type": null, + "exception_class": "RuntimeException", + "failure_category": "activity", + "parallel_group_id": "parallel-activities:5:2", + "activity_attempt_id": "01M7TFMH7JE9SDVJ7VR807EVPT", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:5:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 5 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfmh0k7bth61zv6erffdnr", + "parallel_group_base_sequence": 5 + }, + "workflow_task_id": "01m7tfmh0rce8ey59n2b3gsmae", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:26.309124Z" + }, + { + "id": "01m7tfmkspws6jrb7zgqgkb58a", + "sequence": 17, + "event_type": "ActivityCompleted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmb2y6dwd7js9zzj1d7bw", + "type": "activity", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:24.651495Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:18.494356Z", + "repair_count": 1, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:24.745799Z", + "last_dispatched_at": "2026-11-19T18:03:24.486915Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:24.486915Z" + }, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoEb2s=" + }, + "activity": { + "id": "01m7tfmb2r4yykrxz5ms6mxh7e", + "type": "slow", + "class": "slow", + "queue": "rv307-95d208", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoEb2s=" + }, + "status": "completed", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "closed_at": "2026-11-19T18:03:27.411433Z", + "attempt_id": "01M7TFMH3BKEDQ2AT39KAVBR6V", + "created_at": "2026-11-19T18:03:18.488686Z", + "started_at": "2026-11-19T18:03:24.651495Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmb2r4yykrxz5ms6mxh7e" + }, + "sequence": 2, + "activity_type": "slow", + "payload_codec": "avro", + "activity_class": "slow", + "attempt_number": 1, + "parallel_group_id": "select-calls:1:2", + "activity_attempt_id": "01M7TFMH3BKEDQ2AT39KAVBR6V", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "activity_execution_id": "01m7tfmb2r4yykrxz5ms6mxh7e", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + }, + "workflow_task_id": "01m7tfmb2y6dwd7js9zzj1d7bw", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:27.413691Z" + }, + { + "id": "01m7tfmmbpcn9ynjszjj5ype78", + "sequence": 18, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmjqb26py8pavrcy62ewh", + "type": "workflow", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:27.545637Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:26.314879Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:37.545600Z", + "last_dispatched_at": "2026-11-19T18:03:26.344774Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:26.344774Z" + }, + "activity": { + "id": "01m7tfmmbkgrbna0pf56qrg0rq", + "type": "compensate", + "class": "compensate", + "queue": "rv307-95d208", + "status": "pending", + "sequence": 7, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCCg5mYWlsLW0xAA==" + }, + "created_at": "2026-11-19T18:03:27.987490Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmmbkgrbna0pf56qrg0rq" + }, + "sequence": 7, + "activity_type": "compensate", + "activity_class": "compensate", + "activity_execution_id": "01m7tfmmbkgrbna0pf56qrg0rq" + }, + "workflow_task_id": "01m7tfmjqb26py8pavrcy62ewh", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:27.989472Z" + }, + { + "id": "01m7tfmphv8902yrdvqezkq2pv", + "sequence": 19, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmmbt42zy7fb4awkmq8pw", + "type": "activity", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:30.216603Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:27.993985Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:30.216970Z" + }, + "activity": { + "id": "01m7tfmmbkgrbna0pf56qrg0rq", + "type": "compensate", + "class": "compensate", + "queue": "rv307-95d208", + "status": "running", + "sequence": 7, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCCg5mYWlsLW0xAA==" + }, + "attempt_id": "01M7TFMPH8CWTANT27NX8VWT2Q", + "created_at": "2026-11-19T18:03:27.987490Z", + "started_at": "2026-11-19T18:03:30.216603Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmmbkgrbna0pf56qrg0rq" + }, + "sequence": 7, + "activity_type": "compensate", + "activity_class": "compensate", + "attempt_number": 1, + "activity_attempt_id": "01M7TFMPH8CWTANT27NX8VWT2Q", + "activity_execution_id": "01m7tfmmbkgrbna0pf56qrg0rq" + }, + "workflow_task_id": "01m7tfmmbt42zy7fb4awkmq8pw", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:30.233123Z" + }, + { + "id": "01m7tfmpn3yy98fedb7nq4xm38", + "sequence": 20, + "event_type": "ActivityCompleted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmmbt42zy7fb4awkmq8pw", + "type": "activity", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:30.216603Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:27.993985Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:30.306166Z" + }, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoMdW5kb25l" + }, + "activity": { + "id": "01m7tfmmbkgrbna0pf56qrg0rq", + "type": "compensate", + "class": "compensate", + "queue": "rv307-95d208", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoMdW5kb25l" + }, + "status": "completed", + "sequence": 7, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCCg5mYWlsLW0xAA==" + }, + "closed_at": "2026-11-19T18:03:30.337850Z", + "attempt_id": "01M7TFMPH8CWTANT27NX8VWT2Q", + "created_at": "2026-11-19T18:03:27.987490Z", + "started_at": "2026-11-19T18:03:30.216603Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmmbkgrbna0pf56qrg0rq" + }, + "sequence": 7, + "activity_type": "compensate", + "payload_codec": "avro", + "activity_class": "compensate", + "attempt_number": 1, + "activity_attempt_id": "01M7TFMPH8CWTANT27NX8VWT2Q", + "activity_execution_id": "01m7tfmmbkgrbna0pf56qrg0rq" + }, + "workflow_task_id": "01m7tfmmbt42zy7fb4awkmq8pw", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:30.339364Z" + } + ], + "expected": { + "command_sequence": [ + { + "type": "complete_workflow" + } + ] + } +} diff --git a/tests/fixtures/replay_regressions/failure-order-new-run-zero-command-catch-query.json b/tests/fixtures/replay_regressions/failure-order-new-run-zero-command-catch-query.json new file mode 100644 index 00000000..4a6b7654 --- /dev/null +++ b/tests/fixtures/replay_regressions/failure-order-new-run-zero-command-catch-query.json @@ -0,0 +1,1045 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "failure-order-new-run-zero-command-catch-query", + "protocol_version": "1.20", + "bindings": [ + "python" + ], + "parallel_failure_order": { + "issues": [ + "durable-workflow/sdk-python#307" + ], + "note": "Recorded by this version on a Server built from durable-workflow/server main 52e3ec00 (docker-compose.test.yml). The run recorded the parallel failure-order marker at sequence 4, directly before its group. m1 failed first, then m0. The task that saw the settled group caught fail-m1, the first recorded failure, and awaited a select() handle, so it recorded no command and no decision." + }, + "workflow": { + "type": "rv348_handle", + "input": [], + "payload_codec": "avro", + "run_id": "01m7tfmajd2mzmq8bjygdsf5an", + "workflow_id": "rv307-rv348_handle-1c2847", + "query": { + "name": "caught", + "arguments": [] + } + }, + "history": [ + { + "id": "01m7tfmajjgp5q69b444d8cbzz", + "sequence": 1, + "event_type": "StartAccepted", + "namespace": "default", + "payload": { + "memo": null, + "command": { + "id": "01m7tfmajf35nsz3zz911ttetj", + "type": "start", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "outcome": "started_new", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "sequence": 1, + "applied_at": "2026-11-19T18:03:17.966013Z", + "accepted_at": "2026-11-19T18:03:17.966010Z", + "auth_method": "token", + "auth_status": "authorized", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfmajd2mzmq8bjygdsf5an", + "request_route_name": "generated::FqJuJMl97iiXlIDz", + "request_fingerprint": "sha256:f47aee00c4c048c39f0c6e7bba8ebada52b7e85b45eeb39b592043178306f9e4" + }, + "outcome": "started_new", + "business_key": null, + "workflow_type": "rv348_handle", + "workflow_class": "rv348_handle", + "workflow_run_id": "01m7tfmajd2mzmq8bjygdsf5an", + "search_attributes": null, + "visibility_labels": null, + "workflow_command_id": "01m7tfmajf35nsz3zz911ttetj", + "workflow_instance_id": "rv307-rv348_handle-1c2847" + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfmajf35nsz3zz911ttetj", + "recorded_at": "2026-11-19T18:03:17.969955Z" + }, + { + "id": "01m7tfmajrzvkk20crm5f4ptrz", + "sequence": 2, + "event_type": "WorkflowStarted", + "namespace": "default", + "payload": { + "command": { + "id": "01m7tfmajf35nsz3zz911ttetj", + "type": "start", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "outcome": "started_new", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "sequence": 1, + "applied_at": "2026-11-19T18:03:17.966013Z", + "accepted_at": "2026-11-19T18:03:17.966010Z", + "auth_method": "token", + "auth_status": "authorized", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfmajd2mzmq8bjygdsf5an", + "request_route_name": "generated::FqJuJMl97iiXlIDz", + "request_fingerprint": "sha256:f47aee00c4c048c39f0c6e7bba8ebada52b7e85b45eeb39b592043178306f9e4" + }, + "workflow_type": "rv348_handle", + "workflow_class": "rv348_handle", + "run_deadline_at": "2026-11-19T18:13:17+00:00", + "workflow_run_id": "01m7tfmajd2mzmq8bjygdsf5an", + "declared_queries": [ + "caught" + ], + "declared_signals": [], + "declared_updates": [ + "report" + ], + "declared_entry_mode": "canonical", + "run_timeout_seconds": 600, + "workflow_command_id": "01m7tfmajf35nsz3zz911ttetj", + "workflow_instance_id": "rv307-rv348_handle-1c2847", + "declared_entry_method": "handle", + "execution_deadline_at": "2026-11-19T19:03:17+00:00", + "declared_query_contracts": [ + { + "name": "caught", + "parameters": [] + } + ], + "declared_signal_contracts": [], + "declared_update_contracts": [ + { + "name": "report", + "parameters": [ + { + "name": "_", + "type": null, + "default": null, + "position": 0, + "required": false, + "variadic": true, + "allows_null": true, + "default_available": false + } + ] + } + ], + "execution_timeout_seconds": 3600, + "declared_update_validators": [], + "declared_entry_declaring_class": "external:rv348_handle", + "workflow_definition_fingerprint": "sha256:69ed762d44804878ebb703a2ad385119f6f776d0dae561c15f8f84e1707a704a", + "workflow_definition_fingerprint_source": "worker" + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfmajf35nsz3zz911ttetj", + "recorded_at": "2026-11-19T18:03:17.974885Z" + }, + { + "id": "01m7tfmb2j0k3f4w32jxb48e1w", + "sequence": 3, + "event_type": "TimerScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmajt78z8y51sj6e91dwk", + "type": "workflow", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:18.040583Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:17.978566Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:28.040524Z", + "last_dispatched_at": "2026-11-19T18:03:17.982237Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:17.982237Z" + }, + "fire_at": "2026-11-19T18:03:19.478674Z", + "sequence": 1, + "timer_id": "01m7tfmb2ec1j7gtczamgd4nyw", + "delay_seconds": 1, + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + }, + "workflow_task_id": "01m7tfmajt78z8y51sj6e91dwk", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:18.480737Z" + }, + { + "id": "01m7tfmb2vs6syc1x683pty7ew", + "sequence": 4, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmajt78z8y51sj6e91dwk", + "type": "workflow", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:18.040583Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:17.978566Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:28.040524Z", + "last_dispatched_at": "2026-11-19T18:03:17.982237Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:17.982237Z" + }, + "activity": { + "id": "01m7tfmb2r4yykrxz5ms6mxh7e", + "type": "slow", + "class": "slow", + "queue": "rv307-95d208", + "status": "pending", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "created_at": "2026-11-19T18:03:18.488686Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmb2r4yykrxz5ms6mxh7e" + }, + "sequence": 2, + "activity_type": "slow", + "activity_class": "slow", + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "activity_execution_id": "01m7tfmb2r4yykrxz5ms6mxh7e", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + }, + "workflow_task_id": "01m7tfmajt78z8y51sj6e91dwk", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:18.490538Z" + }, + { + "id": "01m7tfmd1558qccnq7tp4gx0rr", + "sequence": 5, + "event_type": "TimerFired", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmb2ncmszyfrjdst60e59", + "type": "timer", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:20.466569Z", + "connection": "mysql", + "lease_owner": "01m7tfmb2ncmszyfrjdst60e59", + "available_at": "2026-11-19T18:03:19.478674Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:30.466572Z" + }, + "fire_at": "2026-11-19T18:03:19.478674Z", + "fired_at": "2026-11-19T18:03:20.484504Z", + "sequence": 1, + "timer_id": "01m7tfmb2ec1j7gtczamgd4nyw", + "delay_seconds": 1, + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + }, + "workflow_task_id": "01m7tfmb2ncmszyfrjdst60e59", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:20.485608Z" + }, + { + "id": "01m7tfmd1921d4bkmatrk6ebdc", + "sequence": 6, + "event_type": "SelectionResolved", + "namespace": "default", + "payload": { + "outcome": "completed", + "member_key": "t", + "member_size": 1, + "member_index": 0, + "operation_kind": "timer", + "operation_identity": "01m7tfmb2ec1j7gtczamgd4nyw", + "selection_group_id": "select-calls:1:2", + "resolution_event_id": "01m7tfmd1558qccnq7tp4gx0rr", + "member_base_sequence": 1, + "selection_group_size": 2, + "resolution_event_type": "TimerFired", + "selection_group_base_sequence": 1 + }, + "workflow_task_id": null, + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:20.489579Z" + }, + { + "id": "01m7tfme1bwdba5r29kxxd6cf2", + "sequence": 7, + "event_type": "TimerScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmd1cj5fqtp8jwx6ct2mw", + "type": "workflow", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:21.093175Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:20.491921Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:31.093133Z", + "last_dispatched_at": "2026-11-19T18:03:20.510053Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:20.510053Z" + }, + "fire_at": "2026-11-19T18:03:22.512049Z", + "sequence": 3, + "timer_id": "01m7tfme183tr41x27sy93ae5t", + "delay_seconds": 1 + }, + "workflow_task_id": "01m7tfmd1cj5fqtp8jwx6ct2mw", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:21.514136Z" + }, + { + "id": "01m7tfmg2f61p1djskt19m30dx", + "sequence": 8, + "event_type": "TimerFired", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfme1ef1bzv4s1q937682x", + "type": "timer", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:23.566024Z", + "connection": "mysql", + "lease_owner": "01m7tfme1ef1bzv4s1q937682x", + "available_at": "2026-11-19T18:03:22.512049Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:33.566032Z" + }, + "fire_at": "2026-11-19T18:03:22.512049Z", + "fired_at": "2026-11-19T18:03:23.597085Z", + "sequence": 3, + "timer_id": "01m7tfme183tr41x27sy93ae5t", + "delay_seconds": 1 + }, + "workflow_task_id": "01m7tfme1ef1bzv4s1q937682x", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:23.599095Z" + }, + { + "id": "01m7tfmh0ewhpdffmw6f53xw1t", + "sequence": 9, + "event_type": "VersionMarkerRecorded", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmg2k7zfpt95rnxpnttbd", + "type": "workflow", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:24.111580Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:23.603148Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:34.111540Z", + "last_dispatched_at": "2026-11-19T18:03:23.630339Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:23.630339Z" + }, + "version": 1, + "sequence": 4, + "change_id": "durable_workflow.parallel_failure_order", + "max_supported": 1, + "min_supported": 1 + }, + "workflow_task_id": "01m7tfmg2k7zfpt95rnxpnttbd", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:24.556837Z" + }, + { + "id": "01m7tfmh0p07a6z70f3arbd67m", + "sequence": 10, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmg2k7zfpt95rnxpnttbd", + "type": "workflow", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:24.111580Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:23.603148Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:34.111540Z", + "last_dispatched_at": "2026-11-19T18:03:23.630339Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:23.630339Z" + }, + "activity": { + "id": "01m7tfmh0k7bth61zv6erffdnr", + "type": "m0", + "class": "m0", + "queue": "rv307-95d208", + "status": "pending", + "sequence": 5, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "created_at": "2026-11-19T18:03:24.563929Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmh0k7bth61zv6erffdnr" + }, + "sequence": 5, + "activity_type": "m0", + "activity_class": "m0", + "parallel_group_id": "parallel-activities:5:2", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:5:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 5 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfmh0k7bth61zv6erffdnr", + "parallel_group_base_sequence": 5 + }, + "workflow_task_id": "01m7tfmg2k7zfpt95rnxpnttbd", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:24.565789Z" + }, + { + "id": "01m7tfmh0vyz2awxy1qfqtytnp", + "sequence": 11, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmg2k7zfpt95rnxpnttbd", + "type": "workflow", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:24.111580Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:23.603148Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:34.111540Z", + "last_dispatched_at": "2026-11-19T18:03:23.630339Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:23.630339Z" + }, + "activity": { + "id": "01m7tfmh0s97v0b4ekhqj06q1z", + "type": "m1", + "class": "m1", + "queue": "rv307-95d208", + "status": "pending", + "sequence": 6, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "created_at": "2026-11-19T18:03:24.569756Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmh0s97v0b4ekhqj06q1z" + }, + "sequence": 6, + "activity_type": "m1", + "activity_class": "m1", + "parallel_group_id": "parallel-activities:5:2", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:5:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 5 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfmh0s97v0b4ekhqj06q1z", + "parallel_group_base_sequence": 5 + }, + "workflow_task_id": "01m7tfmg2k7zfpt95rnxpnttbd", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:24.570640Z" + }, + { + "id": "01m7tfmh3hhn1f2nskrftt3p0h", + "sequence": 12, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmb2y6dwd7js9zzj1d7bw", + "type": "activity", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:24.651495Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:18.494356Z", + "repair_count": 1, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:24.651591Z", + "last_dispatched_at": "2026-11-19T18:03:24.486915Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:24.486915Z" + }, + "activity": { + "id": "01m7tfmb2r4yykrxz5ms6mxh7e", + "type": "slow", + "class": "slow", + "queue": "rv307-95d208", + "status": "running", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "attempt_id": "01M7TFMH3BKEDQ2AT39KAVBR6V", + "created_at": "2026-11-19T18:03:18.488686Z", + "started_at": "2026-11-19T18:03:24.651495Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmb2r4yykrxz5ms6mxh7e" + }, + "sequence": 2, + "activity_type": "slow", + "activity_class": "slow", + "attempt_number": 1, + "parallel_group_id": "select-calls:1:2", + "activity_attempt_id": "01M7TFMH3BKEDQ2AT39KAVBR6V", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "activity_execution_id": "01m7tfmb2r4yykrxz5ms6mxh7e", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + }, + "workflow_task_id": "01m7tfmb2y6dwd7js9zzj1d7bw", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:24.656289Z" + }, + { + "id": "01m7tfmh7q122cv6edc7013fpb", + "sequence": 13, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmh0rce8ey59n2b3gsmae", + "type": "activity", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:24.785950Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:24.567858Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:24.786065Z" + }, + "activity": { + "id": "01m7tfmh0k7bth61zv6erffdnr", + "type": "m0", + "class": "m0", + "queue": "rv307-95d208", + "status": "running", + "sequence": 5, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "attempt_id": "01M7TFMH7JE9SDVJ7VR807EVPT", + "created_at": "2026-11-19T18:03:24.563929Z", + "started_at": "2026-11-19T18:03:24.785950Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmh0k7bth61zv6erffdnr" + }, + "sequence": 5, + "activity_type": "m0", + "activity_class": "m0", + "attempt_number": 1, + "parallel_group_id": "parallel-activities:5:2", + "activity_attempt_id": "01M7TFMH7JE9SDVJ7VR807EVPT", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:5:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 5 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfmh0k7bth61zv6erffdnr", + "parallel_group_base_sequence": 5 + }, + "workflow_task_id": "01m7tfmh0rce8ey59n2b3gsmae", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:24.790941Z" + }, + { + "id": "01m7tfmhasmzqs85em7rpmsq11", + "sequence": 14, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmh0w83cfz7kg6z4jxzqm", + "type": "activity", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:24.885211Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:24.572340Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:24.885300Z" + }, + "activity": { + "id": "01m7tfmh0s97v0b4ekhqj06q1z", + "type": "m1", + "class": "m1", + "queue": "rv307-95d208", + "status": "running", + "sequence": 6, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "attempt_id": "01M7TFMHAN3Y9AK69R60K108T9", + "created_at": "2026-11-19T18:03:24.569756Z", + "started_at": "2026-11-19T18:03:24.885211Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmh0s97v0b4ekhqj06q1z" + }, + "sequence": 6, + "activity_type": "m1", + "activity_class": "m1", + "attempt_number": 1, + "parallel_group_id": "parallel-activities:5:2", + "activity_attempt_id": "01M7TFMHAN3Y9AK69R60K108T9", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:5:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 5 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfmh0s97v0b4ekhqj06q1z", + "parallel_group_base_sequence": 5 + }, + "workflow_task_id": "01m7tfmh0w83cfz7kg6z4jxzqm", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:24.889189Z" + }, + { + "id": "01m7tfmheabrw4zq6gdmnyw0q6", + "sequence": 15, + "event_type": "ActivityFailed", + "namespace": "default", + "payload": { + "code": 0, + "task": { + "id": "01m7tfmh0w83cfz7kg6z4jxzqm", + "type": "activity", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:24.885211Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:24.572340Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:24.958755Z" + }, + "message": "fail-m1", + "activity": { + "id": "01m7tfmh0s97v0b4ekhqj06q1z", + "type": "m1", + "class": "m1", + "queue": "rv307-95d208", + "status": "failed", + "sequence": 6, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "closed_at": "2026-11-19T18:03:25.000007Z", + "exception": { + "code": 0, + "type": null, + "message": "fail-m1", + "non_retryable": true + }, + "attempt_id": "01M7TFMHAN3Y9AK69R60K108T9", + "created_at": "2026-11-19T18:03:24.569756Z", + "started_at": "2026-11-19T18:03:24.885211Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmh0s97v0b4ekhqj06q1z" + }, + "sequence": 6, + "exception": { + "type": null, + "message": "fail-m1", + "code": 0, + "non_retryable": true + }, + "failure_id": "01m7tfmhe7jx4mjg3xfm1dt4t8", + "activity_type": "m1", + "non_retryable": true, + "activity_class": "m1", + "attempt_number": 1, + "exception_type": null, + "exception_class": "RuntimeException", + "failure_category": "activity", + "parallel_group_id": "parallel-activities:5:2", + "activity_attempt_id": "01M7TFMHAN3Y9AK69R60K108T9", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:5:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 5 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfmh0s97v0b4ekhqj06q1z", + "parallel_group_base_sequence": 5 + }, + "workflow_task_id": "01m7tfmh0w83cfz7kg6z4jxzqm", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:25.001498Z" + }, + { + "id": "01m7tfmjq52v4y8q4sw055kg8r", + "sequence": 16, + "event_type": "ActivityFailed", + "namespace": "default", + "payload": { + "code": 0, + "task": { + "id": "01m7tfmh0rce8ey59n2b3gsmae", + "type": "activity", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:24.785950Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:24.567858Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:24.848965Z" + }, + "message": "fail-m0", + "activity": { + "id": "01m7tfmh0k7bth61zv6erffdnr", + "type": "m0", + "class": "m0", + "queue": "rv307-95d208", + "status": "failed", + "sequence": 5, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "closed_at": "2026-11-19T18:03:26.307727Z", + "exception": { + "code": 0, + "type": null, + "message": "fail-m0", + "non_retryable": true + }, + "attempt_id": "01M7TFMH7JE9SDVJ7VR807EVPT", + "created_at": "2026-11-19T18:03:24.563929Z", + "started_at": "2026-11-19T18:03:24.785950Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmh0k7bth61zv6erffdnr" + }, + "sequence": 5, + "exception": { + "type": null, + "message": "fail-m0", + "code": 0, + "non_retryable": true + }, + "failure_id": "01m7tfmjq3pnbgqfk7vy8pfqjq", + "activity_type": "m0", + "non_retryable": true, + "activity_class": "m0", + "attempt_number": 1, + "exception_type": null, + "exception_class": "RuntimeException", + "failure_category": "activity", + "parallel_group_id": "parallel-activities:5:2", + "activity_attempt_id": "01M7TFMH7JE9SDVJ7VR807EVPT", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:5:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 5 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfmh0k7bth61zv6erffdnr", + "parallel_group_base_sequence": 5 + }, + "workflow_task_id": "01m7tfmh0rce8ey59n2b3gsmae", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:26.309124Z" + } + ], + "expected": { + "query_result": "fail-m1" + } +} diff --git a/tests/fixtures/replay_regressions/failure-order-new-run-zero-command-catch.json b/tests/fixtures/replay_regressions/failure-order-new-run-zero-command-catch.json new file mode 100644 index 00000000..3723510c --- /dev/null +++ b/tests/fixtures/replay_regressions/failure-order-new-run-zero-command-catch.json @@ -0,0 +1,1041 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "failure-order-new-run-zero-command-catch", + "protocol_version": "1.20", + "bindings": [ + "python" + ], + "parallel_failure_order": { + "issues": [ + "durable-workflow/sdk-python#307" + ], + "note": "Recorded by this version on a Server built from durable-workflow/server main 52e3ec00 (docker-compose.test.yml). The run recorded the parallel failure-order marker at sequence 4, directly before its group. m1 failed first, then m0. The task that saw the settled group caught fail-m1, the first recorded failure, and awaited a select() handle, so it recorded no command and no decision." + }, + "workflow": { + "type": "rv348_handle", + "input": [], + "payload_codec": "avro", + "run_id": "01m7tfmajd2mzmq8bjygdsf5an", + "workflow_id": "rv307-rv348_handle-1c2847" + }, + "history": [ + { + "id": "01m7tfmajjgp5q69b444d8cbzz", + "sequence": 1, + "event_type": "StartAccepted", + "namespace": "default", + "payload": { + "memo": null, + "command": { + "id": "01m7tfmajf35nsz3zz911ttetj", + "type": "start", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "outcome": "started_new", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "sequence": 1, + "applied_at": "2026-11-19T18:03:17.966013Z", + "accepted_at": "2026-11-19T18:03:17.966010Z", + "auth_method": "token", + "auth_status": "authorized", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfmajd2mzmq8bjygdsf5an", + "request_route_name": "generated::FqJuJMl97iiXlIDz", + "request_fingerprint": "sha256:f47aee00c4c048c39f0c6e7bba8ebada52b7e85b45eeb39b592043178306f9e4" + }, + "outcome": "started_new", + "business_key": null, + "workflow_type": "rv348_handle", + "workflow_class": "rv348_handle", + "workflow_run_id": "01m7tfmajd2mzmq8bjygdsf5an", + "search_attributes": null, + "visibility_labels": null, + "workflow_command_id": "01m7tfmajf35nsz3zz911ttetj", + "workflow_instance_id": "rv307-rv348_handle-1c2847" + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfmajf35nsz3zz911ttetj", + "recorded_at": "2026-11-19T18:03:17.969955Z" + }, + { + "id": "01m7tfmajrzvkk20crm5f4ptrz", + "sequence": 2, + "event_type": "WorkflowStarted", + "namespace": "default", + "payload": { + "command": { + "id": "01m7tfmajf35nsz3zz911ttetj", + "type": "start", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "outcome": "started_new", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "sequence": 1, + "applied_at": "2026-11-19T18:03:17.966013Z", + "accepted_at": "2026-11-19T18:03:17.966010Z", + "auth_method": "token", + "auth_status": "authorized", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfmajd2mzmq8bjygdsf5an", + "request_route_name": "generated::FqJuJMl97iiXlIDz", + "request_fingerprint": "sha256:f47aee00c4c048c39f0c6e7bba8ebada52b7e85b45eeb39b592043178306f9e4" + }, + "workflow_type": "rv348_handle", + "workflow_class": "rv348_handle", + "run_deadline_at": "2026-11-19T18:13:17+00:00", + "workflow_run_id": "01m7tfmajd2mzmq8bjygdsf5an", + "declared_queries": [ + "caught" + ], + "declared_signals": [], + "declared_updates": [ + "report" + ], + "declared_entry_mode": "canonical", + "run_timeout_seconds": 600, + "workflow_command_id": "01m7tfmajf35nsz3zz911ttetj", + "workflow_instance_id": "rv307-rv348_handle-1c2847", + "declared_entry_method": "handle", + "execution_deadline_at": "2026-11-19T19:03:17+00:00", + "declared_query_contracts": [ + { + "name": "caught", + "parameters": [] + } + ], + "declared_signal_contracts": [], + "declared_update_contracts": [ + { + "name": "report", + "parameters": [ + { + "name": "_", + "type": null, + "default": null, + "position": 0, + "required": false, + "variadic": true, + "allows_null": true, + "default_available": false + } + ] + } + ], + "execution_timeout_seconds": 3600, + "declared_update_validators": [], + "declared_entry_declaring_class": "external:rv348_handle", + "workflow_definition_fingerprint": "sha256:69ed762d44804878ebb703a2ad385119f6f776d0dae561c15f8f84e1707a704a", + "workflow_definition_fingerprint_source": "worker" + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfmajf35nsz3zz911ttetj", + "recorded_at": "2026-11-19T18:03:17.974885Z" + }, + { + "id": "01m7tfmb2j0k3f4w32jxb48e1w", + "sequence": 3, + "event_type": "TimerScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmajt78z8y51sj6e91dwk", + "type": "workflow", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:18.040583Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:17.978566Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:28.040524Z", + "last_dispatched_at": "2026-11-19T18:03:17.982237Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:17.982237Z" + }, + "fire_at": "2026-11-19T18:03:19.478674Z", + "sequence": 1, + "timer_id": "01m7tfmb2ec1j7gtczamgd4nyw", + "delay_seconds": 1, + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + }, + "workflow_task_id": "01m7tfmajt78z8y51sj6e91dwk", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:18.480737Z" + }, + { + "id": "01m7tfmb2vs6syc1x683pty7ew", + "sequence": 4, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmajt78z8y51sj6e91dwk", + "type": "workflow", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:18.040583Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:17.978566Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:28.040524Z", + "last_dispatched_at": "2026-11-19T18:03:17.982237Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:17.982237Z" + }, + "activity": { + "id": "01m7tfmb2r4yykrxz5ms6mxh7e", + "type": "slow", + "class": "slow", + "queue": "rv307-95d208", + "status": "pending", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "created_at": "2026-11-19T18:03:18.488686Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmb2r4yykrxz5ms6mxh7e" + }, + "sequence": 2, + "activity_type": "slow", + "activity_class": "slow", + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "activity_execution_id": "01m7tfmb2r4yykrxz5ms6mxh7e", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + }, + "workflow_task_id": "01m7tfmajt78z8y51sj6e91dwk", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:18.490538Z" + }, + { + "id": "01m7tfmd1558qccnq7tp4gx0rr", + "sequence": 5, + "event_type": "TimerFired", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmb2ncmszyfrjdst60e59", + "type": "timer", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:20.466569Z", + "connection": "mysql", + "lease_owner": "01m7tfmb2ncmszyfrjdst60e59", + "available_at": "2026-11-19T18:03:19.478674Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:30.466572Z" + }, + "fire_at": "2026-11-19T18:03:19.478674Z", + "fired_at": "2026-11-19T18:03:20.484504Z", + "sequence": 1, + "timer_id": "01m7tfmb2ec1j7gtczamgd4nyw", + "delay_seconds": 1, + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + }, + "workflow_task_id": "01m7tfmb2ncmszyfrjdst60e59", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:20.485608Z" + }, + { + "id": "01m7tfmd1921d4bkmatrk6ebdc", + "sequence": 6, + "event_type": "SelectionResolved", + "namespace": "default", + "payload": { + "outcome": "completed", + "member_key": "t", + "member_size": 1, + "member_index": 0, + "operation_kind": "timer", + "operation_identity": "01m7tfmb2ec1j7gtczamgd4nyw", + "selection_group_id": "select-calls:1:2", + "resolution_event_id": "01m7tfmd1558qccnq7tp4gx0rr", + "member_base_sequence": 1, + "selection_group_size": 2, + "resolution_event_type": "TimerFired", + "selection_group_base_sequence": 1 + }, + "workflow_task_id": null, + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:20.489579Z" + }, + { + "id": "01m7tfme1bwdba5r29kxxd6cf2", + "sequence": 7, + "event_type": "TimerScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmd1cj5fqtp8jwx6ct2mw", + "type": "workflow", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:21.093175Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:20.491921Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:31.093133Z", + "last_dispatched_at": "2026-11-19T18:03:20.510053Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:20.510053Z" + }, + "fire_at": "2026-11-19T18:03:22.512049Z", + "sequence": 3, + "timer_id": "01m7tfme183tr41x27sy93ae5t", + "delay_seconds": 1 + }, + "workflow_task_id": "01m7tfmd1cj5fqtp8jwx6ct2mw", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:21.514136Z" + }, + { + "id": "01m7tfmg2f61p1djskt19m30dx", + "sequence": 8, + "event_type": "TimerFired", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfme1ef1bzv4s1q937682x", + "type": "timer", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:23.566024Z", + "connection": "mysql", + "lease_owner": "01m7tfme1ef1bzv4s1q937682x", + "available_at": "2026-11-19T18:03:22.512049Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:33.566032Z" + }, + "fire_at": "2026-11-19T18:03:22.512049Z", + "fired_at": "2026-11-19T18:03:23.597085Z", + "sequence": 3, + "timer_id": "01m7tfme183tr41x27sy93ae5t", + "delay_seconds": 1 + }, + "workflow_task_id": "01m7tfme1ef1bzv4s1q937682x", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:23.599095Z" + }, + { + "id": "01m7tfmh0ewhpdffmw6f53xw1t", + "sequence": 9, + "event_type": "VersionMarkerRecorded", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmg2k7zfpt95rnxpnttbd", + "type": "workflow", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:24.111580Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:23.603148Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:34.111540Z", + "last_dispatched_at": "2026-11-19T18:03:23.630339Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:23.630339Z" + }, + "version": 1, + "sequence": 4, + "change_id": "durable_workflow.parallel_failure_order", + "max_supported": 1, + "min_supported": 1 + }, + "workflow_task_id": "01m7tfmg2k7zfpt95rnxpnttbd", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:24.556837Z" + }, + { + "id": "01m7tfmh0p07a6z70f3arbd67m", + "sequence": 10, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmg2k7zfpt95rnxpnttbd", + "type": "workflow", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:24.111580Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:23.603148Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:34.111540Z", + "last_dispatched_at": "2026-11-19T18:03:23.630339Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:23.630339Z" + }, + "activity": { + "id": "01m7tfmh0k7bth61zv6erffdnr", + "type": "m0", + "class": "m0", + "queue": "rv307-95d208", + "status": "pending", + "sequence": 5, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "created_at": "2026-11-19T18:03:24.563929Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmh0k7bth61zv6erffdnr" + }, + "sequence": 5, + "activity_type": "m0", + "activity_class": "m0", + "parallel_group_id": "parallel-activities:5:2", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:5:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 5 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfmh0k7bth61zv6erffdnr", + "parallel_group_base_sequence": 5 + }, + "workflow_task_id": "01m7tfmg2k7zfpt95rnxpnttbd", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:24.565789Z" + }, + { + "id": "01m7tfmh0vyz2awxy1qfqtytnp", + "sequence": 11, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmg2k7zfpt95rnxpnttbd", + "type": "workflow", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:24.111580Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:23.603148Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:34.111540Z", + "last_dispatched_at": "2026-11-19T18:03:23.630339Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:23.630339Z" + }, + "activity": { + "id": "01m7tfmh0s97v0b4ekhqj06q1z", + "type": "m1", + "class": "m1", + "queue": "rv307-95d208", + "status": "pending", + "sequence": 6, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "created_at": "2026-11-19T18:03:24.569756Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmh0s97v0b4ekhqj06q1z" + }, + "sequence": 6, + "activity_type": "m1", + "activity_class": "m1", + "parallel_group_id": "parallel-activities:5:2", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:5:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 5 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfmh0s97v0b4ekhqj06q1z", + "parallel_group_base_sequence": 5 + }, + "workflow_task_id": "01m7tfmg2k7zfpt95rnxpnttbd", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:24.570640Z" + }, + { + "id": "01m7tfmh3hhn1f2nskrftt3p0h", + "sequence": 12, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmb2y6dwd7js9zzj1d7bw", + "type": "activity", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:24.651495Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:18.494356Z", + "repair_count": 1, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:24.651591Z", + "last_dispatched_at": "2026-11-19T18:03:24.486915Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:24.486915Z" + }, + "activity": { + "id": "01m7tfmb2r4yykrxz5ms6mxh7e", + "type": "slow", + "class": "slow", + "queue": "rv307-95d208", + "status": "running", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "attempt_id": "01M7TFMH3BKEDQ2AT39KAVBR6V", + "created_at": "2026-11-19T18:03:18.488686Z", + "started_at": "2026-11-19T18:03:24.651495Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmb2r4yykrxz5ms6mxh7e" + }, + "sequence": 2, + "activity_type": "slow", + "activity_class": "slow", + "attempt_number": 1, + "parallel_group_id": "select-calls:1:2", + "activity_attempt_id": "01M7TFMH3BKEDQ2AT39KAVBR6V", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "activity_execution_id": "01m7tfmb2r4yykrxz5ms6mxh7e", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + }, + "workflow_task_id": "01m7tfmb2y6dwd7js9zzj1d7bw", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:24.656289Z" + }, + { + "id": "01m7tfmh7q122cv6edc7013fpb", + "sequence": 13, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmh0rce8ey59n2b3gsmae", + "type": "activity", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:24.785950Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:24.567858Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:24.786065Z" + }, + "activity": { + "id": "01m7tfmh0k7bth61zv6erffdnr", + "type": "m0", + "class": "m0", + "queue": "rv307-95d208", + "status": "running", + "sequence": 5, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "attempt_id": "01M7TFMH7JE9SDVJ7VR807EVPT", + "created_at": "2026-11-19T18:03:24.563929Z", + "started_at": "2026-11-19T18:03:24.785950Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmh0k7bth61zv6erffdnr" + }, + "sequence": 5, + "activity_type": "m0", + "activity_class": "m0", + "attempt_number": 1, + "parallel_group_id": "parallel-activities:5:2", + "activity_attempt_id": "01M7TFMH7JE9SDVJ7VR807EVPT", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:5:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 5 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfmh0k7bth61zv6erffdnr", + "parallel_group_base_sequence": 5 + }, + "workflow_task_id": "01m7tfmh0rce8ey59n2b3gsmae", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:24.790941Z" + }, + { + "id": "01m7tfmhasmzqs85em7rpmsq11", + "sequence": 14, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfmh0w83cfz7kg6z4jxzqm", + "type": "activity", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:24.885211Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:24.572340Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:24.885300Z" + }, + "activity": { + "id": "01m7tfmh0s97v0b4ekhqj06q1z", + "type": "m1", + "class": "m1", + "queue": "rv307-95d208", + "status": "running", + "sequence": 6, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "attempt_id": "01M7TFMHAN3Y9AK69R60K108T9", + "created_at": "2026-11-19T18:03:24.569756Z", + "started_at": "2026-11-19T18:03:24.885211Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmh0s97v0b4ekhqj06q1z" + }, + "sequence": 6, + "activity_type": "m1", + "activity_class": "m1", + "attempt_number": 1, + "parallel_group_id": "parallel-activities:5:2", + "activity_attempt_id": "01M7TFMHAN3Y9AK69R60K108T9", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:5:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 5 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfmh0s97v0b4ekhqj06q1z", + "parallel_group_base_sequence": 5 + }, + "workflow_task_id": "01m7tfmh0w83cfz7kg6z4jxzqm", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:24.889189Z" + }, + { + "id": "01m7tfmheabrw4zq6gdmnyw0q6", + "sequence": 15, + "event_type": "ActivityFailed", + "namespace": "default", + "payload": { + "code": 0, + "task": { + "id": "01m7tfmh0w83cfz7kg6z4jxzqm", + "type": "activity", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:24.885211Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:24.572340Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:24.958755Z" + }, + "message": "fail-m1", + "activity": { + "id": "01m7tfmh0s97v0b4ekhqj06q1z", + "type": "m1", + "class": "m1", + "queue": "rv307-95d208", + "status": "failed", + "sequence": 6, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "closed_at": "2026-11-19T18:03:25.000007Z", + "exception": { + "code": 0, + "type": null, + "message": "fail-m1", + "non_retryable": true + }, + "attempt_id": "01M7TFMHAN3Y9AK69R60K108T9", + "created_at": "2026-11-19T18:03:24.569756Z", + "started_at": "2026-11-19T18:03:24.885211Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmh0s97v0b4ekhqj06q1z" + }, + "sequence": 6, + "exception": { + "type": null, + "message": "fail-m1", + "code": 0, + "non_retryable": true + }, + "failure_id": "01m7tfmhe7jx4mjg3xfm1dt4t8", + "activity_type": "m1", + "non_retryable": true, + "activity_class": "m1", + "attempt_number": 1, + "exception_type": null, + "exception_class": "RuntimeException", + "failure_category": "activity", + "parallel_group_id": "parallel-activities:5:2", + "activity_attempt_id": "01M7TFMHAN3Y9AK69R60K108T9", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:5:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 5 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfmh0s97v0b4ekhqj06q1z", + "parallel_group_base_sequence": 5 + }, + "workflow_task_id": "01m7tfmh0w83cfz7kg6z4jxzqm", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:25.001498Z" + }, + { + "id": "01m7tfmjq52v4y8q4sw055kg8r", + "sequence": 16, + "event_type": "ActivityFailed", + "namespace": "default", + "payload": { + "code": 0, + "task": { + "id": "01m7tfmh0rce8ey59n2b3gsmae", + "type": "activity", + "queue": "rv307-95d208", + "status": "leased", + "leased_at": "2026-11-19T18:03:24.785950Z", + "connection": "mysql", + "lease_owner": "rv307-w-1d9bfa", + "available_at": "2026-11-19T18:03:24.567858Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:24.848965Z" + }, + "message": "fail-m0", + "activity": { + "id": "01m7tfmh0k7bth61zv6erffdnr", + "type": "m0", + "class": "m0", + "queue": "rv307-95d208", + "status": "failed", + "sequence": 5, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "closed_at": "2026-11-19T18:03:26.307727Z", + "exception": { + "code": 0, + "type": null, + "message": "fail-m0", + "non_retryable": true + }, + "attempt_id": "01M7TFMH7JE9SDVJ7VR807EVPT", + "created_at": "2026-11-19T18:03:24.563929Z", + "started_at": "2026-11-19T18:03:24.785950Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfmh0k7bth61zv6erffdnr" + }, + "sequence": 5, + "exception": { + "type": null, + "message": "fail-m0", + "code": 0, + "non_retryable": true + }, + "failure_id": "01m7tfmjq3pnbgqfk7vy8pfqjq", + "activity_type": "m0", + "non_retryable": true, + "activity_class": "m0", + "attempt_number": 1, + "exception_type": null, + "exception_class": "RuntimeException", + "failure_category": "activity", + "parallel_group_id": "parallel-activities:5:2", + "activity_attempt_id": "01M7TFMH7JE9SDVJ7VR807EVPT", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:5:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 5 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfmh0k7bth61zv6erffdnr", + "parallel_group_base_sequence": 5 + }, + "workflow_task_id": "01m7tfmh0rce8ey59n2b3gsmae", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:26.309124Z" + } + ], + "expected": { + "command_sequence": [] + } +} diff --git a/tests/fixtures/replay_regressions/failure-order-released-select-cancel-query.json b/tests/fixtures/replay_regressions/failure-order-released-select-cancel-query.json new file mode 100644 index 00000000..bc38dc77 --- /dev/null +++ b/tests/fixtures/replay_regressions/failure-order-released-select-cancel-query.json @@ -0,0 +1,1012 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "failure-order-released-select-cancel-query", + "protocol_version": "1.20", + "bindings": [ + "python" + ], + "parallel_failure_order": { + "issues": [ + "durable-workflow/sdk-python#307" + ], + "note": "Recorded by published 2.19.1 on a Server built from durable-workflow/server main 52e3ec00 (docker-compose.test.yml): the group (m0, m1) was scheduled without the marker and m1 failed, then m0. The task that saw the settled group catches fail-m0, cancels a select() loser and compensates fail-m0; this version sends the same commands." + }, + "workflow": { + "type": "rv348_cancel", + "input": [], + "payload_codec": "avro", + "run_id": "01m7tfk96p1vr4vnrwf2k7mwv0", + "workflow_id": "rv307-rv348_cancel-1af5ee", + "query": { + "name": "caught", + "arguments": [] + } + }, + "history": [ + { + "id": "01m7tfk96zfcqzq0fzt4gvb691", + "sequence": 1, + "event_type": "StartAccepted", + "namespace": "default", + "payload": { + "memo": null, + "command": { + "id": "01m7tfk96txfwjnf900w6b7bxe", + "type": "start", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "outcome": "started_new", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "sequence": 1, + "applied_at": "2026-11-19T18:02:43.799220Z", + "accepted_at": "2026-11-19T18:02:43.799209Z", + "auth_method": "token", + "auth_status": "authorized", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfk96p1vr4vnrwf2k7mwv0", + "request_route_name": "generated::FqJuJMl97iiXlIDz", + "request_fingerprint": "sha256:0cd5a8b1b56226fec881b1357a0042ba40782bf855be16c912d3224b8d65cf66" + }, + "outcome": "started_new", + "business_key": null, + "workflow_type": "rv348_cancel", + "workflow_class": "rv348_cancel", + "workflow_run_id": "01m7tfk96p1vr4vnrwf2k7mwv0", + "search_attributes": null, + "visibility_labels": null, + "workflow_command_id": "01m7tfk96txfwjnf900w6b7bxe", + "workflow_instance_id": "rv307-rv348_cancel-1af5ee" + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfk96txfwjnf900w6b7bxe", + "recorded_at": "2026-11-19T18:02:43.805823Z" + }, + { + "id": "01m7tfk975p13m1k1bs73t658v", + "sequence": 2, + "event_type": "WorkflowStarted", + "namespace": "default", + "payload": { + "command": { + "id": "01m7tfk96txfwjnf900w6b7bxe", + "type": "start", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "outcome": "started_new", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "sequence": 1, + "applied_at": "2026-11-19T18:02:43.799220Z", + "accepted_at": "2026-11-19T18:02:43.799209Z", + "auth_method": "token", + "auth_status": "authorized", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfk96p1vr4vnrwf2k7mwv0", + "request_route_name": "generated::FqJuJMl97iiXlIDz", + "request_fingerprint": "sha256:0cd5a8b1b56226fec881b1357a0042ba40782bf855be16c912d3224b8d65cf66" + }, + "workflow_type": "rv348_cancel", + "workflow_class": "rv348_cancel", + "run_deadline_at": "2026-11-19T18:12:43+00:00", + "workflow_run_id": "01m7tfk96p1vr4vnrwf2k7mwv0", + "declared_queries": [ + "caught" + ], + "declared_signals": [], + "declared_updates": [ + "report" + ], + "declared_entry_mode": "canonical", + "run_timeout_seconds": 600, + "workflow_command_id": "01m7tfk96txfwjnf900w6b7bxe", + "workflow_instance_id": "rv307-rv348_cancel-1af5ee", + "declared_entry_method": "handle", + "execution_deadline_at": "2026-11-19T19:02:43+00:00", + "declared_query_contracts": [ + { + "name": "caught", + "parameters": [] + } + ], + "declared_signal_contracts": [], + "declared_update_contracts": [ + { + "name": "report", + "parameters": [ + { + "name": "_", + "type": null, + "default": null, + "position": 0, + "required": false, + "variadic": true, + "allows_null": true, + "default_available": false + } + ] + } + ], + "execution_timeout_seconds": 3600, + "declared_update_validators": [], + "declared_entry_declaring_class": "external:rv348_cancel", + "workflow_definition_fingerprint": "sha256:4f8f85902f58f49284c524502cb9366458411dc0768ba01d6bef05a43541d21f", + "workflow_definition_fingerprint_source": "worker" + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfk96txfwjnf900w6b7bxe", + "recorded_at": "2026-11-19T18:02:43.811611Z" + }, + { + "id": "01m7tfk9q5ppwba7xzpqqwy7ty", + "sequence": 3, + "event_type": "TimerScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfk978ccwhqf13q93eg8ax", + "type": "workflow", + "queue": "rv307-8e959d", + "status": "leased", + "leased_at": "2026-11-19T18:02:43.883316Z", + "connection": "mysql", + "lease_owner": "rv307-w-19f78f", + "available_at": "2026-11-19T18:02:43.816194Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:53.883238Z", + "last_dispatched_at": "2026-11-19T18:02:43.820116Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:43.820116Z" + }, + "fire_at": "2026-11-19T18:02:45.323760Z", + "sequence": 1, + "timer_id": "01m7tfk9q38v1smwprkqhb6dgq", + "delay_seconds": 1, + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + }, + "workflow_task_id": "01m7tfk978ccwhqf13q93eg8ax", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:44.325093Z" + }, + { + "id": "01m7tfk9qbms8jmz3whx68gbr8", + "sequence": 4, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfk978ccwhqf13q93eg8ax", + "type": "workflow", + "queue": "rv307-8e959d", + "status": "leased", + "leased_at": "2026-11-19T18:02:43.883316Z", + "connection": "mysql", + "lease_owner": "rv307-w-19f78f", + "available_at": "2026-11-19T18:02:43.816194Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:53.883238Z", + "last_dispatched_at": "2026-11-19T18:02:43.820116Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:43.820116Z" + }, + "activity": { + "id": "01m7tfk9q90ad1n3kyggax7cr9", + "type": "slow", + "class": "slow", + "queue": "rv307-8e959d", + "status": "pending", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "created_at": "2026-11-19T18:02:44.329794Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfk9q90ad1n3kyggax7cr9" + }, + "sequence": 2, + "activity_type": "slow", + "activity_class": "slow", + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "activity_execution_id": "01m7tfk9q90ad1n3kyggax7cr9", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + }, + "workflow_task_id": "01m7tfk978ccwhqf13q93eg8ax", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:44.330905Z" + }, + { + "id": "01m7tfkc40r3bjg0q4bn6rgmhd", + "sequence": 5, + "event_type": "TimerFired", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfk9q8270rb64bs4nveskr", + "type": "timer", + "queue": "rv307-8e959d", + "status": "leased", + "leased_at": "2026-11-19T18:02:46.765765Z", + "connection": "mysql", + "lease_owner": "01m7tfk9q8270rb64bs4nveskr", + "available_at": "2026-11-19T18:02:45.323760Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:56.765769Z" + }, + "fire_at": "2026-11-19T18:02:45.323760Z", + "fired_at": "2026-11-19T18:02:46.782499Z", + "sequence": 1, + "timer_id": "01m7tfk9q38v1smwprkqhb6dgq", + "delay_seconds": 1, + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + }, + "workflow_task_id": "01m7tfk9q8270rb64bs4nveskr", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:46.784048Z" + }, + { + "id": "01m7tfkc45d9ztxjf4d5kchf8p", + "sequence": 6, + "event_type": "SelectionResolved", + "namespace": "default", + "payload": { + "outcome": "completed", + "member_key": "t", + "member_size": 1, + "member_index": 0, + "operation_kind": "timer", + "operation_identity": "01m7tfk9q38v1smwprkqhb6dgq", + "selection_group_id": "select-calls:1:2", + "resolution_event_id": "01m7tfkc40r3bjg0q4bn6rgmhd", + "member_base_sequence": 1, + "selection_group_size": 2, + "resolution_event_type": "TimerFired", + "selection_group_base_sequence": 1 + }, + "workflow_task_id": null, + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:46.789725Z" + }, + { + "id": "01m7tfkcm6q48sthxdftepr0z6", + "sequence": 7, + "event_type": "TimerScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfkc48avwsavasvsj3esz8", + "type": "workflow", + "queue": "rv307-8e959d", + "status": "leased", + "leased_at": "2026-11-19T18:02:46.907807Z", + "connection": "mysql", + "lease_owner": "rv307-w-19f78f", + "available_at": "2026-11-19T18:02:46.792023Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:56.907768Z", + "last_dispatched_at": "2026-11-19T18:02:46.812847Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:46.812847Z" + }, + "fire_at": "2026-11-19T18:02:48.301156Z", + "sequence": 3, + "timer_id": "01m7tfkcm5e8mgsdpzcmh9cdge", + "delay_seconds": 1 + }, + "workflow_task_id": "01m7tfkc48avwsavasvsj3esz8", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:47.302320Z" + }, + { + "id": "01m7tfkf51n5petjghf8spznk0", + "sequence": 8, + "event_type": "TimerFired", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfkcm82egyep2ggssvtmk7", + "type": "timer", + "queue": "rv307-8e959d", + "status": "leased", + "leased_at": "2026-11-19T18:02:49.853853Z", + "connection": "mysql", + "lease_owner": "01m7tfkcm82egyep2ggssvtmk7", + "available_at": "2026-11-19T18:02:48.301156Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:59.853860Z" + }, + "fire_at": "2026-11-19T18:02:48.301156Z", + "fired_at": "2026-11-19T18:02:49.886204Z", + "sequence": 3, + "timer_id": "01m7tfkcm5e8mgsdpzcmh9cdge", + "delay_seconds": 1 + }, + "workflow_task_id": "01m7tfkcm82egyep2ggssvtmk7", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:49.887919Z" + }, + { + "id": "01m7tfkfqvysr7r9j2w4cka3f7", + "sequence": 9, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfkf55t0fp2b9z4xqcwq17", + "type": "workflow", + "queue": "rv307-8e959d", + "status": "leased", + "leased_at": "2026-11-19T18:02:50.012285Z", + "connection": "mysql", + "lease_owner": "rv307-w-19f78f", + "available_at": "2026-11-19T18:02:49.893199Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:00.012245Z", + "last_dispatched_at": "2026-11-19T18:02:49.930005Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:49.930005Z" + }, + "activity": { + "id": "01m7tfkfqrsdc6xarvd8m4k4bh", + "type": "m0", + "class": "m0", + "queue": "rv307-8e959d", + "status": "pending", + "sequence": 4, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "created_at": "2026-11-19T18:02:50.488097Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfkfqrsdc6xarvd8m4k4bh" + }, + "sequence": 4, + "activity_type": "m0", + "activity_class": "m0", + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfkfqrsdc6xarvd8m4k4bh", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfkf55t0fp2b9z4xqcwq17", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:50.490214Z" + }, + { + "id": "01m7tfkfr6n6sf1ybxbh86gf81", + "sequence": 10, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfkf55t0fp2b9z4xqcwq17", + "type": "workflow", + "queue": "rv307-8e959d", + "status": "leased", + "leased_at": "2026-11-19T18:02:50.012285Z", + "connection": "mysql", + "lease_owner": "rv307-w-19f78f", + "available_at": "2026-11-19T18:02:49.893199Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:00.012245Z", + "last_dispatched_at": "2026-11-19T18:02:49.930005Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:49.930005Z" + }, + "activity": { + "id": "01m7tfkfr30p7j002dxayv9gkc", + "type": "m1", + "class": "m1", + "queue": "rv307-8e959d", + "status": "pending", + "sequence": 5, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "created_at": "2026-11-19T18:02:50.499158Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfkfr30p7j002dxayv9gkc" + }, + "sequence": 5, + "activity_type": "m1", + "activity_class": "m1", + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfkfr30p7j002dxayv9gkc", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfkf55t0fp2b9z4xqcwq17", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:50.501069Z" + }, + { + "id": "01m7tfkftzys2v7he7a7dwyrpe", + "sequence": 11, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfk9qcb33ezaka0b7mf50x", + "type": "activity", + "queue": "rv307-8e959d", + "status": "leased", + "leased_at": "2026-11-19T18:02:50.584599Z", + "connection": "mysql", + "lease_owner": "rv307-w-19f78f", + "available_at": "2026-11-19T18:02:44.332793Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:50.584720Z" + }, + "activity": { + "id": "01m7tfk9q90ad1n3kyggax7cr9", + "type": "slow", + "class": "slow", + "queue": "rv307-8e959d", + "status": "running", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "attempt_id": "01M7TFKFTR32G62EJCYVHK8N2E", + "created_at": "2026-11-19T18:02:44.329794Z", + "started_at": "2026-11-19T18:02:50.584599Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfk9q90ad1n3kyggax7cr9" + }, + "sequence": 2, + "activity_type": "slow", + "activity_class": "slow", + "attempt_number": 1, + "parallel_group_id": "select-calls:1:2", + "activity_attempt_id": "01M7TFKFTR32G62EJCYVHK8N2E", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "activity_execution_id": "01m7tfk9q90ad1n3kyggax7cr9", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + }, + "workflow_task_id": "01m7tfk9qcb33ezaka0b7mf50x", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:50.590973Z" + }, + { + "id": "01m7tfkg0phjcyxepjt1vafbhd", + "sequence": 12, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfkfqzd1hmgj9q53tn07p7", + "type": "activity", + "queue": "rv307-8e959d", + "status": "leased", + "leased_at": "2026-11-19T18:02:50.763285Z", + "connection": "mysql", + "lease_owner": "rv307-w-19f78f", + "available_at": "2026-11-19T18:02:50.495104Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:50.763452Z" + }, + "activity": { + "id": "01m7tfkfqrsdc6xarvd8m4k4bh", + "type": "m0", + "class": "m0", + "queue": "rv307-8e959d", + "status": "running", + "sequence": 4, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "attempt_id": "01M7TFKG0BY81HXS6MNMAWHG36", + "created_at": "2026-11-19T18:02:50.488097Z", + "started_at": "2026-11-19T18:02:50.763285Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfkfqrsdc6xarvd8m4k4bh" + }, + "sequence": 4, + "activity_type": "m0", + "activity_class": "m0", + "attempt_number": 1, + "parallel_group_id": "parallel-activities:4:2", + "activity_attempt_id": "01M7TFKG0BY81HXS6MNMAWHG36", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfkfqrsdc6xarvd8m4k4bh", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfkfqzd1hmgj9q53tn07p7", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:50.773344Z" + }, + { + "id": "01m7tfkg7er2yv979fs2pqwtvk", + "sequence": 13, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfkfr8pkj8tf6cvb5w31x0", + "type": "activity", + "queue": "rv307-8e959d", + "status": "leased", + "leased_at": "2026-11-19T18:02:50.976518Z", + "connection": "mysql", + "lease_owner": "rv307-w-19f78f", + "available_at": "2026-11-19T18:02:50.504755Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:50.976682Z" + }, + "activity": { + "id": "01m7tfkfr30p7j002dxayv9gkc", + "type": "m1", + "class": "m1", + "queue": "rv307-8e959d", + "status": "running", + "sequence": 5, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "attempt_id": "01M7TFKG70HWZXWTKE2F841T68", + "created_at": "2026-11-19T18:02:50.499158Z", + "started_at": "2026-11-19T18:02:50.976518Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfkfr30p7j002dxayv9gkc" + }, + "sequence": 5, + "activity_type": "m1", + "activity_class": "m1", + "attempt_number": 1, + "parallel_group_id": "parallel-activities:4:2", + "activity_attempt_id": "01M7TFKG70HWZXWTKE2F841T68", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfkfr30p7j002dxayv9gkc", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfkfr8pkj8tf6cvb5w31x0", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:50.989732Z" + }, + { + "id": "01m7tfkgc7804qv2d6fn26qmj3", + "sequence": 14, + "event_type": "ActivityFailed", + "namespace": "default", + "payload": { + "code": 0, + "task": { + "id": "01m7tfkfr8pkj8tf6cvb5w31x0", + "type": "activity", + "queue": "rv307-8e959d", + "status": "leased", + "leased_at": "2026-11-19T18:02:50.976518Z", + "connection": "mysql", + "lease_owner": "rv307-w-19f78f", + "available_at": "2026-11-19T18:02:50.504755Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:51.109491Z" + }, + "message": "fail-m1", + "activity": { + "id": "01m7tfkfr30p7j002dxayv9gkc", + "type": "m1", + "class": "m1", + "queue": "rv307-8e959d", + "status": "failed", + "sequence": 5, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "closed_at": "2026-11-19T18:02:51.141600Z", + "exception": { + "code": 0, + "type": null, + "message": "fail-m1", + "non_retryable": true + }, + "attempt_id": "01M7TFKG70HWZXWTKE2F841T68", + "created_at": "2026-11-19T18:02:50.499158Z", + "started_at": "2026-11-19T18:02:50.976518Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfkfr30p7j002dxayv9gkc" + }, + "sequence": 5, + "exception": { + "type": null, + "message": "fail-m1", + "code": 0, + "non_retryable": true + }, + "failure_id": "01m7tfkgc5h6x68np5d9qjfb18", + "activity_type": "m1", + "non_retryable": true, + "activity_class": "m1", + "attempt_number": 1, + "exception_type": null, + "exception_class": "RuntimeException", + "failure_category": "activity", + "parallel_group_id": "parallel-activities:4:2", + "activity_attempt_id": "01M7TFKG70HWZXWTKE2F841T68", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfkfr30p7j002dxayv9gkc", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfkfr8pkj8tf6cvb5w31x0", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:51.142816Z" + }, + { + "id": "01m7tfkhn2qhx6fvcn2yx61dbv", + "sequence": 15, + "event_type": "ActivityFailed", + "namespace": "default", + "payload": { + "code": 0, + "task": { + "id": "01m7tfkfqzd1hmgj9q53tn07p7", + "type": "activity", + "queue": "rv307-8e959d", + "status": "leased", + "leased_at": "2026-11-19T18:02:50.763285Z", + "connection": "mysql", + "lease_owner": "rv307-w-19f78f", + "available_at": "2026-11-19T18:02:50.495104Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:50.901820Z" + }, + "message": "fail-m0", + "activity": { + "id": "01m7tfkfqrsdc6xarvd8m4k4bh", + "type": "m0", + "class": "m0", + "queue": "rv307-8e959d", + "status": "failed", + "sequence": 4, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "closed_at": "2026-11-19T18:02:52.447729Z", + "exception": { + "code": 0, + "type": null, + "message": "fail-m0", + "non_retryable": true + }, + "attempt_id": "01M7TFKG0BY81HXS6MNMAWHG36", + "created_at": "2026-11-19T18:02:50.488097Z", + "started_at": "2026-11-19T18:02:50.763285Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfkfqrsdc6xarvd8m4k4bh" + }, + "sequence": 4, + "exception": { + "type": null, + "message": "fail-m0", + "code": 0, + "non_retryable": true + }, + "failure_id": "01m7tfkhmzrb7h1fkt3nm0rb61", + "activity_type": "m0", + "non_retryable": true, + "activity_class": "m0", + "attempt_number": 1, + "exception_type": null, + "exception_class": "RuntimeException", + "failure_category": "activity", + "parallel_group_id": "parallel-activities:4:2", + "activity_attempt_id": "01M7TFKG0BY81HXS6MNMAWHG36", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfkfqrsdc6xarvd8m4k4bh", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfkfqzd1hmgj9q53tn07p7", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:52.449810Z" + } + ], + "expected": { + "query_result": "fail-m0" + } +} diff --git a/tests/fixtures/replay_regressions/failure-order-released-select-cancel.json b/tests/fixtures/replay_regressions/failure-order-released-select-cancel.json new file mode 100644 index 00000000..50f7511c --- /dev/null +++ b/tests/fixtures/replay_regressions/failure-order-released-select-cancel.json @@ -0,0 +1,1019 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "failure-order-released-select-cancel", + "protocol_version": "1.20", + "bindings": [ + "python" + ], + "parallel_failure_order": { + "issues": [ + "durable-workflow/sdk-python#307" + ], + "note": "Recorded by published 2.19.1 on a Server built from durable-workflow/server main 52e3ec00 (docker-compose.test.yml): the group (m0, m1) was scheduled without the marker and m1 failed, then m0. The task that saw the settled group catches fail-m0, cancels a select() loser and compensates fail-m0; this version sends the same commands." + }, + "workflow": { + "type": "rv348_cancel", + "input": [], + "payload_codec": "avro", + "run_id": "01m7tfk96p1vr4vnrwf2k7mwv0", + "workflow_id": "rv307-rv348_cancel-1af5ee" + }, + "history": [ + { + "id": "01m7tfk96zfcqzq0fzt4gvb691", + "sequence": 1, + "event_type": "StartAccepted", + "namespace": "default", + "payload": { + "memo": null, + "command": { + "id": "01m7tfk96txfwjnf900w6b7bxe", + "type": "start", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "outcome": "started_new", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "sequence": 1, + "applied_at": "2026-11-19T18:02:43.799220Z", + "accepted_at": "2026-11-19T18:02:43.799209Z", + "auth_method": "token", + "auth_status": "authorized", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfk96p1vr4vnrwf2k7mwv0", + "request_route_name": "generated::FqJuJMl97iiXlIDz", + "request_fingerprint": "sha256:0cd5a8b1b56226fec881b1357a0042ba40782bf855be16c912d3224b8d65cf66" + }, + "outcome": "started_new", + "business_key": null, + "workflow_type": "rv348_cancel", + "workflow_class": "rv348_cancel", + "workflow_run_id": "01m7tfk96p1vr4vnrwf2k7mwv0", + "search_attributes": null, + "visibility_labels": null, + "workflow_command_id": "01m7tfk96txfwjnf900w6b7bxe", + "workflow_instance_id": "rv307-rv348_cancel-1af5ee" + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfk96txfwjnf900w6b7bxe", + "recorded_at": "2026-11-19T18:02:43.805823Z" + }, + { + "id": "01m7tfk975p13m1k1bs73t658v", + "sequence": 2, + "event_type": "WorkflowStarted", + "namespace": "default", + "payload": { + "command": { + "id": "01m7tfk96txfwjnf900w6b7bxe", + "type": "start", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "outcome": "started_new", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "sequence": 1, + "applied_at": "2026-11-19T18:02:43.799220Z", + "accepted_at": "2026-11-19T18:02:43.799209Z", + "auth_method": "token", + "auth_status": "authorized", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfk96p1vr4vnrwf2k7mwv0", + "request_route_name": "generated::FqJuJMl97iiXlIDz", + "request_fingerprint": "sha256:0cd5a8b1b56226fec881b1357a0042ba40782bf855be16c912d3224b8d65cf66" + }, + "workflow_type": "rv348_cancel", + "workflow_class": "rv348_cancel", + "run_deadline_at": "2026-11-19T18:12:43+00:00", + "workflow_run_id": "01m7tfk96p1vr4vnrwf2k7mwv0", + "declared_queries": [ + "caught" + ], + "declared_signals": [], + "declared_updates": [ + "report" + ], + "declared_entry_mode": "canonical", + "run_timeout_seconds": 600, + "workflow_command_id": "01m7tfk96txfwjnf900w6b7bxe", + "workflow_instance_id": "rv307-rv348_cancel-1af5ee", + "declared_entry_method": "handle", + "execution_deadline_at": "2026-11-19T19:02:43+00:00", + "declared_query_contracts": [ + { + "name": "caught", + "parameters": [] + } + ], + "declared_signal_contracts": [], + "declared_update_contracts": [ + { + "name": "report", + "parameters": [ + { + "name": "_", + "type": null, + "default": null, + "position": 0, + "required": false, + "variadic": true, + "allows_null": true, + "default_available": false + } + ] + } + ], + "execution_timeout_seconds": 3600, + "declared_update_validators": [], + "declared_entry_declaring_class": "external:rv348_cancel", + "workflow_definition_fingerprint": "sha256:4f8f85902f58f49284c524502cb9366458411dc0768ba01d6bef05a43541d21f", + "workflow_definition_fingerprint_source": "worker" + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfk96txfwjnf900w6b7bxe", + "recorded_at": "2026-11-19T18:02:43.811611Z" + }, + { + "id": "01m7tfk9q5ppwba7xzpqqwy7ty", + "sequence": 3, + "event_type": "TimerScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfk978ccwhqf13q93eg8ax", + "type": "workflow", + "queue": "rv307-8e959d", + "status": "leased", + "leased_at": "2026-11-19T18:02:43.883316Z", + "connection": "mysql", + "lease_owner": "rv307-w-19f78f", + "available_at": "2026-11-19T18:02:43.816194Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:53.883238Z", + "last_dispatched_at": "2026-11-19T18:02:43.820116Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:43.820116Z" + }, + "fire_at": "2026-11-19T18:02:45.323760Z", + "sequence": 1, + "timer_id": "01m7tfk9q38v1smwprkqhb6dgq", + "delay_seconds": 1, + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + }, + "workflow_task_id": "01m7tfk978ccwhqf13q93eg8ax", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:44.325093Z" + }, + { + "id": "01m7tfk9qbms8jmz3whx68gbr8", + "sequence": 4, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfk978ccwhqf13q93eg8ax", + "type": "workflow", + "queue": "rv307-8e959d", + "status": "leased", + "leased_at": "2026-11-19T18:02:43.883316Z", + "connection": "mysql", + "lease_owner": "rv307-w-19f78f", + "available_at": "2026-11-19T18:02:43.816194Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:53.883238Z", + "last_dispatched_at": "2026-11-19T18:02:43.820116Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:43.820116Z" + }, + "activity": { + "id": "01m7tfk9q90ad1n3kyggax7cr9", + "type": "slow", + "class": "slow", + "queue": "rv307-8e959d", + "status": "pending", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "created_at": "2026-11-19T18:02:44.329794Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfk9q90ad1n3kyggax7cr9" + }, + "sequence": 2, + "activity_type": "slow", + "activity_class": "slow", + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "activity_execution_id": "01m7tfk9q90ad1n3kyggax7cr9", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + }, + "workflow_task_id": "01m7tfk978ccwhqf13q93eg8ax", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:44.330905Z" + }, + { + "id": "01m7tfkc40r3bjg0q4bn6rgmhd", + "sequence": 5, + "event_type": "TimerFired", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfk9q8270rb64bs4nveskr", + "type": "timer", + "queue": "rv307-8e959d", + "status": "leased", + "leased_at": "2026-11-19T18:02:46.765765Z", + "connection": "mysql", + "lease_owner": "01m7tfk9q8270rb64bs4nveskr", + "available_at": "2026-11-19T18:02:45.323760Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:56.765769Z" + }, + "fire_at": "2026-11-19T18:02:45.323760Z", + "fired_at": "2026-11-19T18:02:46.782499Z", + "sequence": 1, + "timer_id": "01m7tfk9q38v1smwprkqhb6dgq", + "delay_seconds": 1, + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + }, + "workflow_task_id": "01m7tfk9q8270rb64bs4nveskr", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:46.784048Z" + }, + { + "id": "01m7tfkc45d9ztxjf4d5kchf8p", + "sequence": 6, + "event_type": "SelectionResolved", + "namespace": "default", + "payload": { + "outcome": "completed", + "member_key": "t", + "member_size": 1, + "member_index": 0, + "operation_kind": "timer", + "operation_identity": "01m7tfk9q38v1smwprkqhb6dgq", + "selection_group_id": "select-calls:1:2", + "resolution_event_id": "01m7tfkc40r3bjg0q4bn6rgmhd", + "member_base_sequence": 1, + "selection_group_size": 2, + "resolution_event_type": "TimerFired", + "selection_group_base_sequence": 1 + }, + "workflow_task_id": null, + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:46.789725Z" + }, + { + "id": "01m7tfkcm6q48sthxdftepr0z6", + "sequence": 7, + "event_type": "TimerScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfkc48avwsavasvsj3esz8", + "type": "workflow", + "queue": "rv307-8e959d", + "status": "leased", + "leased_at": "2026-11-19T18:02:46.907807Z", + "connection": "mysql", + "lease_owner": "rv307-w-19f78f", + "available_at": "2026-11-19T18:02:46.792023Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:56.907768Z", + "last_dispatched_at": "2026-11-19T18:02:46.812847Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:46.812847Z" + }, + "fire_at": "2026-11-19T18:02:48.301156Z", + "sequence": 3, + "timer_id": "01m7tfkcm5e8mgsdpzcmh9cdge", + "delay_seconds": 1 + }, + "workflow_task_id": "01m7tfkc48avwsavasvsj3esz8", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:47.302320Z" + }, + { + "id": "01m7tfkf51n5petjghf8spznk0", + "sequence": 8, + "event_type": "TimerFired", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfkcm82egyep2ggssvtmk7", + "type": "timer", + "queue": "rv307-8e959d", + "status": "leased", + "leased_at": "2026-11-19T18:02:49.853853Z", + "connection": "mysql", + "lease_owner": "01m7tfkcm82egyep2ggssvtmk7", + "available_at": "2026-11-19T18:02:48.301156Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:59.853860Z" + }, + "fire_at": "2026-11-19T18:02:48.301156Z", + "fired_at": "2026-11-19T18:02:49.886204Z", + "sequence": 3, + "timer_id": "01m7tfkcm5e8mgsdpzcmh9cdge", + "delay_seconds": 1 + }, + "workflow_task_id": "01m7tfkcm82egyep2ggssvtmk7", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:49.887919Z" + }, + { + "id": "01m7tfkfqvysr7r9j2w4cka3f7", + "sequence": 9, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfkf55t0fp2b9z4xqcwq17", + "type": "workflow", + "queue": "rv307-8e959d", + "status": "leased", + "leased_at": "2026-11-19T18:02:50.012285Z", + "connection": "mysql", + "lease_owner": "rv307-w-19f78f", + "available_at": "2026-11-19T18:02:49.893199Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:00.012245Z", + "last_dispatched_at": "2026-11-19T18:02:49.930005Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:49.930005Z" + }, + "activity": { + "id": "01m7tfkfqrsdc6xarvd8m4k4bh", + "type": "m0", + "class": "m0", + "queue": "rv307-8e959d", + "status": "pending", + "sequence": 4, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "created_at": "2026-11-19T18:02:50.488097Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfkfqrsdc6xarvd8m4k4bh" + }, + "sequence": 4, + "activity_type": "m0", + "activity_class": "m0", + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfkfqrsdc6xarvd8m4k4bh", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfkf55t0fp2b9z4xqcwq17", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:50.490214Z" + }, + { + "id": "01m7tfkfr6n6sf1ybxbh86gf81", + "sequence": 10, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfkf55t0fp2b9z4xqcwq17", + "type": "workflow", + "queue": "rv307-8e959d", + "status": "leased", + "leased_at": "2026-11-19T18:02:50.012285Z", + "connection": "mysql", + "lease_owner": "rv307-w-19f78f", + "available_at": "2026-11-19T18:02:49.893199Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:00.012245Z", + "last_dispatched_at": "2026-11-19T18:02:49.930005Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:49.930005Z" + }, + "activity": { + "id": "01m7tfkfr30p7j002dxayv9gkc", + "type": "m1", + "class": "m1", + "queue": "rv307-8e959d", + "status": "pending", + "sequence": 5, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "created_at": "2026-11-19T18:02:50.499158Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfkfr30p7j002dxayv9gkc" + }, + "sequence": 5, + "activity_type": "m1", + "activity_class": "m1", + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfkfr30p7j002dxayv9gkc", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfkf55t0fp2b9z4xqcwq17", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:50.501069Z" + }, + { + "id": "01m7tfkftzys2v7he7a7dwyrpe", + "sequence": 11, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfk9qcb33ezaka0b7mf50x", + "type": "activity", + "queue": "rv307-8e959d", + "status": "leased", + "leased_at": "2026-11-19T18:02:50.584599Z", + "connection": "mysql", + "lease_owner": "rv307-w-19f78f", + "available_at": "2026-11-19T18:02:44.332793Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:50.584720Z" + }, + "activity": { + "id": "01m7tfk9q90ad1n3kyggax7cr9", + "type": "slow", + "class": "slow", + "queue": "rv307-8e959d", + "status": "running", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "attempt_id": "01M7TFKFTR32G62EJCYVHK8N2E", + "created_at": "2026-11-19T18:02:44.329794Z", + "started_at": "2026-11-19T18:02:50.584599Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfk9q90ad1n3kyggax7cr9" + }, + "sequence": 2, + "activity_type": "slow", + "activity_class": "slow", + "attempt_number": 1, + "parallel_group_id": "select-calls:1:2", + "activity_attempt_id": "01M7TFKFTR32G62EJCYVHK8N2E", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "activity_execution_id": "01m7tfk9q90ad1n3kyggax7cr9", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + }, + "workflow_task_id": "01m7tfk9qcb33ezaka0b7mf50x", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:50.590973Z" + }, + { + "id": "01m7tfkg0phjcyxepjt1vafbhd", + "sequence": 12, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfkfqzd1hmgj9q53tn07p7", + "type": "activity", + "queue": "rv307-8e959d", + "status": "leased", + "leased_at": "2026-11-19T18:02:50.763285Z", + "connection": "mysql", + "lease_owner": "rv307-w-19f78f", + "available_at": "2026-11-19T18:02:50.495104Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:50.763452Z" + }, + "activity": { + "id": "01m7tfkfqrsdc6xarvd8m4k4bh", + "type": "m0", + "class": "m0", + "queue": "rv307-8e959d", + "status": "running", + "sequence": 4, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "attempt_id": "01M7TFKG0BY81HXS6MNMAWHG36", + "created_at": "2026-11-19T18:02:50.488097Z", + "started_at": "2026-11-19T18:02:50.763285Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfkfqrsdc6xarvd8m4k4bh" + }, + "sequence": 4, + "activity_type": "m0", + "activity_class": "m0", + "attempt_number": 1, + "parallel_group_id": "parallel-activities:4:2", + "activity_attempt_id": "01M7TFKG0BY81HXS6MNMAWHG36", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfkfqrsdc6xarvd8m4k4bh", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfkfqzd1hmgj9q53tn07p7", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:50.773344Z" + }, + { + "id": "01m7tfkg7er2yv979fs2pqwtvk", + "sequence": 13, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfkfr8pkj8tf6cvb5w31x0", + "type": "activity", + "queue": "rv307-8e959d", + "status": "leased", + "leased_at": "2026-11-19T18:02:50.976518Z", + "connection": "mysql", + "lease_owner": "rv307-w-19f78f", + "available_at": "2026-11-19T18:02:50.504755Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:50.976682Z" + }, + "activity": { + "id": "01m7tfkfr30p7j002dxayv9gkc", + "type": "m1", + "class": "m1", + "queue": "rv307-8e959d", + "status": "running", + "sequence": 5, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "attempt_id": "01M7TFKG70HWZXWTKE2F841T68", + "created_at": "2026-11-19T18:02:50.499158Z", + "started_at": "2026-11-19T18:02:50.976518Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfkfr30p7j002dxayv9gkc" + }, + "sequence": 5, + "activity_type": "m1", + "activity_class": "m1", + "attempt_number": 1, + "parallel_group_id": "parallel-activities:4:2", + "activity_attempt_id": "01M7TFKG70HWZXWTKE2F841T68", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfkfr30p7j002dxayv9gkc", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfkfr8pkj8tf6cvb5w31x0", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:50.989732Z" + }, + { + "id": "01m7tfkgc7804qv2d6fn26qmj3", + "sequence": 14, + "event_type": "ActivityFailed", + "namespace": "default", + "payload": { + "code": 0, + "task": { + "id": "01m7tfkfr8pkj8tf6cvb5w31x0", + "type": "activity", + "queue": "rv307-8e959d", + "status": "leased", + "leased_at": "2026-11-19T18:02:50.976518Z", + "connection": "mysql", + "lease_owner": "rv307-w-19f78f", + "available_at": "2026-11-19T18:02:50.504755Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:51.109491Z" + }, + "message": "fail-m1", + "activity": { + "id": "01m7tfkfr30p7j002dxayv9gkc", + "type": "m1", + "class": "m1", + "queue": "rv307-8e959d", + "status": "failed", + "sequence": 5, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "closed_at": "2026-11-19T18:02:51.141600Z", + "exception": { + "code": 0, + "type": null, + "message": "fail-m1", + "non_retryable": true + }, + "attempt_id": "01M7TFKG70HWZXWTKE2F841T68", + "created_at": "2026-11-19T18:02:50.499158Z", + "started_at": "2026-11-19T18:02:50.976518Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfkfr30p7j002dxayv9gkc" + }, + "sequence": 5, + "exception": { + "type": null, + "message": "fail-m1", + "code": 0, + "non_retryable": true + }, + "failure_id": "01m7tfkgc5h6x68np5d9qjfb18", + "activity_type": "m1", + "non_retryable": true, + "activity_class": "m1", + "attempt_number": 1, + "exception_type": null, + "exception_class": "RuntimeException", + "failure_category": "activity", + "parallel_group_id": "parallel-activities:4:2", + "activity_attempt_id": "01M7TFKG70HWZXWTKE2F841T68", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfkfr30p7j002dxayv9gkc", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfkfr8pkj8tf6cvb5w31x0", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:51.142816Z" + }, + { + "id": "01m7tfkhn2qhx6fvcn2yx61dbv", + "sequence": 15, + "event_type": "ActivityFailed", + "namespace": "default", + "payload": { + "code": 0, + "task": { + "id": "01m7tfkfqzd1hmgj9q53tn07p7", + "type": "activity", + "queue": "rv307-8e959d", + "status": "leased", + "leased_at": "2026-11-19T18:02:50.763285Z", + "connection": "mysql", + "lease_owner": "rv307-w-19f78f", + "available_at": "2026-11-19T18:02:50.495104Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:50.901820Z" + }, + "message": "fail-m0", + "activity": { + "id": "01m7tfkfqrsdc6xarvd8m4k4bh", + "type": "m0", + "class": "m0", + "queue": "rv307-8e959d", + "status": "failed", + "sequence": 4, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "closed_at": "2026-11-19T18:02:52.447729Z", + "exception": { + "code": 0, + "type": null, + "message": "fail-m0", + "non_retryable": true + }, + "attempt_id": "01M7TFKG0BY81HXS6MNMAWHG36", + "created_at": "2026-11-19T18:02:50.488097Z", + "started_at": "2026-11-19T18:02:50.763285Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfkfqrsdc6xarvd8m4k4bh" + }, + "sequence": 4, + "exception": { + "type": null, + "message": "fail-m0", + "code": 0, + "non_retryable": true + }, + "failure_id": "01m7tfkhmzrb7h1fkt3nm0rb61", + "activity_type": "m0", + "non_retryable": true, + "activity_class": "m0", + "attempt_number": 1, + "exception_type": null, + "exception_class": "RuntimeException", + "failure_category": "activity", + "parallel_group_id": "parallel-activities:4:2", + "activity_attempt_id": "01M7TFKG0BY81HXS6MNMAWHG36", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfkfqrsdc6xarvd8m4k4bh", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfkfqzd1hmgj9q53tn07p7", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:52.449810Z" + } + ], + "expected": { + "command_sequence": [ + { + "type": "cancel_selection_operation" + }, + { + "type": "schedule_activity", + "activity_type": "compensate", + "arguments": [ + "fail-m0" + ] + } + ] + } +} diff --git a/tests/fixtures/replay_regressions/failure-order-released-update-before-new-worker-query.json b/tests/fixtures/replay_regressions/failure-order-released-update-before-new-worker-query.json new file mode 100644 index 00000000..c247bb77 --- /dev/null +++ b/tests/fixtures/replay_regressions/failure-order-released-update-before-new-worker-query.json @@ -0,0 +1,882 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "failure-order-released-update-before-new-worker-query", + "protocol_version": "1.20", + "bindings": [ + "python" + ], + "parallel_failure_order": { + "issues": [ + "durable-workflow/sdk-python#307" + ], + "note": "Recorded on a Server built from durable-workflow/server main 52e3ec00 (docker-compose.test.yml). Published 2.19.1 scheduled the group without the marker and, after m1 then m0 failed, answered an update with {compensating: fail-m0}. This is the history the next workflow task received; a worker of this version ran it and compensates fail-m0, matching the recorded update." + }, + "workflow": { + "type": "rv348_plain", + "input": [], + "payload_codec": "avro", + "run_id": "01m7tfm3yw32tg5eartkhtbmha", + "workflow_id": "rv307-rv348_plain-969f72", + "query": { + "name": "caught", + "arguments": [] + } + }, + "history": [ + { + "id": "01m7tfm3z1hjhej16rx9zzwhcs", + "sequence": 1, + "event_type": "StartAccepted", + "namespace": "default", + "payload": { + "memo": null, + "command": { + "id": "01m7tfm3yym2gnn3pt5hsx2vv9", + "type": "start", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "outcome": "started_new", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "sequence": 1, + "applied_at": "2026-11-19T18:03:11.197425Z", + "accepted_at": "2026-11-19T18:03:11.197423Z", + "auth_method": "token", + "auth_status": "authorized", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfm3yw32tg5eartkhtbmha", + "request_route_name": "generated::FqJuJMl97iiXlIDz", + "request_fingerprint": "sha256:e3f59c6f64bc66ea91e92991ab872ea6eca4e831ad1fbabb7ebf36b7b136b9be" + }, + "outcome": "started_new", + "business_key": null, + "workflow_type": "rv348_plain", + "workflow_class": "rv348_plain", + "workflow_run_id": "01m7tfm3yw32tg5eartkhtbmha", + "search_attributes": null, + "visibility_labels": null, + "workflow_command_id": "01m7tfm3yym2gnn3pt5hsx2vv9", + "workflow_instance_id": "rv307-rv348_plain-969f72" + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfm3yym2gnn3pt5hsx2vv9", + "recorded_at": "2026-11-19T18:03:11.200743Z" + }, + { + "id": "01m7tfm3z4cx71efh954xgbwz2", + "sequence": 2, + "event_type": "WorkflowStarted", + "namespace": "default", + "payload": { + "command": { + "id": "01m7tfm3yym2gnn3pt5hsx2vv9", + "type": "start", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "outcome": "started_new", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "sequence": 1, + "applied_at": "2026-11-19T18:03:11.197425Z", + "accepted_at": "2026-11-19T18:03:11.197423Z", + "auth_method": "token", + "auth_status": "authorized", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfm3yw32tg5eartkhtbmha", + "request_route_name": "generated::FqJuJMl97iiXlIDz", + "request_fingerprint": "sha256:e3f59c6f64bc66ea91e92991ab872ea6eca4e831ad1fbabb7ebf36b7b136b9be" + }, + "workflow_type": "rv348_plain", + "workflow_class": "rv348_plain", + "run_deadline_at": "2026-11-19T18:13:11+00:00", + "workflow_run_id": "01m7tfm3yw32tg5eartkhtbmha", + "declared_queries": [ + "caught" + ], + "declared_signals": [], + "declared_updates": [ + "report" + ], + "declared_entry_mode": "canonical", + "run_timeout_seconds": 600, + "workflow_command_id": "01m7tfm3yym2gnn3pt5hsx2vv9", + "workflow_instance_id": "rv307-rv348_plain-969f72", + "declared_entry_method": "handle", + "execution_deadline_at": "2026-11-19T19:03:11+00:00", + "declared_query_contracts": [ + { + "name": "caught", + "parameters": [] + } + ], + "declared_signal_contracts": [], + "declared_update_contracts": [ + { + "name": "report", + "parameters": [ + { + "name": "_", + "type": null, + "default": null, + "position": 0, + "required": false, + "variadic": true, + "allows_null": true, + "default_available": false + } + ] + } + ], + "execution_timeout_seconds": 3600, + "declared_update_validators": [], + "declared_entry_declaring_class": "external:rv348_plain", + "workflow_definition_fingerprint": "sha256:4cca49f53721ff6d13fd7cd42526e63e2497bbd302876fde0ff10eb9748ac44b", + "workflow_definition_fingerprint_source": "worker" + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfm3yym2gnn3pt5hsx2vv9", + "recorded_at": "2026-11-19T18:03:11.203745Z" + }, + { + "id": "01m7tfm4bwk71w8axj3jqetxgp", + "sequence": 3, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfm3z50n0je42yt0fevysb", + "type": "workflow", + "queue": "rv307-474a8e", + "status": "leased", + "leased_at": "2026-11-19T18:03:11.251003Z", + "connection": "mysql", + "lease_owner": "rv307-w-d4abff", + "available_at": "2026-11-19T18:03:11.205523Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:21.250959Z", + "last_dispatched_at": "2026-11-19T18:03:11.207285Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:11.207285Z" + }, + "activity": { + "id": "01m7tfm4bvvtg0rmp0kr0w9ck2", + "type": "m0", + "class": "m0", + "queue": "rv307-474a8e", + "status": "pending", + "sequence": 1, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "created_at": "2026-11-19T18:03:11.611513Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfm4bvvtg0rmp0kr0w9ck2" + }, + "sequence": 1, + "activity_type": "m0", + "activity_class": "m0", + "parallel_group_id": "parallel-activities:1:2", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 1 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfm4bvvtg0rmp0kr0w9ck2", + "parallel_group_base_sequence": 1 + }, + "workflow_task_id": "01m7tfm3z50n0je42yt0fevysb", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:11.612487Z" + }, + { + "id": "01m7tfm4c2r7gabnqy2890rf31", + "sequence": 4, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfm3z50n0je42yt0fevysb", + "type": "workflow", + "queue": "rv307-474a8e", + "status": "leased", + "leased_at": "2026-11-19T18:03:11.251003Z", + "connection": "mysql", + "lease_owner": "rv307-w-d4abff", + "available_at": "2026-11-19T18:03:11.205523Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:21.250959Z", + "last_dispatched_at": "2026-11-19T18:03:11.207285Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:11.207285Z" + }, + "activity": { + "id": "01m7tfm4c0q1xx5tyc93p1w731", + "type": "m1", + "class": "m1", + "queue": "rv307-474a8e", + "status": "pending", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "created_at": "2026-11-19T18:03:11.616913Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfm4c0q1xx5tyc93p1w731" + }, + "sequence": 2, + "activity_type": "m1", + "activity_class": "m1", + "parallel_group_id": "parallel-activities:1:2", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 1 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfm4c0q1xx5tyc93p1w731", + "parallel_group_base_sequence": 1 + }, + "workflow_task_id": "01m7tfm3z50n0je42yt0fevysb", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:11.617688Z" + }, + { + "id": "01m7tfm4epm84kmz9gwgcv2pg8", + "sequence": 5, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfm4bzh83tp3754qvknbf5", + "type": "activity", + "queue": "rv307-474a8e", + "status": "leased", + "leased_at": "2026-11-19T18:03:11.695781Z", + "connection": "mysql", + "lease_owner": "rv307-w-d4abff", + "available_at": "2026-11-19T18:03:11.615206Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:11.695954Z" + }, + "activity": { + "id": "01m7tfm4bvvtg0rmp0kr0w9ck2", + "type": "m0", + "class": "m0", + "queue": "rv307-474a8e", + "status": "running", + "sequence": 1, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "attempt_id": "01M7TFM4EF3PTY17A9B0CDG6D8", + "created_at": "2026-11-19T18:03:11.611513Z", + "started_at": "2026-11-19T18:03:11.695781Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfm4bvvtg0rmp0kr0w9ck2" + }, + "sequence": 1, + "activity_type": "m0", + "activity_class": "m0", + "attempt_number": 1, + "parallel_group_id": "parallel-activities:1:2", + "activity_attempt_id": "01M7TFM4EF3PTY17A9B0CDG6D8", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 1 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfm4bvvtg0rmp0kr0w9ck2", + "parallel_group_base_sequence": 1 + }, + "workflow_task_id": "01m7tfm4bzh83tp3754qvknbf5", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:11.702308Z" + }, + { + "id": "01m7tfm4jrg64z2aw1xc6tpanc", + "sequence": 6, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfm4c33way08w985jma6vv", + "type": "activity", + "queue": "rv307-474a8e", + "status": "leased", + "leased_at": "2026-11-19T18:03:11.827788Z", + "connection": "mysql", + "lease_owner": "rv307-w-d4abff", + "available_at": "2026-11-19T18:03:11.619207Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:11.827870Z" + }, + "activity": { + "id": "01m7tfm4c0q1xx5tyc93p1w731", + "type": "m1", + "class": "m1", + "queue": "rv307-474a8e", + "status": "running", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "attempt_id": "01M7TFM4JKR3SEEJR8Q5HE1KMD", + "created_at": "2026-11-19T18:03:11.616913Z", + "started_at": "2026-11-19T18:03:11.827788Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfm4c0q1xx5tyc93p1w731" + }, + "sequence": 2, + "activity_type": "m1", + "activity_class": "m1", + "attempt_number": 1, + "parallel_group_id": "parallel-activities:1:2", + "activity_attempt_id": "01M7TFM4JKR3SEEJR8Q5HE1KMD", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 1 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfm4c0q1xx5tyc93p1w731", + "parallel_group_base_sequence": 1 + }, + "workflow_task_id": "01m7tfm4c33way08w985jma6vv", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:11.831851Z" + }, + { + "id": "01m7tfm4nqfbh62tycsn3rw88e", + "sequence": 7, + "event_type": "ActivityFailed", + "namespace": "default", + "payload": { + "code": 0, + "task": { + "id": "01m7tfm4c33way08w985jma6vv", + "type": "activity", + "queue": "rv307-474a8e", + "status": "leased", + "leased_at": "2026-11-19T18:03:11.827788Z", + "connection": "mysql", + "lease_owner": "rv307-w-d4abff", + "available_at": "2026-11-19T18:03:11.619207Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:11.891793Z" + }, + "message": "fail-m1", + "activity": { + "id": "01m7tfm4c0q1xx5tyc93p1w731", + "type": "m1", + "class": "m1", + "queue": "rv307-474a8e", + "status": "failed", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "closed_at": "2026-11-19T18:03:11.925626Z", + "exception": { + "code": 0, + "type": null, + "message": "fail-m1", + "non_retryable": true + }, + "attempt_id": "01M7TFM4JKR3SEEJR8Q5HE1KMD", + "created_at": "2026-11-19T18:03:11.616913Z", + "started_at": "2026-11-19T18:03:11.827788Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfm4c0q1xx5tyc93p1w731" + }, + "sequence": 2, + "exception": { + "type": null, + "message": "fail-m1", + "code": 0, + "non_retryable": true + }, + "failure_id": "01m7tfm4nntbjxs3bqtfzz53xg", + "activity_type": "m1", + "non_retryable": true, + "activity_class": "m1", + "attempt_number": 1, + "exception_type": null, + "exception_class": "RuntimeException", + "failure_category": "activity", + "parallel_group_id": "parallel-activities:1:2", + "activity_attempt_id": "01M7TFM4JKR3SEEJR8Q5HE1KMD", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 1 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfm4c0q1xx5tyc93p1w731", + "parallel_group_base_sequence": 1 + }, + "workflow_task_id": "01m7tfm4c33way08w985jma6vv", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:11.927075Z" + }, + { + "id": "01m7tfm5xkdvsmfdt93gbq9dnp", + "sequence": 8, + "event_type": "ActivityFailed", + "namespace": "default", + "payload": { + "code": 0, + "task": { + "id": "01m7tfm4bzh83tp3754qvknbf5", + "type": "activity", + "queue": "rv307-474a8e", + "status": "leased", + "leased_at": "2026-11-19T18:03:11.695781Z", + "connection": "mysql", + "lease_owner": "rv307-w-d4abff", + "available_at": "2026-11-19T18:03:11.615206Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:11.771243Z" + }, + "message": "fail-m0", + "activity": { + "id": "01m7tfm4bvvtg0rmp0kr0w9ck2", + "type": "m0", + "class": "m0", + "queue": "rv307-474a8e", + "status": "failed", + "sequence": 1, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "closed_at": "2026-11-19T18:03:13.201365Z", + "exception": { + "code": 0, + "type": null, + "message": "fail-m0", + "non_retryable": true + }, + "attempt_id": "01M7TFM4EF3PTY17A9B0CDG6D8", + "created_at": "2026-11-19T18:03:11.611513Z", + "started_at": "2026-11-19T18:03:11.695781Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfm4bvvtg0rmp0kr0w9ck2" + }, + "sequence": 1, + "exception": { + "type": null, + "message": "fail-m0", + "code": 0, + "non_retryable": true + }, + "failure_id": "01m7tfm5xgn5387eyfyw63pwe5", + "activity_type": "m0", + "non_retryable": true, + "activity_class": "m0", + "attempt_number": 1, + "exception_type": null, + "exception_class": "RuntimeException", + "failure_category": "activity", + "parallel_group_id": "parallel-activities:1:2", + "activity_attempt_id": "01M7TFM4EF3PTY17A9B0CDG6D8", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 1 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfm4bvvtg0rmp0kr0w9ck2", + "parallel_group_base_sequence": 1 + }, + "workflow_task_id": "01m7tfm4bzh83tp3754qvknbf5", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:13.202607Z" + }, + { + "id": "01m7tfm6hmhg1mnq83jd8gtx8y", + "sequence": 9, + "event_type": "UpdateAccepted", + "namespace": "default", + "payload": { + "command": { + "id": "01m7tfm6hezgd7e7g6yj5aeaz1", + "type": "update", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNw4GCG5hbWUKDHJlcG9ydBJhcmd1bWVudHMMACJ2YWxpZGF0aW9uX2Vycm9ycwwAAA==" + }, + "sequence": 2, + "request_id": "sdk-python-update-c13747e0472d4ae88f5aaef547134edd", + "accepted_at": "2026-11-19T18:03:13.834335Z", + "auth_method": "token", + "auth_status": "authorized", + "target_name": "report", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows/rv307-rv348_plain-969f72/update/report", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfm3yw32tg5eartkhtbmha", + "message_sequence": 1, + "request_route_name": "generated::YF2YckSNfLzTOm1b", + "request_fingerprint": "sha256:d860d168c54a7ca8faeb9d2abaf3fc7dab777e010c0570014d3296f3635cd545" + }, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "update_id": "01m7tfm6hf6e9xny27zpccw2qj", + "request_id": "sdk-python-update-c13747e0472d4ae88f5aaef547134edd", + "update_name": "report", + "ordering_state": "ready", + "workflow_run_id": "01m7tfm3yw32tg5eartkhtbmha", + "workflow_command_id": "01m7tfm6hezgd7e7g6yj5aeaz1", + "workflow_instance_id": "rv307-rv348_plain-969f72", + "queued_behind_command_id": null, + "queued_behind_command_type": null, + "queued_behind_command_sequence": null + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfm6hezgd7e7g6yj5aeaz1", + "recorded_at": "2026-11-19T18:03:13.842821Z" + }, + { + "id": "01m7tfm6w0xbmjkthzpy41m0x4", + "sequence": 10, + "event_type": "UpdateApplied", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfm4nvjjg6arrdnprps3s6", + "type": "workflow", + "queue": "rv307-474a8e", + "status": "leased", + "leased_at": "2026-11-19T18:03:13.949405Z", + "connection": "mysql", + "lease_owner": "rv307-w-d4abff", + "available_at": "2026-11-19T18:03:11.931698Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:23.949365Z", + "last_dispatched_at": "2026-11-19T18:03:11.950855Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:11.950855Z" + }, + "command": { + "id": "01m7tfm6hezgd7e7g6yj5aeaz1", + "type": "update", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNw4GCG5hbWUKDHJlcG9ydBJhcmd1bWVudHMMACJ2YWxpZGF0aW9uX2Vycm9ycwwAAA==" + }, + "sequence": 2, + "request_id": "sdk-python-update-c13747e0472d4ae88f5aaef547134edd", + "accepted_at": "2026-11-19T18:03:13.834335Z", + "auth_method": "token", + "auth_status": "authorized", + "target_name": "report", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows/rv307-rv348_plain-969f72/update/report", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfm3yw32tg5eartkhtbmha", + "message_sequence": 1, + "request_route_name": "generated::YF2YckSNfLzTOm1b", + "request_fingerprint": "sha256:d860d168c54a7ca8faeb9d2abaf3fc7dab777e010c0570014d3296f3635cd545" + }, + "sequence": 3, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "update_id": "01m7tfm6hf6e9xny27zpccw2qj", + "update_name": "report", + "workflow_run_id": "01m7tfm3yw32tg5eartkhtbmha", + "workflow_command_id": "01m7tfm6hezgd7e7g6yj5aeaz1", + "workflow_instance_id": "rv307-rv348_plain-969f72" + }, + "workflow_task_id": "01m7tfm4nvjjg6arrdnprps3s6", + "workflow_command_id": "01m7tfm6hezgd7e7g6yj5aeaz1", + "recorded_at": "2026-11-19T18:03:14.175478Z" + }, + { + "id": "01m7tfm6w7pd5j4fv2t6arxrr4", + "sequence": 11, + "event_type": "UpdateCompleted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfm4nvjjg6arrdnprps3s6", + "type": "workflow", + "queue": "rv307-474a8e", + "status": "leased", + "leased_at": "2026-11-19T18:03:13.949405Z", + "connection": "mysql", + "lease_owner": "rv307-w-d4abff", + "available_at": "2026-11-19T18:03:11.931698Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:23.949365Z", + "last_dispatched_at": "2026-11-19T18:03:11.950855Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:11.950855Z" + }, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNw4CGGNvbXBlbnNhdGluZwoOZmFpbC1tMAA=" + }, + "command": { + "id": "01m7tfm6hezgd7e7g6yj5aeaz1", + "type": "update", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNw4GCG5hbWUKDHJlcG9ydBJhcmd1bWVudHMMACJ2YWxpZGF0aW9uX2Vycm9ycwwAAA==" + }, + "sequence": 2, + "request_id": "sdk-python-update-c13747e0472d4ae88f5aaef547134edd", + "accepted_at": "2026-11-19T18:03:13.834335Z", + "auth_method": "token", + "auth_status": "authorized", + "target_name": "report", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows/rv307-rv348_plain-969f72/update/report", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfm3yw32tg5eartkhtbmha", + "message_sequence": 1, + "request_route_name": "generated::YF2YckSNfLzTOm1b", + "request_fingerprint": "sha256:d860d168c54a7ca8faeb9d2abaf3fc7dab777e010c0570014d3296f3635cd545" + }, + "sequence": 3, + "update_id": "01m7tfm6hf6e9xny27zpccw2qj", + "update_name": "report", + "workflow_run_id": "01m7tfm3yw32tg5eartkhtbmha", + "workflow_command_id": "01m7tfm6hezgd7e7g6yj5aeaz1", + "workflow_instance_id": "rv307-rv348_plain-969f72" + }, + "workflow_task_id": "01m7tfm4nvjjg6arrdnprps3s6", + "workflow_command_id": "01m7tfm6hezgd7e7g6yj5aeaz1", + "recorded_at": "2026-11-19T18:03:14.181921Z" + }, + { + "id": "01m7tfm6wdfzwts79p7ae4vp6j", + "sequence": 12, + "event_type": "MessageCursorAdvanced", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfm4nvjjg6arrdnprps3s6", + "type": "workflow", + "queue": "rv307-474a8e", + "status": "leased", + "leased_at": "2026-11-19T18:03:13.949405Z", + "connection": "mysql", + "lease_owner": "rv307-w-d4abff", + "available_at": "2026-11-19T18:03:11.931698Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:23.949365Z", + "last_dispatched_at": "2026-11-19T18:03:11.950855Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:11.950855Z" + }, + "stream_key": "instance:rv307-rv348_plain-969f72", + "new_position": 1, + "previous_position": 0 + }, + "workflow_task_id": "01m7tfm4nvjjg6arrdnprps3s6", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:14.188992Z" + } + ], + "expected": { + "query_result": "fail-m0" + } +} diff --git a/tests/fixtures/replay_regressions/failure-order-released-update-before-new-worker.json b/tests/fixtures/replay_regressions/failure-order-released-update-before-new-worker.json new file mode 100644 index 00000000..8eaf096f --- /dev/null +++ b/tests/fixtures/replay_regressions/failure-order-released-update-before-new-worker.json @@ -0,0 +1,886 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "failure-order-released-update-before-new-worker", + "protocol_version": "1.20", + "bindings": [ + "python" + ], + "parallel_failure_order": { + "issues": [ + "durable-workflow/sdk-python#307" + ], + "note": "Recorded on a Server built from durable-workflow/server main 52e3ec00 (docker-compose.test.yml). Published 2.19.1 scheduled the group without the marker and, after m1 then m0 failed, answered an update with {compensating: fail-m0}. This is the history the next workflow task received; a worker of this version ran it and compensates fail-m0, matching the recorded update." + }, + "workflow": { + "type": "rv348_plain", + "input": [], + "payload_codec": "avro", + "run_id": "01m7tfm3yw32tg5eartkhtbmha", + "workflow_id": "rv307-rv348_plain-969f72" + }, + "history": [ + { + "id": "01m7tfm3z1hjhej16rx9zzwhcs", + "sequence": 1, + "event_type": "StartAccepted", + "namespace": "default", + "payload": { + "memo": null, + "command": { + "id": "01m7tfm3yym2gnn3pt5hsx2vv9", + "type": "start", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "outcome": "started_new", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "sequence": 1, + "applied_at": "2026-11-19T18:03:11.197425Z", + "accepted_at": "2026-11-19T18:03:11.197423Z", + "auth_method": "token", + "auth_status": "authorized", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfm3yw32tg5eartkhtbmha", + "request_route_name": "generated::FqJuJMl97iiXlIDz", + "request_fingerprint": "sha256:e3f59c6f64bc66ea91e92991ab872ea6eca4e831ad1fbabb7ebf36b7b136b9be" + }, + "outcome": "started_new", + "business_key": null, + "workflow_type": "rv348_plain", + "workflow_class": "rv348_plain", + "workflow_run_id": "01m7tfm3yw32tg5eartkhtbmha", + "search_attributes": null, + "visibility_labels": null, + "workflow_command_id": "01m7tfm3yym2gnn3pt5hsx2vv9", + "workflow_instance_id": "rv307-rv348_plain-969f72" + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfm3yym2gnn3pt5hsx2vv9", + "recorded_at": "2026-11-19T18:03:11.200743Z" + }, + { + "id": "01m7tfm3z4cx71efh954xgbwz2", + "sequence": 2, + "event_type": "WorkflowStarted", + "namespace": "default", + "payload": { + "command": { + "id": "01m7tfm3yym2gnn3pt5hsx2vv9", + "type": "start", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "outcome": "started_new", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "sequence": 1, + "applied_at": "2026-11-19T18:03:11.197425Z", + "accepted_at": "2026-11-19T18:03:11.197423Z", + "auth_method": "token", + "auth_status": "authorized", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfm3yw32tg5eartkhtbmha", + "request_route_name": "generated::FqJuJMl97iiXlIDz", + "request_fingerprint": "sha256:e3f59c6f64bc66ea91e92991ab872ea6eca4e831ad1fbabb7ebf36b7b136b9be" + }, + "workflow_type": "rv348_plain", + "workflow_class": "rv348_plain", + "run_deadline_at": "2026-11-19T18:13:11+00:00", + "workflow_run_id": "01m7tfm3yw32tg5eartkhtbmha", + "declared_queries": [ + "caught" + ], + "declared_signals": [], + "declared_updates": [ + "report" + ], + "declared_entry_mode": "canonical", + "run_timeout_seconds": 600, + "workflow_command_id": "01m7tfm3yym2gnn3pt5hsx2vv9", + "workflow_instance_id": "rv307-rv348_plain-969f72", + "declared_entry_method": "handle", + "execution_deadline_at": "2026-11-19T19:03:11+00:00", + "declared_query_contracts": [ + { + "name": "caught", + "parameters": [] + } + ], + "declared_signal_contracts": [], + "declared_update_contracts": [ + { + "name": "report", + "parameters": [ + { + "name": "_", + "type": null, + "default": null, + "position": 0, + "required": false, + "variadic": true, + "allows_null": true, + "default_available": false + } + ] + } + ], + "execution_timeout_seconds": 3600, + "declared_update_validators": [], + "declared_entry_declaring_class": "external:rv348_plain", + "workflow_definition_fingerprint": "sha256:4cca49f53721ff6d13fd7cd42526e63e2497bbd302876fde0ff10eb9748ac44b", + "workflow_definition_fingerprint_source": "worker" + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfm3yym2gnn3pt5hsx2vv9", + "recorded_at": "2026-11-19T18:03:11.203745Z" + }, + { + "id": "01m7tfm4bwk71w8axj3jqetxgp", + "sequence": 3, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfm3z50n0je42yt0fevysb", + "type": "workflow", + "queue": "rv307-474a8e", + "status": "leased", + "leased_at": "2026-11-19T18:03:11.251003Z", + "connection": "mysql", + "lease_owner": "rv307-w-d4abff", + "available_at": "2026-11-19T18:03:11.205523Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:21.250959Z", + "last_dispatched_at": "2026-11-19T18:03:11.207285Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:11.207285Z" + }, + "activity": { + "id": "01m7tfm4bvvtg0rmp0kr0w9ck2", + "type": "m0", + "class": "m0", + "queue": "rv307-474a8e", + "status": "pending", + "sequence": 1, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "created_at": "2026-11-19T18:03:11.611513Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfm4bvvtg0rmp0kr0w9ck2" + }, + "sequence": 1, + "activity_type": "m0", + "activity_class": "m0", + "parallel_group_id": "parallel-activities:1:2", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 1 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfm4bvvtg0rmp0kr0w9ck2", + "parallel_group_base_sequence": 1 + }, + "workflow_task_id": "01m7tfm3z50n0je42yt0fevysb", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:11.612487Z" + }, + { + "id": "01m7tfm4c2r7gabnqy2890rf31", + "sequence": 4, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfm3z50n0je42yt0fevysb", + "type": "workflow", + "queue": "rv307-474a8e", + "status": "leased", + "leased_at": "2026-11-19T18:03:11.251003Z", + "connection": "mysql", + "lease_owner": "rv307-w-d4abff", + "available_at": "2026-11-19T18:03:11.205523Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:21.250959Z", + "last_dispatched_at": "2026-11-19T18:03:11.207285Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:11.207285Z" + }, + "activity": { + "id": "01m7tfm4c0q1xx5tyc93p1w731", + "type": "m1", + "class": "m1", + "queue": "rv307-474a8e", + "status": "pending", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "created_at": "2026-11-19T18:03:11.616913Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfm4c0q1xx5tyc93p1w731" + }, + "sequence": 2, + "activity_type": "m1", + "activity_class": "m1", + "parallel_group_id": "parallel-activities:1:2", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 1 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfm4c0q1xx5tyc93p1w731", + "parallel_group_base_sequence": 1 + }, + "workflow_task_id": "01m7tfm3z50n0je42yt0fevysb", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:11.617688Z" + }, + { + "id": "01m7tfm4epm84kmz9gwgcv2pg8", + "sequence": 5, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfm4bzh83tp3754qvknbf5", + "type": "activity", + "queue": "rv307-474a8e", + "status": "leased", + "leased_at": "2026-11-19T18:03:11.695781Z", + "connection": "mysql", + "lease_owner": "rv307-w-d4abff", + "available_at": "2026-11-19T18:03:11.615206Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:11.695954Z" + }, + "activity": { + "id": "01m7tfm4bvvtg0rmp0kr0w9ck2", + "type": "m0", + "class": "m0", + "queue": "rv307-474a8e", + "status": "running", + "sequence": 1, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "attempt_id": "01M7TFM4EF3PTY17A9B0CDG6D8", + "created_at": "2026-11-19T18:03:11.611513Z", + "started_at": "2026-11-19T18:03:11.695781Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfm4bvvtg0rmp0kr0w9ck2" + }, + "sequence": 1, + "activity_type": "m0", + "activity_class": "m0", + "attempt_number": 1, + "parallel_group_id": "parallel-activities:1:2", + "activity_attempt_id": "01M7TFM4EF3PTY17A9B0CDG6D8", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 1 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfm4bvvtg0rmp0kr0w9ck2", + "parallel_group_base_sequence": 1 + }, + "workflow_task_id": "01m7tfm4bzh83tp3754qvknbf5", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:11.702308Z" + }, + { + "id": "01m7tfm4jrg64z2aw1xc6tpanc", + "sequence": 6, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfm4c33way08w985jma6vv", + "type": "activity", + "queue": "rv307-474a8e", + "status": "leased", + "leased_at": "2026-11-19T18:03:11.827788Z", + "connection": "mysql", + "lease_owner": "rv307-w-d4abff", + "available_at": "2026-11-19T18:03:11.619207Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:11.827870Z" + }, + "activity": { + "id": "01m7tfm4c0q1xx5tyc93p1w731", + "type": "m1", + "class": "m1", + "queue": "rv307-474a8e", + "status": "running", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "attempt_id": "01M7TFM4JKR3SEEJR8Q5HE1KMD", + "created_at": "2026-11-19T18:03:11.616913Z", + "started_at": "2026-11-19T18:03:11.827788Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfm4c0q1xx5tyc93p1w731" + }, + "sequence": 2, + "activity_type": "m1", + "activity_class": "m1", + "attempt_number": 1, + "parallel_group_id": "parallel-activities:1:2", + "activity_attempt_id": "01M7TFM4JKR3SEEJR8Q5HE1KMD", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 1 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfm4c0q1xx5tyc93p1w731", + "parallel_group_base_sequence": 1 + }, + "workflow_task_id": "01m7tfm4c33way08w985jma6vv", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:11.831851Z" + }, + { + "id": "01m7tfm4nqfbh62tycsn3rw88e", + "sequence": 7, + "event_type": "ActivityFailed", + "namespace": "default", + "payload": { + "code": 0, + "task": { + "id": "01m7tfm4c33way08w985jma6vv", + "type": "activity", + "queue": "rv307-474a8e", + "status": "leased", + "leased_at": "2026-11-19T18:03:11.827788Z", + "connection": "mysql", + "lease_owner": "rv307-w-d4abff", + "available_at": "2026-11-19T18:03:11.619207Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:11.891793Z" + }, + "message": "fail-m1", + "activity": { + "id": "01m7tfm4c0q1xx5tyc93p1w731", + "type": "m1", + "class": "m1", + "queue": "rv307-474a8e", + "status": "failed", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "closed_at": "2026-11-19T18:03:11.925626Z", + "exception": { + "code": 0, + "type": null, + "message": "fail-m1", + "non_retryable": true + }, + "attempt_id": "01M7TFM4JKR3SEEJR8Q5HE1KMD", + "created_at": "2026-11-19T18:03:11.616913Z", + "started_at": "2026-11-19T18:03:11.827788Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfm4c0q1xx5tyc93p1w731" + }, + "sequence": 2, + "exception": { + "type": null, + "message": "fail-m1", + "code": 0, + "non_retryable": true + }, + "failure_id": "01m7tfm4nntbjxs3bqtfzz53xg", + "activity_type": "m1", + "non_retryable": true, + "activity_class": "m1", + "attempt_number": 1, + "exception_type": null, + "exception_class": "RuntimeException", + "failure_category": "activity", + "parallel_group_id": "parallel-activities:1:2", + "activity_attempt_id": "01M7TFM4JKR3SEEJR8Q5HE1KMD", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 1 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfm4c0q1xx5tyc93p1w731", + "parallel_group_base_sequence": 1 + }, + "workflow_task_id": "01m7tfm4c33way08w985jma6vv", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:11.927075Z" + }, + { + "id": "01m7tfm5xkdvsmfdt93gbq9dnp", + "sequence": 8, + "event_type": "ActivityFailed", + "namespace": "default", + "payload": { + "code": 0, + "task": { + "id": "01m7tfm4bzh83tp3754qvknbf5", + "type": "activity", + "queue": "rv307-474a8e", + "status": "leased", + "leased_at": "2026-11-19T18:03:11.695781Z", + "connection": "mysql", + "lease_owner": "rv307-w-d4abff", + "available_at": "2026-11-19T18:03:11.615206Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:08:11.771243Z" + }, + "message": "fail-m0", + "activity": { + "id": "01m7tfm4bvvtg0rmp0kr0w9ck2", + "type": "m0", + "class": "m0", + "queue": "rv307-474a8e", + "status": "failed", + "sequence": 1, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "closed_at": "2026-11-19T18:03:13.201365Z", + "exception": { + "code": 0, + "type": null, + "message": "fail-m0", + "non_retryable": true + }, + "attempt_id": "01M7TFM4EF3PTY17A9B0CDG6D8", + "created_at": "2026-11-19T18:03:11.611513Z", + "started_at": "2026-11-19T18:03:11.695781Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfm4bvvtg0rmp0kr0w9ck2" + }, + "sequence": 1, + "exception": { + "type": null, + "message": "fail-m0", + "code": 0, + "non_retryable": true + }, + "failure_id": "01m7tfm5xgn5387eyfyw63pwe5", + "activity_type": "m0", + "non_retryable": true, + "activity_class": "m0", + "attempt_number": 1, + "exception_type": null, + "exception_class": "RuntimeException", + "failure_category": "activity", + "parallel_group_id": "parallel-activities:1:2", + "activity_attempt_id": "01M7TFM4EF3PTY17A9B0CDG6D8", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:1:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 1 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfm4bvvtg0rmp0kr0w9ck2", + "parallel_group_base_sequence": 1 + }, + "workflow_task_id": "01m7tfm4bzh83tp3754qvknbf5", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:13.202607Z" + }, + { + "id": "01m7tfm6hmhg1mnq83jd8gtx8y", + "sequence": 9, + "event_type": "UpdateAccepted", + "namespace": "default", + "payload": { + "command": { + "id": "01m7tfm6hezgd7e7g6yj5aeaz1", + "type": "update", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNw4GCG5hbWUKDHJlcG9ydBJhcmd1bWVudHMMACJ2YWxpZGF0aW9uX2Vycm9ycwwAAA==" + }, + "sequence": 2, + "request_id": "sdk-python-update-c13747e0472d4ae88f5aaef547134edd", + "accepted_at": "2026-11-19T18:03:13.834335Z", + "auth_method": "token", + "auth_status": "authorized", + "target_name": "report", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows/rv307-rv348_plain-969f72/update/report", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfm3yw32tg5eartkhtbmha", + "message_sequence": 1, + "request_route_name": "generated::YF2YckSNfLzTOm1b", + "request_fingerprint": "sha256:d860d168c54a7ca8faeb9d2abaf3fc7dab777e010c0570014d3296f3635cd545" + }, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "update_id": "01m7tfm6hf6e9xny27zpccw2qj", + "request_id": "sdk-python-update-c13747e0472d4ae88f5aaef547134edd", + "update_name": "report", + "ordering_state": "ready", + "workflow_run_id": "01m7tfm3yw32tg5eartkhtbmha", + "workflow_command_id": "01m7tfm6hezgd7e7g6yj5aeaz1", + "workflow_instance_id": "rv307-rv348_plain-969f72", + "queued_behind_command_id": null, + "queued_behind_command_type": null, + "queued_behind_command_sequence": null + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfm6hezgd7e7g6yj5aeaz1", + "recorded_at": "2026-11-19T18:03:13.842821Z" + }, + { + "id": "01m7tfm6w0xbmjkthzpy41m0x4", + "sequence": 10, + "event_type": "UpdateApplied", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfm4nvjjg6arrdnprps3s6", + "type": "workflow", + "queue": "rv307-474a8e", + "status": "leased", + "leased_at": "2026-11-19T18:03:13.949405Z", + "connection": "mysql", + "lease_owner": "rv307-w-d4abff", + "available_at": "2026-11-19T18:03:11.931698Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:23.949365Z", + "last_dispatched_at": "2026-11-19T18:03:11.950855Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:11.950855Z" + }, + "command": { + "id": "01m7tfm6hezgd7e7g6yj5aeaz1", + "type": "update", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNw4GCG5hbWUKDHJlcG9ydBJhcmd1bWVudHMMACJ2YWxpZGF0aW9uX2Vycm9ycwwAAA==" + }, + "sequence": 2, + "request_id": "sdk-python-update-c13747e0472d4ae88f5aaef547134edd", + "accepted_at": "2026-11-19T18:03:13.834335Z", + "auth_method": "token", + "auth_status": "authorized", + "target_name": "report", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows/rv307-rv348_plain-969f72/update/report", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfm3yw32tg5eartkhtbmha", + "message_sequence": 1, + "request_route_name": "generated::YF2YckSNfLzTOm1b", + "request_fingerprint": "sha256:d860d168c54a7ca8faeb9d2abaf3fc7dab777e010c0570014d3296f3635cd545" + }, + "sequence": 3, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "update_id": "01m7tfm6hf6e9xny27zpccw2qj", + "update_name": "report", + "workflow_run_id": "01m7tfm3yw32tg5eartkhtbmha", + "workflow_command_id": "01m7tfm6hezgd7e7g6yj5aeaz1", + "workflow_instance_id": "rv307-rv348_plain-969f72" + }, + "workflow_task_id": "01m7tfm4nvjjg6arrdnprps3s6", + "workflow_command_id": "01m7tfm6hezgd7e7g6yj5aeaz1", + "recorded_at": "2026-11-19T18:03:14.175478Z" + }, + { + "id": "01m7tfm6w7pd5j4fv2t6arxrr4", + "sequence": 11, + "event_type": "UpdateCompleted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfm4nvjjg6arrdnprps3s6", + "type": "workflow", + "queue": "rv307-474a8e", + "status": "leased", + "leased_at": "2026-11-19T18:03:13.949405Z", + "connection": "mysql", + "lease_owner": "rv307-w-d4abff", + "available_at": "2026-11-19T18:03:11.931698Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:23.949365Z", + "last_dispatched_at": "2026-11-19T18:03:11.950855Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:11.950855Z" + }, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNw4CGGNvbXBlbnNhdGluZwoOZmFpbC1tMAA=" + }, + "command": { + "id": "01m7tfm6hezgd7e7g6yj5aeaz1", + "type": "update", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNw4GCG5hbWUKDHJlcG9ydBJhcmd1bWVudHMMACJ2YWxpZGF0aW9uX2Vycm9ycwwAAA==" + }, + "sequence": 2, + "request_id": "sdk-python-update-c13747e0472d4ae88f5aaef547134edd", + "accepted_at": "2026-11-19T18:03:13.834335Z", + "auth_method": "token", + "auth_status": "authorized", + "target_name": "report", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows/rv307-rv348_plain-969f72/update/report", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfm3yw32tg5eartkhtbmha", + "message_sequence": 1, + "request_route_name": "generated::YF2YckSNfLzTOm1b", + "request_fingerprint": "sha256:d860d168c54a7ca8faeb9d2abaf3fc7dab777e010c0570014d3296f3635cd545" + }, + "sequence": 3, + "update_id": "01m7tfm6hf6e9xny27zpccw2qj", + "update_name": "report", + "workflow_run_id": "01m7tfm3yw32tg5eartkhtbmha", + "workflow_command_id": "01m7tfm6hezgd7e7g6yj5aeaz1", + "workflow_instance_id": "rv307-rv348_plain-969f72" + }, + "workflow_task_id": "01m7tfm4nvjjg6arrdnprps3s6", + "workflow_command_id": "01m7tfm6hezgd7e7g6yj5aeaz1", + "recorded_at": "2026-11-19T18:03:14.181921Z" + }, + { + "id": "01m7tfm6wdfzwts79p7ae4vp6j", + "sequence": 12, + "event_type": "MessageCursorAdvanced", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfm4nvjjg6arrdnprps3s6", + "type": "workflow", + "queue": "rv307-474a8e", + "status": "leased", + "leased_at": "2026-11-19T18:03:13.949405Z", + "connection": "mysql", + "lease_owner": "rv307-w-d4abff", + "available_at": "2026-11-19T18:03:11.931698Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:03:23.949365Z", + "last_dispatched_at": "2026-11-19T18:03:11.950855Z", + "last_dispatch_attempt_at": "2026-11-19T18:03:11.950855Z" + }, + "stream_key": "instance:rv307-rv348_plain-969f72", + "new_position": 1, + "previous_position": 0 + }, + "workflow_task_id": "01m7tfm4nvjjg6arrdnprps3s6", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:03:14.188992Z" + } + ], + "expected": { + "command_sequence": [ + { + "type": "schedule_activity", + "activity_type": "compensate", + "arguments": [ + "fail-m0" + ] + } + ] + } +} diff --git a/tests/fixtures/replay_regressions/failure-order-released-update-between-tasks-query.json b/tests/fixtures/replay_regressions/failure-order-released-update-between-tasks-query.json new file mode 100644 index 00000000..402396ca --- /dev/null +++ b/tests/fixtures/replay_regressions/failure-order-released-update-between-tasks-query.json @@ -0,0 +1,1523 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "failure-order-released-update-between-tasks-query", + "protocol_version": "1.20", + "bindings": [ + "python" + ], + "parallel_failure_order": { + "issues": [ + "durable-workflow/sdk-python#307" + ], + "note": "Recorded on a Server built from durable-workflow/server main 52e3ec00 (docker-compose.test.yml). Published 2.19.1 ran the zero-command catch of fail-m0 and answered an update with {compensating: fail-m0}; a worker of this version ran the remaining tasks and compensated fail-m0. This is the history of its last task." + }, + "workflow": { + "type": "rv348_handle", + "input": [], + "payload_codec": "avro", + "run_id": "01m7tfjvyq564n3r52xph03nz8", + "workflow_id": "rv307-rv348_handle-6e9a63", + "query": { + "name": "caught", + "arguments": [] + } + }, + "history": [ + { + "id": "01m7tfjvyxhjfb1jnt7f0d1c66", + "sequence": 1, + "event_type": "StartAccepted", + "namespace": "default", + "payload": { + "memo": null, + "command": { + "id": "01m7tfjvyt8zwh2ccm9n5zbdhd", + "type": "start", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "outcome": "started_new", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "sequence": 1, + "applied_at": "2026-11-19T18:02:30.232740Z", + "accepted_at": "2026-11-19T18:02:30.232736Z", + "auth_method": "token", + "auth_status": "authorized", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfjvyq564n3r52xph03nz8", + "request_route_name": "generated::FqJuJMl97iiXlIDz", + "request_fingerprint": "sha256:e5e68285751d4aa76a22d2ea737bb10e6ee57ea6e5e339b1a62d530d40a99ba1" + }, + "outcome": "started_new", + "business_key": null, + "workflow_type": "rv348_handle", + "workflow_class": "rv348_handle", + "workflow_run_id": "01m7tfjvyq564n3r52xph03nz8", + "search_attributes": null, + "visibility_labels": null, + "workflow_command_id": "01m7tfjvyt8zwh2ccm9n5zbdhd", + "workflow_instance_id": "rv307-rv348_handle-6e9a63" + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfjvyt8zwh2ccm9n5zbdhd", + "recorded_at": "2026-11-19T18:02:30.236558Z" + }, + { + "id": "01m7tfjvz0jbm2e2xh8ma9k6ev", + "sequence": 2, + "event_type": "WorkflowStarted", + "namespace": "default", + "payload": { + "command": { + "id": "01m7tfjvyt8zwh2ccm9n5zbdhd", + "type": "start", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "outcome": "started_new", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "sequence": 1, + "applied_at": "2026-11-19T18:02:30.232740Z", + "accepted_at": "2026-11-19T18:02:30.232736Z", + "auth_method": "token", + "auth_status": "authorized", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfjvyq564n3r52xph03nz8", + "request_route_name": "generated::FqJuJMl97iiXlIDz", + "request_fingerprint": "sha256:e5e68285751d4aa76a22d2ea737bb10e6ee57ea6e5e339b1a62d530d40a99ba1" + }, + "workflow_type": "rv348_handle", + "workflow_class": "rv348_handle", + "run_deadline_at": "2026-11-19T18:12:30+00:00", + "workflow_run_id": "01m7tfjvyq564n3r52xph03nz8", + "declared_queries": [ + "caught" + ], + "declared_signals": [], + "declared_updates": [ + "report" + ], + "declared_entry_mode": "canonical", + "run_timeout_seconds": 600, + "workflow_command_id": "01m7tfjvyt8zwh2ccm9n5zbdhd", + "workflow_instance_id": "rv307-rv348_handle-6e9a63", + "declared_entry_method": "handle", + "execution_deadline_at": "2026-11-19T19:02:30+00:00", + "declared_query_contracts": [ + { + "name": "caught", + "parameters": [] + } + ], + "declared_signal_contracts": [], + "declared_update_contracts": [ + { + "name": "report", + "parameters": [ + { + "name": "_", + "type": null, + "default": null, + "position": 0, + "required": false, + "variadic": true, + "allows_null": true, + "default_available": false + } + ] + } + ], + "execution_timeout_seconds": 3600, + "declared_update_validators": [], + "declared_entry_declaring_class": "external:rv348_handle", + "workflow_definition_fingerprint": "sha256:69ed762d44804878ebb703a2ad385119f6f776d0dae561c15f8f84e1707a704a", + "workflow_definition_fingerprint_source": "worker" + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfjvyt8zwh2ccm9n5zbdhd", + "recorded_at": "2026-11-19T18:02:30.239683Z" + }, + { + "id": "01m7tfjwf3kqkct8sgg13v901w", + "sequence": 3, + "event_type": "TimerScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfjvz1964m1nbs2m582070", + "type": "workflow", + "queue": "rv307-4704a8", + "status": "leased", + "leased_at": "2026-11-19T18:02:30.306973Z", + "connection": "mysql", + "lease_owner": "rv307-w-32445d", + "available_at": "2026-11-19T18:02:30.241603Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:40.306911Z", + "last_dispatched_at": "2026-11-19T18:02:30.243567Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:30.243567Z" + }, + "fire_at": "2026-11-19T18:02:31.753185Z", + "sequence": 1, + "timer_id": "01m7tfjwf131zf70c1pthg1gtz", + "delay_seconds": 1, + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + }, + "workflow_task_id": "01m7tfjvz1964m1nbs2m582070", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:30.754427Z" + }, + { + "id": "01m7tfjwf9mw3nf0m06jag0rpp", + "sequence": 4, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfjvz1964m1nbs2m582070", + "type": "workflow", + "queue": "rv307-4704a8", + "status": "leased", + "leased_at": "2026-11-19T18:02:30.306973Z", + "connection": "mysql", + "lease_owner": "rv307-w-32445d", + "available_at": "2026-11-19T18:02:30.241603Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:40.306911Z", + "last_dispatched_at": "2026-11-19T18:02:30.243567Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:30.243567Z" + }, + "activity": { + "id": "01m7tfjwf7yh5ffwcwgdyt5cck", + "type": "slow", + "class": "slow", + "queue": "rv307-4704a8", + "status": "pending", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "created_at": "2026-11-19T18:02:30.759721Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfjwf7yh5ffwcwgdyt5cck" + }, + "sequence": 2, + "activity_type": "slow", + "activity_class": "slow", + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "activity_execution_id": "01m7tfjwf7yh5ffwcwgdyt5cck", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + }, + "workflow_task_id": "01m7tfjvz1964m1nbs2m582070", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:30.760884Z" + }, + { + "id": "01m7tfjy636brag78mp7f2etay", + "sequence": 5, + "event_type": "TimerFired", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfjwf6tfreseph1kvnbawh", + "type": "timer", + "queue": "rv307-4704a8", + "status": "leased", + "leased_at": "2026-11-19T18:02:32.497518Z", + "connection": "mysql", + "lease_owner": "01m7tfjwf6tfreseph1kvnbawh", + "available_at": "2026-11-19T18:02:31.753185Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:42.497522Z" + }, + "fire_at": "2026-11-19T18:02:31.753185Z", + "fired_at": "2026-11-19T18:02:32.513880Z", + "sequence": 1, + "timer_id": "01m7tfjwf131zf70c1pthg1gtz", + "delay_seconds": 1, + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + }, + "workflow_task_id": "01m7tfjwf6tfreseph1kvnbawh", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:32.514927Z" + }, + { + "id": "01m7tfjy66g1ev3wsexrdec2mh", + "sequence": 6, + "event_type": "SelectionResolved", + "namespace": "default", + "payload": { + "outcome": "completed", + "member_key": "t", + "member_size": 1, + "member_index": 0, + "operation_kind": "timer", + "operation_identity": "01m7tfjwf131zf70c1pthg1gtz", + "selection_group_id": "select-calls:1:2", + "resolution_event_id": "01m7tfjy636brag78mp7f2etay", + "member_base_sequence": 1, + "selection_group_size": 2, + "resolution_event_type": "TimerFired", + "selection_group_base_sequence": 1 + }, + "workflow_task_id": null, + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:32.518142Z" + }, + { + "id": "01m7tfjzf9vnczgmw2m5ryv8xk", + "sequence": 7, + "event_type": "TimerScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfjy680jgfhnpdzb5aj7p6", + "type": "workflow", + "queue": "rv307-4704a8", + "status": "leased", + "leased_at": "2026-11-19T18:02:33.402266Z", + "connection": "mysql", + "lease_owner": "rv307-w-32445d", + "available_at": "2026-11-19T18:02:32.520491Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:43.402145Z", + "last_dispatched_at": "2026-11-19T18:02:32.538006Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:32.538006Z" + }, + "fire_at": "2026-11-19T18:02:34.830949Z", + "sequence": 3, + "timer_id": "01m7tfjzf7q72xkgzbp7w34va7", + "delay_seconds": 1 + }, + "workflow_task_id": "01m7tfjy680jgfhnpdzb5aj7p6", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:33.832578Z" + }, + { + "id": "01m7tfk17chkh6pa62r3g749we", + "sequence": 8, + "event_type": "TimerFired", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfjzfctvmyh9rz4vq2512w", + "type": "timer", + "queue": "rv307-4704a8", + "status": "leased", + "leased_at": "2026-11-19T18:02:35.602330Z", + "connection": "mysql", + "lease_owner": "01m7tfjzfctvmyh9rz4vq2512w", + "available_at": "2026-11-19T18:02:34.830949Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:45.602335Z" + }, + "fire_at": "2026-11-19T18:02:34.830949Z", + "fired_at": "2026-11-19T18:02:35.626529Z", + "sequence": 3, + "timer_id": "01m7tfjzf7q72xkgzbp7w34va7", + "delay_seconds": 1 + }, + "workflow_task_id": "01m7tfjzfctvmyh9rz4vq2512w", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:35.627926Z" + }, + { + "id": "01m7tfk2dtqaqa7ph7s6ka0kr0", + "sequence": 9, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfk17f668ma8m55583xw3h", + "type": "workflow", + "queue": "rv307-4704a8", + "status": "leased", + "leased_at": "2026-11-19T18:02:36.417139Z", + "connection": "mysql", + "lease_owner": "rv307-w-32445d", + "available_at": "2026-11-19T18:02:35.630901Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:46.417107Z", + "last_dispatched_at": "2026-11-19T18:02:35.654654Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:35.654654Z" + }, + "activity": { + "id": "01m7tfk2dpe7zwym36as2r13pc", + "type": "m0", + "class": "m0", + "queue": "rv307-4704a8", + "status": "pending", + "sequence": 4, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "created_at": "2026-11-19T18:02:36.854843Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfk2dpe7zwym36as2r13pc" + }, + "sequence": 4, + "activity_type": "m0", + "activity_class": "m0", + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfk2dpe7zwym36as2r13pc", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfk17f668ma8m55583xw3h", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:36.857406Z" + }, + { + "id": "01m7tfk2e2mqvmgge6y3mjbrwc", + "sequence": 10, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfk17f668ma8m55583xw3h", + "type": "workflow", + "queue": "rv307-4704a8", + "status": "leased", + "leased_at": "2026-11-19T18:02:36.417139Z", + "connection": "mysql", + "lease_owner": "rv307-w-32445d", + "available_at": "2026-11-19T18:02:35.630901Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:46.417107Z", + "last_dispatched_at": "2026-11-19T18:02:35.654654Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:35.654654Z" + }, + "activity": { + "id": "01m7tfk2e1rd6x1h4rks9q8h1m", + "type": "m1", + "class": "m1", + "queue": "rv307-4704a8", + "status": "pending", + "sequence": 5, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "created_at": "2026-11-19T18:02:36.865400Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfk2e1rd6x1h4rks9q8h1m" + }, + "sequence": 5, + "activity_type": "m1", + "activity_class": "m1", + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfk2e1rd6x1h4rks9q8h1m", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfk17f668ma8m55583xw3h", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:36.866397Z" + }, + { + "id": "01m7tfk2g72rnm9xta2wvhevvd", + "sequence": 11, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfjwfa9v2ws6xfbyt09jjz", + "type": "activity", + "queue": "rv307-4704a8", + "status": "leased", + "leased_at": "2026-11-19T18:02:36.930722Z", + "connection": "mysql", + "lease_owner": "rv307-w-32445d", + "available_at": "2026-11-19T18:02:30.762733Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:36.930804Z" + }, + "activity": { + "id": "01m7tfjwf7yh5ffwcwgdyt5cck", + "type": "slow", + "class": "slow", + "queue": "rv307-4704a8", + "status": "running", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "attempt_id": "01M7TFK2G2RTR8F0FA5JWDT4DR", + "created_at": "2026-11-19T18:02:30.759721Z", + "started_at": "2026-11-19T18:02:36.930722Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfjwf7yh5ffwcwgdyt5cck" + }, + "sequence": 2, + "activity_type": "slow", + "activity_class": "slow", + "attempt_number": 1, + "parallel_group_id": "select-calls:1:2", + "activity_attempt_id": "01M7TFK2G2RTR8F0FA5JWDT4DR", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "activity_execution_id": "01m7tfjwf7yh5ffwcwgdyt5cck", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + }, + "workflow_task_id": "01m7tfjwfa9v2ws6xfbyt09jjz", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:36.934760Z" + }, + { + "id": "01m7tfk2k0rxrn8v326f3ja5j7", + "sequence": 12, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfk2dybkwr9dc5arwgd3d6", + "type": "activity", + "queue": "rv307-4704a8", + "status": "leased", + "leased_at": "2026-11-19T18:02:37.020000Z", + "connection": "mysql", + "lease_owner": "rv307-w-32445d", + "available_at": "2026-11-19T18:02:36.862214Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:37.020089Z" + }, + "activity": { + "id": "01m7tfk2dpe7zwym36as2r13pc", + "type": "m0", + "class": "m0", + "queue": "rv307-4704a8", + "status": "running", + "sequence": 4, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "attempt_id": "01M7TFK2JW051RRZ877NTRK82W", + "created_at": "2026-11-19T18:02:36.854843Z", + "started_at": "2026-11-19T18:02:37.020000Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfk2dpe7zwym36as2r13pc" + }, + "sequence": 4, + "activity_type": "m0", + "activity_class": "m0", + "attempt_number": 1, + "parallel_group_id": "parallel-activities:4:2", + "activity_attempt_id": "01M7TFK2JW051RRZ877NTRK82W", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfk2dpe7zwym36as2r13pc", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfk2dybkwr9dc5arwgd3d6", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:37.024253Z" + }, + { + "id": "01m7tfk2nszsxkbxmyd1r8w0ap", + "sequence": 13, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfk2e41fj1b5599zer5akp", + "type": "activity", + "queue": "rv307-4704a8", + "status": "leased", + "leased_at": "2026-11-19T18:02:37.108870Z", + "connection": "mysql", + "lease_owner": "rv307-w-32445d", + "available_at": "2026-11-19T18:02:36.868284Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:37.108961Z" + }, + "activity": { + "id": "01m7tfk2e1rd6x1h4rks9q8h1m", + "type": "m1", + "class": "m1", + "queue": "rv307-4704a8", + "status": "running", + "sequence": 5, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "attempt_id": "01M7TFK2NMPZ2GTRSERF2DVTVS", + "created_at": "2026-11-19T18:02:36.865400Z", + "started_at": "2026-11-19T18:02:37.108870Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfk2e1rd6x1h4rks9q8h1m" + }, + "sequence": 5, + "activity_type": "m1", + "activity_class": "m1", + "attempt_number": 1, + "parallel_group_id": "parallel-activities:4:2", + "activity_attempt_id": "01M7TFK2NMPZ2GTRSERF2DVTVS", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfk2e1rd6x1h4rks9q8h1m", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfk2e41fj1b5599zer5akp", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:37.113029Z" + }, + { + "id": "01m7tfk2reth2f7dsebq0yyx68", + "sequence": 14, + "event_type": "ActivityFailed", + "namespace": "default", + "payload": { + "code": 0, + "task": { + "id": "01m7tfk2e41fj1b5599zer5akp", + "type": "activity", + "queue": "rv307-4704a8", + "status": "leased", + "leased_at": "2026-11-19T18:02:37.108870Z", + "connection": "mysql", + "lease_owner": "rv307-w-32445d", + "available_at": "2026-11-19T18:02:36.868284Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:37.168218Z" + }, + "message": "fail-m1", + "activity": { + "id": "01m7tfk2e1rd6x1h4rks9q8h1m", + "type": "m1", + "class": "m1", + "queue": "rv307-4704a8", + "status": "failed", + "sequence": 5, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "closed_at": "2026-11-19T18:02:37.196150Z", + "exception": { + "code": 0, + "type": null, + "message": "fail-m1", + "non_retryable": true + }, + "attempt_id": "01M7TFK2NMPZ2GTRSERF2DVTVS", + "created_at": "2026-11-19T18:02:36.865400Z", + "started_at": "2026-11-19T18:02:37.108870Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfk2e1rd6x1h4rks9q8h1m" + }, + "sequence": 5, + "exception": { + "type": null, + "message": "fail-m1", + "code": 0, + "non_retryable": true + }, + "failure_id": "01m7tfk2rbn2a6gxmwfn6jgjcb", + "activity_type": "m1", + "non_retryable": true, + "activity_class": "m1", + "attempt_number": 1, + "exception_type": null, + "exception_class": "RuntimeException", + "failure_category": "activity", + "parallel_group_id": "parallel-activities:4:2", + "activity_attempt_id": "01M7TFK2NMPZ2GTRSERF2DVTVS", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfk2e1rd6x1h4rks9q8h1m", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfk2e41fj1b5599zer5akp", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:37.197735Z" + }, + { + "id": "01m7tfk4100ync6haw3sqj9x0r", + "sequence": 15, + "event_type": "ActivityFailed", + "namespace": "default", + "payload": { + "code": 0, + "task": { + "id": "01m7tfk2dybkwr9dc5arwgd3d6", + "type": "activity", + "queue": "rv307-4704a8", + "status": "leased", + "leased_at": "2026-11-19T18:02:37.020000Z", + "connection": "mysql", + "lease_owner": "rv307-w-32445d", + "available_at": "2026-11-19T18:02:36.862214Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:37.077232Z" + }, + "message": "fail-m0", + "activity": { + "id": "01m7tfk2dpe7zwym36as2r13pc", + "type": "m0", + "class": "m0", + "queue": "rv307-4704a8", + "status": "failed", + "sequence": 4, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "closed_at": "2026-11-19T18:02:38.495173Z", + "exception": { + "code": 0, + "type": null, + "message": "fail-m0", + "non_retryable": true + }, + "attempt_id": "01M7TFK2JW051RRZ877NTRK82W", + "created_at": "2026-11-19T18:02:36.854843Z", + "started_at": "2026-11-19T18:02:37.020000Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfk2dpe7zwym36as2r13pc" + }, + "sequence": 4, + "exception": { + "type": null, + "message": "fail-m0", + "code": 0, + "non_retryable": true + }, + "failure_id": "01m7tfk40y2ey7crwkn9s30zw7", + "activity_type": "m0", + "non_retryable": true, + "activity_class": "m0", + "attempt_number": 1, + "exception_type": null, + "exception_class": "RuntimeException", + "failure_category": "activity", + "parallel_group_id": "parallel-activities:4:2", + "activity_attempt_id": "01M7TFK2JW051RRZ877NTRK82W", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfk2dpe7zwym36as2r13pc", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfk2dybkwr9dc5arwgd3d6", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:38.496393Z" + }, + { + "id": "01m7tfk51n35yy5njf0xrbc2bc", + "sequence": 16, + "event_type": "UpdateAccepted", + "namespace": "default", + "payload": { + "command": { + "id": "01m7tfk51j6kch8zagwdree11k", + "type": "update", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNw4GCG5hbWUKDHJlcG9ydBJhcmd1bWVudHMMACJ2YWxpZGF0aW9uX2Vycm9ycwwAAA==" + }, + "sequence": 2, + "request_id": "sdk-python-update-f775b3ab26ba4f3eb8b97e22a29e9f40", + "accepted_at": "2026-11-19T18:02:39.535895Z", + "auth_method": "token", + "auth_status": "authorized", + "target_name": "report", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows/rv307-rv348_handle-6e9a63/update/report", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfjvyq564n3r52xph03nz8", + "message_sequence": 1, + "request_route_name": "generated::YF2YckSNfLzTOm1b", + "request_fingerprint": "sha256:3039b4cfc66100d1c91e62db99f08b97f22a369ebfd6e1b08249c4e7f66b2788" + }, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "update_id": "01m7tfk51kksbqv2gxkekpx2vf", + "request_id": "sdk-python-update-f775b3ab26ba4f3eb8b97e22a29e9f40", + "update_name": "report", + "ordering_state": "ready", + "workflow_run_id": "01m7tfjvyq564n3r52xph03nz8", + "workflow_command_id": "01m7tfk51j6kch8zagwdree11k", + "workflow_instance_id": "rv307-rv348_handle-6e9a63", + "queued_behind_command_id": null, + "queued_behind_command_type": null, + "queued_behind_command_sequence": null + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfk51j6kch8zagwdree11k", + "recorded_at": "2026-11-19T18:02:39.540869Z" + }, + { + "id": "01m7tfk5b424hdethtx4r8w5nq", + "sequence": 17, + "event_type": "UpdateApplied", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfk416bbxfd6b7vfb9vxbk", + "type": "workflow", + "queue": "rv307-4704a8", + "status": "leased", + "leased_at": "2026-11-19T18:02:39.609742Z", + "connection": "mysql", + "lease_owner": "rv307-w-32445d", + "available_at": "2026-11-19T18:02:38.502057Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:49.609701Z", + "last_dispatched_at": "2026-11-19T18:02:38.531227Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:38.531227Z" + }, + "command": { + "id": "01m7tfk51j6kch8zagwdree11k", + "type": "update", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNw4GCG5hbWUKDHJlcG9ydBJhcmd1bWVudHMMACJ2YWxpZGF0aW9uX2Vycm9ycwwAAA==" + }, + "sequence": 2, + "request_id": "sdk-python-update-f775b3ab26ba4f3eb8b97e22a29e9f40", + "accepted_at": "2026-11-19T18:02:39.535895Z", + "auth_method": "token", + "auth_status": "authorized", + "target_name": "report", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows/rv307-rv348_handle-6e9a63/update/report", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfjvyq564n3r52xph03nz8", + "message_sequence": 1, + "request_route_name": "generated::YF2YckSNfLzTOm1b", + "request_fingerprint": "sha256:3039b4cfc66100d1c91e62db99f08b97f22a369ebfd6e1b08249c4e7f66b2788" + }, + "sequence": 6, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "update_id": "01m7tfk51kksbqv2gxkekpx2vf", + "update_name": "report", + "workflow_run_id": "01m7tfjvyq564n3r52xph03nz8", + "workflow_command_id": "01m7tfk51j6kch8zagwdree11k", + "workflow_instance_id": "rv307-rv348_handle-6e9a63" + }, + "workflow_task_id": "01m7tfk416bbxfd6b7vfb9vxbk", + "workflow_command_id": "01m7tfk51j6kch8zagwdree11k", + "recorded_at": "2026-11-19T18:02:39.843848Z" + }, + { + "id": "01m7tfk5b87vyq710mf7he3ab6", + "sequence": 18, + "event_type": "UpdateCompleted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfk416bbxfd6b7vfb9vxbk", + "type": "workflow", + "queue": "rv307-4704a8", + "status": "leased", + "leased_at": "2026-11-19T18:02:39.609742Z", + "connection": "mysql", + "lease_owner": "rv307-w-32445d", + "available_at": "2026-11-19T18:02:38.502057Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:49.609701Z", + "last_dispatched_at": "2026-11-19T18:02:38.531227Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:38.531227Z" + }, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNw4CGGNvbXBlbnNhdGluZwoOZmFpbC1tMAA=" + }, + "command": { + "id": "01m7tfk51j6kch8zagwdree11k", + "type": "update", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNw4GCG5hbWUKDHJlcG9ydBJhcmd1bWVudHMMACJ2YWxpZGF0aW9uX2Vycm9ycwwAAA==" + }, + "sequence": 2, + "request_id": "sdk-python-update-f775b3ab26ba4f3eb8b97e22a29e9f40", + "accepted_at": "2026-11-19T18:02:39.535895Z", + "auth_method": "token", + "auth_status": "authorized", + "target_name": "report", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows/rv307-rv348_handle-6e9a63/update/report", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfjvyq564n3r52xph03nz8", + "message_sequence": 1, + "request_route_name": "generated::YF2YckSNfLzTOm1b", + "request_fingerprint": "sha256:3039b4cfc66100d1c91e62db99f08b97f22a369ebfd6e1b08249c4e7f66b2788" + }, + "sequence": 6, + "update_id": "01m7tfk51kksbqv2gxkekpx2vf", + "update_name": "report", + "workflow_run_id": "01m7tfjvyq564n3r52xph03nz8", + "workflow_command_id": "01m7tfk51j6kch8zagwdree11k", + "workflow_instance_id": "rv307-rv348_handle-6e9a63" + }, + "workflow_task_id": "01m7tfk416bbxfd6b7vfb9vxbk", + "workflow_command_id": "01m7tfk51j6kch8zagwdree11k", + "recorded_at": "2026-11-19T18:02:39.847400Z" + }, + { + "id": "01m7tfk5bce4sv8faxrmgpg9ep", + "sequence": 19, + "event_type": "MessageCursorAdvanced", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfk416bbxfd6b7vfb9vxbk", + "type": "workflow", + "queue": "rv307-4704a8", + "status": "leased", + "leased_at": "2026-11-19T18:02:39.609742Z", + "connection": "mysql", + "lease_owner": "rv307-w-32445d", + "available_at": "2026-11-19T18:02:38.502057Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:49.609701Z", + "last_dispatched_at": "2026-11-19T18:02:38.531227Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:38.531227Z" + }, + "stream_key": "instance:rv307-rv348_handle-6e9a63", + "new_position": 1, + "previous_position": 0 + }, + "workflow_task_id": "01m7tfk416bbxfd6b7vfb9vxbk", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:39.852473Z" + }, + { + "id": "01m7tfk5dkjfgf1hp8x67zrxe6", + "sequence": 20, + "event_type": "ActivityCompleted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfjwfa9v2ws6xfbyt09jjz", + "type": "activity", + "queue": "rv307-4704a8", + "status": "leased", + "leased_at": "2026-11-19T18:02:36.930722Z", + "connection": "mysql", + "lease_owner": "rv307-w-32445d", + "available_at": "2026-11-19T18:02:30.762733Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:36.987464Z" + }, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoEb2s=" + }, + "activity": { + "id": "01m7tfjwf7yh5ffwcwgdyt5cck", + "type": "slow", + "class": "slow", + "queue": "rv307-4704a8", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoEb2s=" + }, + "status": "completed", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "closed_at": "2026-11-19T18:02:39.921390Z", + "attempt_id": "01M7TFK2G2RTR8F0FA5JWDT4DR", + "created_at": "2026-11-19T18:02:30.759721Z", + "started_at": "2026-11-19T18:02:36.930722Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfjwf7yh5ffwcwgdyt5cck" + }, + "sequence": 2, + "activity_type": "slow", + "payload_codec": "avro", + "activity_class": "slow", + "attempt_number": 1, + "parallel_group_id": "select-calls:1:2", + "activity_attempt_id": "01M7TFK2G2RTR8F0FA5JWDT4DR", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "activity_execution_id": "01m7tfjwf7yh5ffwcwgdyt5cck", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + }, + "workflow_task_id": "01m7tfjwfa9v2ws6xfbyt09jjz", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:39.922670Z" + }, + { + "id": "01m7tfk5ytsew7e48smxq9hk25", + "sequence": 21, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfk5dw6aambstd8xh4f80y", + "type": "workflow", + "queue": "rv307-4704a8", + "status": "leased", + "leased_at": "2026-11-19T18:02:40.043577Z", + "connection": "mysql", + "lease_owner": "rv307-w-32445d", + "available_at": "2026-11-19T18:02:39.932103Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:50.043539Z", + "last_dispatched_at": "2026-11-19T18:02:39.967089Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:39.967089Z" + }, + "activity": { + "id": "01m7tfk5yqw3fad0k3xrmde3xk", + "type": "compensate", + "class": "compensate", + "queue": "rv307-4704a8", + "status": "pending", + "sequence": 6, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCCg5mYWlsLW0wAA==" + }, + "created_at": "2026-11-19T18:02:40.471729Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfk5yqw3fad0k3xrmde3xk" + }, + "sequence": 6, + "activity_type": "compensate", + "activity_class": "compensate", + "activity_execution_id": "01m7tfk5yqw3fad0k3xrmde3xk" + }, + "workflow_task_id": "01m7tfk5dw6aambstd8xh4f80y", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:40.473999Z" + }, + { + "id": "01m7tfk8236zpnwkdsx107a1jk", + "sequence": 22, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfk5yycyhefsryjf9bzkef", + "type": "activity", + "queue": "rv307-4704a8", + "status": "leased", + "leased_at": "2026-11-19T18:02:42.622979Z", + "connection": "mysql", + "lease_owner": "rv307-w-32445d", + "available_at": "2026-11-19T18:02:40.477877Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:42.623086Z" + }, + "activity": { + "id": "01m7tfk5yqw3fad0k3xrmde3xk", + "type": "compensate", + "class": "compensate", + "queue": "rv307-4704a8", + "status": "running", + "sequence": 6, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCCg5mYWlsLW0wAA==" + }, + "attempt_id": "01M7TFK81Z3XBMH87JTNQT6V3D", + "created_at": "2026-11-19T18:02:40.471729Z", + "started_at": "2026-11-19T18:02:42.622979Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfk5yqw3fad0k3xrmde3xk" + }, + "sequence": 6, + "activity_type": "compensate", + "activity_class": "compensate", + "attempt_number": 1, + "activity_attempt_id": "01M7TFK81Z3XBMH87JTNQT6V3D", + "activity_execution_id": "01m7tfk5yqw3fad0k3xrmde3xk" + }, + "workflow_task_id": "01m7tfk5yycyhefsryjf9bzkef", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:42.627350Z" + }, + { + "id": "01m7tfk854gdv5mqjn3smp8yp0", + "sequence": 23, + "event_type": "ActivityCompleted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfk5yycyhefsryjf9bzkef", + "type": "activity", + "queue": "rv307-4704a8", + "status": "leased", + "leased_at": "2026-11-19T18:02:42.622979Z", + "connection": "mysql", + "lease_owner": "rv307-w-32445d", + "available_at": "2026-11-19T18:02:40.477877Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:42.694941Z" + }, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoMdW5kb25l" + }, + "activity": { + "id": "01m7tfk5yqw3fad0k3xrmde3xk", + "type": "compensate", + "class": "compensate", + "queue": "rv307-4704a8", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoMdW5kb25l" + }, + "status": "completed", + "sequence": 6, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCCg5mYWlsLW0wAA==" + }, + "closed_at": "2026-11-19T18:02:42.722563Z", + "attempt_id": "01M7TFK81Z3XBMH87JTNQT6V3D", + "created_at": "2026-11-19T18:02:40.471729Z", + "started_at": "2026-11-19T18:02:42.622979Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfk5yqw3fad0k3xrmde3xk" + }, + "sequence": 6, + "activity_type": "compensate", + "payload_codec": "avro", + "activity_class": "compensate", + "attempt_number": 1, + "activity_attempt_id": "01M7TFK81Z3XBMH87JTNQT6V3D", + "activity_execution_id": "01m7tfk5yqw3fad0k3xrmde3xk" + }, + "workflow_task_id": "01m7tfk5yycyhefsryjf9bzkef", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:42.724023Z" + } + ], + "expected": { + "query_result": "fail-m0" + } +} diff --git a/tests/fixtures/replay_regressions/failure-order-released-update-between-tasks.json b/tests/fixtures/replay_regressions/failure-order-released-update-between-tasks.json new file mode 100644 index 00000000..4c485126 --- /dev/null +++ b/tests/fixtures/replay_regressions/failure-order-released-update-between-tasks.json @@ -0,0 +1,1523 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "failure-order-released-update-between-tasks", + "protocol_version": "1.20", + "bindings": [ + "python" + ], + "parallel_failure_order": { + "issues": [ + "durable-workflow/sdk-python#307" + ], + "note": "Recorded on a Server built from durable-workflow/server main 52e3ec00 (docker-compose.test.yml). Published 2.19.1 ran the zero-command catch of fail-m0 and answered an update with {compensating: fail-m0}; a worker of this version ran the remaining tasks and compensated fail-m0. This is the history of its last task." + }, + "workflow": { + "type": "rv348_handle", + "input": [], + "payload_codec": "avro", + "run_id": "01m7tfjvyq564n3r52xph03nz8", + "workflow_id": "rv307-rv348_handle-6e9a63" + }, + "history": [ + { + "id": "01m7tfjvyxhjfb1jnt7f0d1c66", + "sequence": 1, + "event_type": "StartAccepted", + "namespace": "default", + "payload": { + "memo": null, + "command": { + "id": "01m7tfjvyt8zwh2ccm9n5zbdhd", + "type": "start", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "outcome": "started_new", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "sequence": 1, + "applied_at": "2026-11-19T18:02:30.232740Z", + "accepted_at": "2026-11-19T18:02:30.232736Z", + "auth_method": "token", + "auth_status": "authorized", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfjvyq564n3r52xph03nz8", + "request_route_name": "generated::FqJuJMl97iiXlIDz", + "request_fingerprint": "sha256:e5e68285751d4aa76a22d2ea737bb10e6ee57ea6e5e339b1a62d530d40a99ba1" + }, + "outcome": "started_new", + "business_key": null, + "workflow_type": "rv348_handle", + "workflow_class": "rv348_handle", + "workflow_run_id": "01m7tfjvyq564n3r52xph03nz8", + "search_attributes": null, + "visibility_labels": null, + "workflow_command_id": "01m7tfjvyt8zwh2ccm9n5zbdhd", + "workflow_instance_id": "rv307-rv348_handle-6e9a63" + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfjvyt8zwh2ccm9n5zbdhd", + "recorded_at": "2026-11-19T18:02:30.236558Z" + }, + { + "id": "01m7tfjvz0jbm2e2xh8ma9k6ev", + "sequence": 2, + "event_type": "WorkflowStarted", + "namespace": "default", + "payload": { + "command": { + "id": "01m7tfjvyt8zwh2ccm9n5zbdhd", + "type": "start", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "outcome": "started_new", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "sequence": 1, + "applied_at": "2026-11-19T18:02:30.232740Z", + "accepted_at": "2026-11-19T18:02:30.232736Z", + "auth_method": "token", + "auth_status": "authorized", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfjvyq564n3r52xph03nz8", + "request_route_name": "generated::FqJuJMl97iiXlIDz", + "request_fingerprint": "sha256:e5e68285751d4aa76a22d2ea737bb10e6ee57ea6e5e339b1a62d530d40a99ba1" + }, + "workflow_type": "rv348_handle", + "workflow_class": "rv348_handle", + "run_deadline_at": "2026-11-19T18:12:30+00:00", + "workflow_run_id": "01m7tfjvyq564n3r52xph03nz8", + "declared_queries": [ + "caught" + ], + "declared_signals": [], + "declared_updates": [ + "report" + ], + "declared_entry_mode": "canonical", + "run_timeout_seconds": 600, + "workflow_command_id": "01m7tfjvyt8zwh2ccm9n5zbdhd", + "workflow_instance_id": "rv307-rv348_handle-6e9a63", + "declared_entry_method": "handle", + "execution_deadline_at": "2026-11-19T19:02:30+00:00", + "declared_query_contracts": [ + { + "name": "caught", + "parameters": [] + } + ], + "declared_signal_contracts": [], + "declared_update_contracts": [ + { + "name": "report", + "parameters": [ + { + "name": "_", + "type": null, + "default": null, + "position": 0, + "required": false, + "variadic": true, + "allows_null": true, + "default_available": false + } + ] + } + ], + "execution_timeout_seconds": 3600, + "declared_update_validators": [], + "declared_entry_declaring_class": "external:rv348_handle", + "workflow_definition_fingerprint": "sha256:69ed762d44804878ebb703a2ad385119f6f776d0dae561c15f8f84e1707a704a", + "workflow_definition_fingerprint_source": "worker" + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfjvyt8zwh2ccm9n5zbdhd", + "recorded_at": "2026-11-19T18:02:30.239683Z" + }, + { + "id": "01m7tfjwf3kqkct8sgg13v901w", + "sequence": 3, + "event_type": "TimerScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfjvz1964m1nbs2m582070", + "type": "workflow", + "queue": "rv307-4704a8", + "status": "leased", + "leased_at": "2026-11-19T18:02:30.306973Z", + "connection": "mysql", + "lease_owner": "rv307-w-32445d", + "available_at": "2026-11-19T18:02:30.241603Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:40.306911Z", + "last_dispatched_at": "2026-11-19T18:02:30.243567Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:30.243567Z" + }, + "fire_at": "2026-11-19T18:02:31.753185Z", + "sequence": 1, + "timer_id": "01m7tfjwf131zf70c1pthg1gtz", + "delay_seconds": 1, + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + }, + "workflow_task_id": "01m7tfjvz1964m1nbs2m582070", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:30.754427Z" + }, + { + "id": "01m7tfjwf9mw3nf0m06jag0rpp", + "sequence": 4, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfjvz1964m1nbs2m582070", + "type": "workflow", + "queue": "rv307-4704a8", + "status": "leased", + "leased_at": "2026-11-19T18:02:30.306973Z", + "connection": "mysql", + "lease_owner": "rv307-w-32445d", + "available_at": "2026-11-19T18:02:30.241603Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:40.306911Z", + "last_dispatched_at": "2026-11-19T18:02:30.243567Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:30.243567Z" + }, + "activity": { + "id": "01m7tfjwf7yh5ffwcwgdyt5cck", + "type": "slow", + "class": "slow", + "queue": "rv307-4704a8", + "status": "pending", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "created_at": "2026-11-19T18:02:30.759721Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfjwf7yh5ffwcwgdyt5cck" + }, + "sequence": 2, + "activity_type": "slow", + "activity_class": "slow", + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "activity_execution_id": "01m7tfjwf7yh5ffwcwgdyt5cck", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + }, + "workflow_task_id": "01m7tfjvz1964m1nbs2m582070", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:30.760884Z" + }, + { + "id": "01m7tfjy636brag78mp7f2etay", + "sequence": 5, + "event_type": "TimerFired", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfjwf6tfreseph1kvnbawh", + "type": "timer", + "queue": "rv307-4704a8", + "status": "leased", + "leased_at": "2026-11-19T18:02:32.497518Z", + "connection": "mysql", + "lease_owner": "01m7tfjwf6tfreseph1kvnbawh", + "available_at": "2026-11-19T18:02:31.753185Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:42.497522Z" + }, + "fire_at": "2026-11-19T18:02:31.753185Z", + "fired_at": "2026-11-19T18:02:32.513880Z", + "sequence": 1, + "timer_id": "01m7tfjwf131zf70c1pthg1gtz", + "delay_seconds": 1, + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + }, + "workflow_task_id": "01m7tfjwf6tfreseph1kvnbawh", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:32.514927Z" + }, + { + "id": "01m7tfjy66g1ev3wsexrdec2mh", + "sequence": 6, + "event_type": "SelectionResolved", + "namespace": "default", + "payload": { + "outcome": "completed", + "member_key": "t", + "member_size": 1, + "member_index": 0, + "operation_kind": "timer", + "operation_identity": "01m7tfjwf131zf70c1pthg1gtz", + "selection_group_id": "select-calls:1:2", + "resolution_event_id": "01m7tfjy636brag78mp7f2etay", + "member_base_sequence": 1, + "selection_group_size": 2, + "resolution_event_type": "TimerFired", + "selection_group_base_sequence": 1 + }, + "workflow_task_id": null, + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:32.518142Z" + }, + { + "id": "01m7tfjzf9vnczgmw2m5ryv8xk", + "sequence": 7, + "event_type": "TimerScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfjy680jgfhnpdzb5aj7p6", + "type": "workflow", + "queue": "rv307-4704a8", + "status": "leased", + "leased_at": "2026-11-19T18:02:33.402266Z", + "connection": "mysql", + "lease_owner": "rv307-w-32445d", + "available_at": "2026-11-19T18:02:32.520491Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:43.402145Z", + "last_dispatched_at": "2026-11-19T18:02:32.538006Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:32.538006Z" + }, + "fire_at": "2026-11-19T18:02:34.830949Z", + "sequence": 3, + "timer_id": "01m7tfjzf7q72xkgzbp7w34va7", + "delay_seconds": 1 + }, + "workflow_task_id": "01m7tfjy680jgfhnpdzb5aj7p6", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:33.832578Z" + }, + { + "id": "01m7tfk17chkh6pa62r3g749we", + "sequence": 8, + "event_type": "TimerFired", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfjzfctvmyh9rz4vq2512w", + "type": "timer", + "queue": "rv307-4704a8", + "status": "leased", + "leased_at": "2026-11-19T18:02:35.602330Z", + "connection": "mysql", + "lease_owner": "01m7tfjzfctvmyh9rz4vq2512w", + "available_at": "2026-11-19T18:02:34.830949Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:45.602335Z" + }, + "fire_at": "2026-11-19T18:02:34.830949Z", + "fired_at": "2026-11-19T18:02:35.626529Z", + "sequence": 3, + "timer_id": "01m7tfjzf7q72xkgzbp7w34va7", + "delay_seconds": 1 + }, + "workflow_task_id": "01m7tfjzfctvmyh9rz4vq2512w", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:35.627926Z" + }, + { + "id": "01m7tfk2dtqaqa7ph7s6ka0kr0", + "sequence": 9, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfk17f668ma8m55583xw3h", + "type": "workflow", + "queue": "rv307-4704a8", + "status": "leased", + "leased_at": "2026-11-19T18:02:36.417139Z", + "connection": "mysql", + "lease_owner": "rv307-w-32445d", + "available_at": "2026-11-19T18:02:35.630901Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:46.417107Z", + "last_dispatched_at": "2026-11-19T18:02:35.654654Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:35.654654Z" + }, + "activity": { + "id": "01m7tfk2dpe7zwym36as2r13pc", + "type": "m0", + "class": "m0", + "queue": "rv307-4704a8", + "status": "pending", + "sequence": 4, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "created_at": "2026-11-19T18:02:36.854843Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfk2dpe7zwym36as2r13pc" + }, + "sequence": 4, + "activity_type": "m0", + "activity_class": "m0", + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfk2dpe7zwym36as2r13pc", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfk17f668ma8m55583xw3h", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:36.857406Z" + }, + { + "id": "01m7tfk2e2mqvmgge6y3mjbrwc", + "sequence": 10, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfk17f668ma8m55583xw3h", + "type": "workflow", + "queue": "rv307-4704a8", + "status": "leased", + "leased_at": "2026-11-19T18:02:36.417139Z", + "connection": "mysql", + "lease_owner": "rv307-w-32445d", + "available_at": "2026-11-19T18:02:35.630901Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:46.417107Z", + "last_dispatched_at": "2026-11-19T18:02:35.654654Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:35.654654Z" + }, + "activity": { + "id": "01m7tfk2e1rd6x1h4rks9q8h1m", + "type": "m1", + "class": "m1", + "queue": "rv307-4704a8", + "status": "pending", + "sequence": 5, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "created_at": "2026-11-19T18:02:36.865400Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfk2e1rd6x1h4rks9q8h1m" + }, + "sequence": 5, + "activity_type": "m1", + "activity_class": "m1", + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfk2e1rd6x1h4rks9q8h1m", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfk17f668ma8m55583xw3h", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:36.866397Z" + }, + { + "id": "01m7tfk2g72rnm9xta2wvhevvd", + "sequence": 11, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfjwfa9v2ws6xfbyt09jjz", + "type": "activity", + "queue": "rv307-4704a8", + "status": "leased", + "leased_at": "2026-11-19T18:02:36.930722Z", + "connection": "mysql", + "lease_owner": "rv307-w-32445d", + "available_at": "2026-11-19T18:02:30.762733Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:36.930804Z" + }, + "activity": { + "id": "01m7tfjwf7yh5ffwcwgdyt5cck", + "type": "slow", + "class": "slow", + "queue": "rv307-4704a8", + "status": "running", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "attempt_id": "01M7TFK2G2RTR8F0FA5JWDT4DR", + "created_at": "2026-11-19T18:02:30.759721Z", + "started_at": "2026-11-19T18:02:36.930722Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfjwf7yh5ffwcwgdyt5cck" + }, + "sequence": 2, + "activity_type": "slow", + "activity_class": "slow", + "attempt_number": 1, + "parallel_group_id": "select-calls:1:2", + "activity_attempt_id": "01M7TFK2G2RTR8F0FA5JWDT4DR", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "activity_execution_id": "01m7tfjwf7yh5ffwcwgdyt5cck", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + }, + "workflow_task_id": "01m7tfjwfa9v2ws6xfbyt09jjz", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:36.934760Z" + }, + { + "id": "01m7tfk2k0rxrn8v326f3ja5j7", + "sequence": 12, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfk2dybkwr9dc5arwgd3d6", + "type": "activity", + "queue": "rv307-4704a8", + "status": "leased", + "leased_at": "2026-11-19T18:02:37.020000Z", + "connection": "mysql", + "lease_owner": "rv307-w-32445d", + "available_at": "2026-11-19T18:02:36.862214Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:37.020089Z" + }, + "activity": { + "id": "01m7tfk2dpe7zwym36as2r13pc", + "type": "m0", + "class": "m0", + "queue": "rv307-4704a8", + "status": "running", + "sequence": 4, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "attempt_id": "01M7TFK2JW051RRZ877NTRK82W", + "created_at": "2026-11-19T18:02:36.854843Z", + "started_at": "2026-11-19T18:02:37.020000Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfk2dpe7zwym36as2r13pc" + }, + "sequence": 4, + "activity_type": "m0", + "activity_class": "m0", + "attempt_number": 1, + "parallel_group_id": "parallel-activities:4:2", + "activity_attempt_id": "01M7TFK2JW051RRZ877NTRK82W", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfk2dpe7zwym36as2r13pc", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfk2dybkwr9dc5arwgd3d6", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:37.024253Z" + }, + { + "id": "01m7tfk2nszsxkbxmyd1r8w0ap", + "sequence": 13, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfk2e41fj1b5599zer5akp", + "type": "activity", + "queue": "rv307-4704a8", + "status": "leased", + "leased_at": "2026-11-19T18:02:37.108870Z", + "connection": "mysql", + "lease_owner": "rv307-w-32445d", + "available_at": "2026-11-19T18:02:36.868284Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:37.108961Z" + }, + "activity": { + "id": "01m7tfk2e1rd6x1h4rks9q8h1m", + "type": "m1", + "class": "m1", + "queue": "rv307-4704a8", + "status": "running", + "sequence": 5, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "attempt_id": "01M7TFK2NMPZ2GTRSERF2DVTVS", + "created_at": "2026-11-19T18:02:36.865400Z", + "started_at": "2026-11-19T18:02:37.108870Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfk2e1rd6x1h4rks9q8h1m" + }, + "sequence": 5, + "activity_type": "m1", + "activity_class": "m1", + "attempt_number": 1, + "parallel_group_id": "parallel-activities:4:2", + "activity_attempt_id": "01M7TFK2NMPZ2GTRSERF2DVTVS", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfk2e1rd6x1h4rks9q8h1m", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfk2e41fj1b5599zer5akp", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:37.113029Z" + }, + { + "id": "01m7tfk2reth2f7dsebq0yyx68", + "sequence": 14, + "event_type": "ActivityFailed", + "namespace": "default", + "payload": { + "code": 0, + "task": { + "id": "01m7tfk2e41fj1b5599zer5akp", + "type": "activity", + "queue": "rv307-4704a8", + "status": "leased", + "leased_at": "2026-11-19T18:02:37.108870Z", + "connection": "mysql", + "lease_owner": "rv307-w-32445d", + "available_at": "2026-11-19T18:02:36.868284Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:37.168218Z" + }, + "message": "fail-m1", + "activity": { + "id": "01m7tfk2e1rd6x1h4rks9q8h1m", + "type": "m1", + "class": "m1", + "queue": "rv307-4704a8", + "status": "failed", + "sequence": 5, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "closed_at": "2026-11-19T18:02:37.196150Z", + "exception": { + "code": 0, + "type": null, + "message": "fail-m1", + "non_retryable": true + }, + "attempt_id": "01M7TFK2NMPZ2GTRSERF2DVTVS", + "created_at": "2026-11-19T18:02:36.865400Z", + "started_at": "2026-11-19T18:02:37.108870Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfk2e1rd6x1h4rks9q8h1m" + }, + "sequence": 5, + "exception": { + "type": null, + "message": "fail-m1", + "code": 0, + "non_retryable": true + }, + "failure_id": "01m7tfk2rbn2a6gxmwfn6jgjcb", + "activity_type": "m1", + "non_retryable": true, + "activity_class": "m1", + "attempt_number": 1, + "exception_type": null, + "exception_class": "RuntimeException", + "failure_category": "activity", + "parallel_group_id": "parallel-activities:4:2", + "activity_attempt_id": "01M7TFK2NMPZ2GTRSERF2DVTVS", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfk2e1rd6x1h4rks9q8h1m", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfk2e41fj1b5599zer5akp", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:37.197735Z" + }, + { + "id": "01m7tfk4100ync6haw3sqj9x0r", + "sequence": 15, + "event_type": "ActivityFailed", + "namespace": "default", + "payload": { + "code": 0, + "task": { + "id": "01m7tfk2dybkwr9dc5arwgd3d6", + "type": "activity", + "queue": "rv307-4704a8", + "status": "leased", + "leased_at": "2026-11-19T18:02:37.020000Z", + "connection": "mysql", + "lease_owner": "rv307-w-32445d", + "available_at": "2026-11-19T18:02:36.862214Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:37.077232Z" + }, + "message": "fail-m0", + "activity": { + "id": "01m7tfk2dpe7zwym36as2r13pc", + "type": "m0", + "class": "m0", + "queue": "rv307-4704a8", + "status": "failed", + "sequence": 4, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "closed_at": "2026-11-19T18:02:38.495173Z", + "exception": { + "code": 0, + "type": null, + "message": "fail-m0", + "non_retryable": true + }, + "attempt_id": "01M7TFK2JW051RRZ877NTRK82W", + "created_at": "2026-11-19T18:02:36.854843Z", + "started_at": "2026-11-19T18:02:37.020000Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfk2dpe7zwym36as2r13pc" + }, + "sequence": 4, + "exception": { + "type": null, + "message": "fail-m0", + "code": 0, + "non_retryable": true + }, + "failure_id": "01m7tfk40y2ey7crwkn9s30zw7", + "activity_type": "m0", + "non_retryable": true, + "activity_class": "m0", + "attempt_number": 1, + "exception_type": null, + "exception_class": "RuntimeException", + "failure_category": "activity", + "parallel_group_id": "parallel-activities:4:2", + "activity_attempt_id": "01M7TFK2JW051RRZ877NTRK82W", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfk2dpe7zwym36as2r13pc", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfk2dybkwr9dc5arwgd3d6", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:38.496393Z" + }, + { + "id": "01m7tfk51n35yy5njf0xrbc2bc", + "sequence": 16, + "event_type": "UpdateAccepted", + "namespace": "default", + "payload": { + "command": { + "id": "01m7tfk51j6kch8zagwdree11k", + "type": "update", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNw4GCG5hbWUKDHJlcG9ydBJhcmd1bWVudHMMACJ2YWxpZGF0aW9uX2Vycm9ycwwAAA==" + }, + "sequence": 2, + "request_id": "sdk-python-update-f775b3ab26ba4f3eb8b97e22a29e9f40", + "accepted_at": "2026-11-19T18:02:39.535895Z", + "auth_method": "token", + "auth_status": "authorized", + "target_name": "report", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows/rv307-rv348_handle-6e9a63/update/report", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfjvyq564n3r52xph03nz8", + "message_sequence": 1, + "request_route_name": "generated::YF2YckSNfLzTOm1b", + "request_fingerprint": "sha256:3039b4cfc66100d1c91e62db99f08b97f22a369ebfd6e1b08249c4e7f66b2788" + }, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "update_id": "01m7tfk51kksbqv2gxkekpx2vf", + "request_id": "sdk-python-update-f775b3ab26ba4f3eb8b97e22a29e9f40", + "update_name": "report", + "ordering_state": "ready", + "workflow_run_id": "01m7tfjvyq564n3r52xph03nz8", + "workflow_command_id": "01m7tfk51j6kch8zagwdree11k", + "workflow_instance_id": "rv307-rv348_handle-6e9a63", + "queued_behind_command_id": null, + "queued_behind_command_type": null, + "queued_behind_command_sequence": null + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfk51j6kch8zagwdree11k", + "recorded_at": "2026-11-19T18:02:39.540869Z" + }, + { + "id": "01m7tfk5b424hdethtx4r8w5nq", + "sequence": 17, + "event_type": "UpdateApplied", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfk416bbxfd6b7vfb9vxbk", + "type": "workflow", + "queue": "rv307-4704a8", + "status": "leased", + "leased_at": "2026-11-19T18:02:39.609742Z", + "connection": "mysql", + "lease_owner": "rv307-w-32445d", + "available_at": "2026-11-19T18:02:38.502057Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:49.609701Z", + "last_dispatched_at": "2026-11-19T18:02:38.531227Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:38.531227Z" + }, + "command": { + "id": "01m7tfk51j6kch8zagwdree11k", + "type": "update", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNw4GCG5hbWUKDHJlcG9ydBJhcmd1bWVudHMMACJ2YWxpZGF0aW9uX2Vycm9ycwwAAA==" + }, + "sequence": 2, + "request_id": "sdk-python-update-f775b3ab26ba4f3eb8b97e22a29e9f40", + "accepted_at": "2026-11-19T18:02:39.535895Z", + "auth_method": "token", + "auth_status": "authorized", + "target_name": "report", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows/rv307-rv348_handle-6e9a63/update/report", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfjvyq564n3r52xph03nz8", + "message_sequence": 1, + "request_route_name": "generated::YF2YckSNfLzTOm1b", + "request_fingerprint": "sha256:3039b4cfc66100d1c91e62db99f08b97f22a369ebfd6e1b08249c4e7f66b2788" + }, + "sequence": 6, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "update_id": "01m7tfk51kksbqv2gxkekpx2vf", + "update_name": "report", + "workflow_run_id": "01m7tfjvyq564n3r52xph03nz8", + "workflow_command_id": "01m7tfk51j6kch8zagwdree11k", + "workflow_instance_id": "rv307-rv348_handle-6e9a63" + }, + "workflow_task_id": "01m7tfk416bbxfd6b7vfb9vxbk", + "workflow_command_id": "01m7tfk51j6kch8zagwdree11k", + "recorded_at": "2026-11-19T18:02:39.843848Z" + }, + { + "id": "01m7tfk5b87vyq710mf7he3ab6", + "sequence": 18, + "event_type": "UpdateCompleted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfk416bbxfd6b7vfb9vxbk", + "type": "workflow", + "queue": "rv307-4704a8", + "status": "leased", + "leased_at": "2026-11-19T18:02:39.609742Z", + "connection": "mysql", + "lease_owner": "rv307-w-32445d", + "available_at": "2026-11-19T18:02:38.502057Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:49.609701Z", + "last_dispatched_at": "2026-11-19T18:02:38.531227Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:38.531227Z" + }, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNw4CGGNvbXBlbnNhdGluZwoOZmFpbC1tMAA=" + }, + "command": { + "id": "01m7tfk51j6kch8zagwdree11k", + "type": "update", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNw4GCG5hbWUKDHJlcG9ydBJhcmd1bWVudHMMACJ2YWxpZGF0aW9uX2Vycm9ycwwAAA==" + }, + "sequence": 2, + "request_id": "sdk-python-update-f775b3ab26ba4f3eb8b97e22a29e9f40", + "accepted_at": "2026-11-19T18:02:39.535895Z", + "auth_method": "token", + "auth_status": "authorized", + "target_name": "report", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows/rv307-rv348_handle-6e9a63/update/report", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfjvyq564n3r52xph03nz8", + "message_sequence": 1, + "request_route_name": "generated::YF2YckSNfLzTOm1b", + "request_fingerprint": "sha256:3039b4cfc66100d1c91e62db99f08b97f22a369ebfd6e1b08249c4e7f66b2788" + }, + "sequence": 6, + "update_id": "01m7tfk51kksbqv2gxkekpx2vf", + "update_name": "report", + "workflow_run_id": "01m7tfjvyq564n3r52xph03nz8", + "workflow_command_id": "01m7tfk51j6kch8zagwdree11k", + "workflow_instance_id": "rv307-rv348_handle-6e9a63" + }, + "workflow_task_id": "01m7tfk416bbxfd6b7vfb9vxbk", + "workflow_command_id": "01m7tfk51j6kch8zagwdree11k", + "recorded_at": "2026-11-19T18:02:39.847400Z" + }, + { + "id": "01m7tfk5bce4sv8faxrmgpg9ep", + "sequence": 19, + "event_type": "MessageCursorAdvanced", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfk416bbxfd6b7vfb9vxbk", + "type": "workflow", + "queue": "rv307-4704a8", + "status": "leased", + "leased_at": "2026-11-19T18:02:39.609742Z", + "connection": "mysql", + "lease_owner": "rv307-w-32445d", + "available_at": "2026-11-19T18:02:38.502057Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:49.609701Z", + "last_dispatched_at": "2026-11-19T18:02:38.531227Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:38.531227Z" + }, + "stream_key": "instance:rv307-rv348_handle-6e9a63", + "new_position": 1, + "previous_position": 0 + }, + "workflow_task_id": "01m7tfk416bbxfd6b7vfb9vxbk", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:39.852473Z" + }, + { + "id": "01m7tfk5dkjfgf1hp8x67zrxe6", + "sequence": 20, + "event_type": "ActivityCompleted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfjwfa9v2ws6xfbyt09jjz", + "type": "activity", + "queue": "rv307-4704a8", + "status": "leased", + "leased_at": "2026-11-19T18:02:36.930722Z", + "connection": "mysql", + "lease_owner": "rv307-w-32445d", + "available_at": "2026-11-19T18:02:30.762733Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:36.987464Z" + }, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoEb2s=" + }, + "activity": { + "id": "01m7tfjwf7yh5ffwcwgdyt5cck", + "type": "slow", + "class": "slow", + "queue": "rv307-4704a8", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoEb2s=" + }, + "status": "completed", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "closed_at": "2026-11-19T18:02:39.921390Z", + "attempt_id": "01M7TFK2G2RTR8F0FA5JWDT4DR", + "created_at": "2026-11-19T18:02:30.759721Z", + "started_at": "2026-11-19T18:02:36.930722Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfjwf7yh5ffwcwgdyt5cck" + }, + "sequence": 2, + "activity_type": "slow", + "payload_codec": "avro", + "activity_class": "slow", + "attempt_number": 1, + "parallel_group_id": "select-calls:1:2", + "activity_attempt_id": "01M7TFK2G2RTR8F0FA5JWDT4DR", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "activity_execution_id": "01m7tfjwf7yh5ffwcwgdyt5cck", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + }, + "workflow_task_id": "01m7tfjwfa9v2ws6xfbyt09jjz", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:39.922670Z" + }, + { + "id": "01m7tfk5ytsew7e48smxq9hk25", + "sequence": 21, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfk5dw6aambstd8xh4f80y", + "type": "workflow", + "queue": "rv307-4704a8", + "status": "leased", + "leased_at": "2026-11-19T18:02:40.043577Z", + "connection": "mysql", + "lease_owner": "rv307-w-32445d", + "available_at": "2026-11-19T18:02:39.932103Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:50.043539Z", + "last_dispatched_at": "2026-11-19T18:02:39.967089Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:39.967089Z" + }, + "activity": { + "id": "01m7tfk5yqw3fad0k3xrmde3xk", + "type": "compensate", + "class": "compensate", + "queue": "rv307-4704a8", + "status": "pending", + "sequence": 6, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCCg5mYWlsLW0wAA==" + }, + "created_at": "2026-11-19T18:02:40.471729Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfk5yqw3fad0k3xrmde3xk" + }, + "sequence": 6, + "activity_type": "compensate", + "activity_class": "compensate", + "activity_execution_id": "01m7tfk5yqw3fad0k3xrmde3xk" + }, + "workflow_task_id": "01m7tfk5dw6aambstd8xh4f80y", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:40.473999Z" + }, + { + "id": "01m7tfk8236zpnwkdsx107a1jk", + "sequence": 22, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfk5yycyhefsryjf9bzkef", + "type": "activity", + "queue": "rv307-4704a8", + "status": "leased", + "leased_at": "2026-11-19T18:02:42.622979Z", + "connection": "mysql", + "lease_owner": "rv307-w-32445d", + "available_at": "2026-11-19T18:02:40.477877Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:42.623086Z" + }, + "activity": { + "id": "01m7tfk5yqw3fad0k3xrmde3xk", + "type": "compensate", + "class": "compensate", + "queue": "rv307-4704a8", + "status": "running", + "sequence": 6, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCCg5mYWlsLW0wAA==" + }, + "attempt_id": "01M7TFK81Z3XBMH87JTNQT6V3D", + "created_at": "2026-11-19T18:02:40.471729Z", + "started_at": "2026-11-19T18:02:42.622979Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfk5yqw3fad0k3xrmde3xk" + }, + "sequence": 6, + "activity_type": "compensate", + "activity_class": "compensate", + "attempt_number": 1, + "activity_attempt_id": "01M7TFK81Z3XBMH87JTNQT6V3D", + "activity_execution_id": "01m7tfk5yqw3fad0k3xrmde3xk" + }, + "workflow_task_id": "01m7tfk5yycyhefsryjf9bzkef", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:42.627350Z" + }, + { + "id": "01m7tfk854gdv5mqjn3smp8yp0", + "sequence": 23, + "event_type": "ActivityCompleted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfk5yycyhefsryjf9bzkef", + "type": "activity", + "queue": "rv307-4704a8", + "status": "leased", + "leased_at": "2026-11-19T18:02:42.622979Z", + "connection": "mysql", + "lease_owner": "rv307-w-32445d", + "available_at": "2026-11-19T18:02:40.477877Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:42.694941Z" + }, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoMdW5kb25l" + }, + "activity": { + "id": "01m7tfk5yqw3fad0k3xrmde3xk", + "type": "compensate", + "class": "compensate", + "queue": "rv307-4704a8", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoMdW5kb25l" + }, + "status": "completed", + "sequence": 6, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCCg5mYWlsLW0wAA==" + }, + "closed_at": "2026-11-19T18:02:42.722563Z", + "attempt_id": "01M7TFK81Z3XBMH87JTNQT6V3D", + "created_at": "2026-11-19T18:02:40.471729Z", + "started_at": "2026-11-19T18:02:42.622979Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfk5yqw3fad0k3xrmde3xk" + }, + "sequence": 6, + "activity_type": "compensate", + "payload_codec": "avro", + "activity_class": "compensate", + "attempt_number": 1, + "activity_attempt_id": "01M7TFK81Z3XBMH87JTNQT6V3D", + "activity_execution_id": "01m7tfk5yqw3fad0k3xrmde3xk" + }, + "workflow_task_id": "01m7tfk5yycyhefsryjf9bzkef", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:42.724023Z" + } + ], + "expected": { + "command_sequence": [ + { + "type": "complete_workflow" + } + ] + } +} diff --git a/tests/fixtures/replay_regressions/failure-order-released-zero-command-catch-complete-query.json b/tests/fixtures/replay_regressions/failure-order-released-zero-command-catch-complete-query.json new file mode 100644 index 00000000..91931911 --- /dev/null +++ b/tests/fixtures/replay_regressions/failure-order-released-zero-command-catch-complete-query.json @@ -0,0 +1,1292 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "failure-order-released-zero-command-catch-complete-query", + "protocol_version": "1.20", + "bindings": [ + "python" + ], + "parallel_failure_order": { + "issues": [ + "durable-workflow/sdk-python#307" + ], + "note": "The history the last task of the 2.19.1 run of failure-order-released-zero-command-catch received on a Server built from durable-workflow/server main 52e3ec00 (docker-compose.test.yml). Replay completes it with fail-m0." + }, + "workflow": { + "type": "rv348_handle", + "input": [], + "payload_codec": "avro", + "run_id": "01m7tfhzghsnm7xgs9xz0g5v2n", + "workflow_id": "rv307-rv348_handle-b54788", + "query": { + "name": "caught", + "arguments": [] + } + }, + "history": [ + { + "id": "01m7tfhzh1505fxjhjgnac3ztn", + "sequence": 1, + "event_type": "StartAccepted", + "namespace": "default", + "payload": { + "memo": null, + "command": { + "id": "01m7tfhzgp2fnepq5he9jhes6q", + "type": "start", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "outcome": "started_new", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "sequence": 1, + "applied_at": "2026-11-19T18:02:01.106839Z", + "accepted_at": "2026-11-19T18:02:01.106831Z", + "auth_method": "token", + "auth_status": "authorized", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfhzghsnm7xgs9xz0g5v2n", + "request_route_name": "generated::FqJuJMl97iiXlIDz", + "request_fingerprint": "sha256:121ee40454411402a1c0d919aa0184dc9c7bbd7f3bb6d0a481223855f3306280" + }, + "outcome": "started_new", + "business_key": null, + "workflow_type": "rv348_handle", + "workflow_class": "rv348_handle", + "workflow_run_id": "01m7tfhzghsnm7xgs9xz0g5v2n", + "search_attributes": null, + "visibility_labels": null, + "workflow_command_id": "01m7tfhzgp2fnepq5he9jhes6q", + "workflow_instance_id": "rv307-rv348_handle-b54788" + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfhzgp2fnepq5he9jhes6q", + "recorded_at": "2026-11-19T18:02:01.117812Z" + }, + { + "id": "01m7tfhzhd4xbby9s36djxsedb", + "sequence": 2, + "event_type": "WorkflowStarted", + "namespace": "default", + "payload": { + "command": { + "id": "01m7tfhzgp2fnepq5he9jhes6q", + "type": "start", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "outcome": "started_new", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "sequence": 1, + "applied_at": "2026-11-19T18:02:01.106839Z", + "accepted_at": "2026-11-19T18:02:01.106831Z", + "auth_method": "token", + "auth_status": "authorized", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfhzghsnm7xgs9xz0g5v2n", + "request_route_name": "generated::FqJuJMl97iiXlIDz", + "request_fingerprint": "sha256:121ee40454411402a1c0d919aa0184dc9c7bbd7f3bb6d0a481223855f3306280" + }, + "workflow_type": "rv348_handle", + "workflow_class": "rv348_handle", + "run_deadline_at": "2026-11-19T18:12:01+00:00", + "workflow_run_id": "01m7tfhzghsnm7xgs9xz0g5v2n", + "declared_queries": [ + "caught" + ], + "declared_signals": [], + "declared_updates": [ + "report" + ], + "declared_entry_mode": "canonical", + "run_timeout_seconds": 600, + "workflow_command_id": "01m7tfhzgp2fnepq5he9jhes6q", + "workflow_instance_id": "rv307-rv348_handle-b54788", + "declared_entry_method": "handle", + "execution_deadline_at": "2026-11-19T19:02:01+00:00", + "declared_query_contracts": [ + { + "name": "caught", + "parameters": [] + } + ], + "declared_signal_contracts": [], + "declared_update_contracts": [ + { + "name": "report", + "parameters": [ + { + "name": "_", + "type": null, + "default": null, + "position": 0, + "required": false, + "variadic": true, + "allows_null": true, + "default_available": false + } + ] + } + ], + "execution_timeout_seconds": 3600, + "declared_update_validators": [], + "declared_entry_declaring_class": "external:rv348_handle", + "workflow_definition_fingerprint": "sha256:69ed762d44804878ebb703a2ad385119f6f776d0dae561c15f8f84e1707a704a", + "workflow_definition_fingerprint_source": "worker" + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfhzgp2fnepq5he9jhes6q", + "recorded_at": "2026-11-19T18:02:01.131946Z" + }, + { + "id": "01m7tfj0dz708dd35zb5hzs07a", + "sequence": 3, + "event_type": "TimerScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfhzhgz2rc1r2pw3w5aek0", + "type": "workflow", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:01.228018Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:01.136331Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:11.227929Z", + "last_dispatched_at": "2026-11-19T18:02:01.144200Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:01.144200Z" + }, + "fire_at": "2026-11-19T18:02:03.042591Z", + "sequence": 1, + "timer_id": "01m7tfj0dt86gabdw3gvr6x7ss", + "delay_seconds": 1, + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + }, + "workflow_task_id": "01m7tfhzhgz2rc1r2pw3w5aek0", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:02.046739Z" + }, + { + "id": "01m7tfj0ea70c9hrb4ct35kt96", + "sequence": 4, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfhzhgz2rc1r2pw3w5aek0", + "type": "workflow", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:01.228018Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:01.136331Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:11.227929Z", + "last_dispatched_at": "2026-11-19T18:02:01.144200Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:01.144200Z" + }, + "activity": { + "id": "01m7tfj0e7d43d71dc0cn963d7", + "type": "slow", + "class": "slow", + "queue": "rv307-f39c42", + "status": "pending", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "created_at": "2026-11-19T18:02:02.055260Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfj0e7d43d71dc0cn963d7" + }, + "sequence": 2, + "activity_type": "slow", + "activity_class": "slow", + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "activity_execution_id": "01m7tfj0e7d43d71dc0cn963d7", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + }, + "workflow_task_id": "01m7tfhzhgz2rc1r2pw3w5aek0", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:02.057530Z" + }, + { + "id": "01m7tfj328eczy13j1b3eyprgy", + "sequence": 5, + "event_type": "TimerFired", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfj0e3qf9j6pk5nmt7zxyf", + "type": "timer", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:04.697890Z", + "connection": "mysql", + "lease_owner": "01m7tfj0e3qf9j6pk5nmt7zxyf", + "available_at": "2026-11-19T18:02:03.042591Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:14.697901Z" + }, + "fire_at": "2026-11-19T18:02:03.042591Z", + "fired_at": "2026-11-19T18:02:04.740400Z", + "sequence": 1, + "timer_id": "01m7tfj0dt86gabdw3gvr6x7ss", + "delay_seconds": 1, + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + }, + "workflow_task_id": "01m7tfj0e3qf9j6pk5nmt7zxyf", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:04.743320Z" + }, + { + "id": "01m7tfj32g0xfzd6d8baneqfa5", + "sequence": 6, + "event_type": "SelectionResolved", + "namespace": "default", + "payload": { + "outcome": "completed", + "member_key": "t", + "member_size": 1, + "member_index": 0, + "operation_kind": "timer", + "operation_identity": "01m7tfj0dt86gabdw3gvr6x7ss", + "selection_group_id": "select-calls:1:2", + "resolution_event_id": "01m7tfj328eczy13j1b3eyprgy", + "member_base_sequence": 1, + "selection_group_size": 2, + "resolution_event_type": "TimerFired", + "selection_group_base_sequence": 1 + }, + "workflow_task_id": null, + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:04.752074Z" + }, + { + "id": "01m7tfj3nksvadck19h10819p2", + "sequence": 7, + "event_type": "TimerScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfj32phrtt1h33xf07rszm", + "type": "workflow", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:04.844857Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:04.757674Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:14.844761Z", + "last_dispatched_at": "2026-11-19T18:02:04.793958Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:04.793958Z" + }, + "fire_at": "2026-11-19T18:02:06.359934Z", + "sequence": 3, + "timer_id": "01m7tfj3ngq7bbbwn7pbph87j1", + "delay_seconds": 1 + }, + "workflow_task_id": "01m7tfj32phrtt1h33xf07rszm", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:05.362057Z" + }, + { + "id": "01m7tfj69wrkgnj84twwz575m1", + "sequence": 8, + "event_type": "TimerFired", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfj3nq6nr10x4sj3x9gz85", + "type": "timer", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:07.974448Z", + "connection": "mysql", + "lease_owner": "01m7tfj3nq6nr10x4sj3x9gz85", + "available_at": "2026-11-19T18:02:06.359934Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:17.974454Z" + }, + "fire_at": "2026-11-19T18:02:06.359934Z", + "fired_at": "2026-11-19T18:02:08.057448Z", + "sequence": 3, + "timer_id": "01m7tfj3ngq7bbbwn7pbph87j1", + "delay_seconds": 1 + }, + "workflow_task_id": "01m7tfj3nq6nr10x4sj3x9gz85", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:08.059422Z" + }, + { + "id": "01m7tfj6x51pmsjbshkcwgry97", + "sequence": 9, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfj6a55pj0bm6br3ryrwkf", + "type": "workflow", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:08.190103Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:08.068110Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:18.190054Z", + "last_dispatched_at": "2026-11-19T18:02:08.099972Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:08.099972Z" + }, + "activity": { + "id": "01m7tfj6x1hxdjh54ks46ysn4f", + "type": "m0", + "class": "m0", + "queue": "rv307-f39c42", + "status": "pending", + "sequence": 4, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "created_at": "2026-11-19T18:02:08.674048Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfj6x1hxdjh54ks46ysn4f" + }, + "sequence": 4, + "activity_type": "m0", + "activity_class": "m0", + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfj6x1hxdjh54ks46ysn4f", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfj6a55pj0bm6br3ryrwkf", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:08.676278Z" + }, + { + "id": "01m7tfj6xcsksft6xttf9jf471", + "sequence": 10, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfj6a55pj0bm6br3ryrwkf", + "type": "workflow", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:08.190103Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:08.068110Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:18.190054Z", + "last_dispatched_at": "2026-11-19T18:02:08.099972Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:08.099972Z" + }, + "activity": { + "id": "01m7tfj6xb9j38aqpyrbd9f1wr", + "type": "m1", + "class": "m1", + "queue": "rv307-f39c42", + "status": "pending", + "sequence": 5, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "created_at": "2026-11-19T18:02:08.683384Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfj6xb9j38aqpyrbd9f1wr" + }, + "sequence": 5, + "activity_type": "m1", + "activity_class": "m1", + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfj6xb9j38aqpyrbd9f1wr", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfj6a55pj0bm6br3ryrwkf", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:08.684419Z" + }, + { + "id": "01m7tfj70bh8q54z37pfbbn4em", + "sequence": 11, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfj0edp9fwsb4ahn8rjk72", + "type": "activity", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:08.769739Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:02.060445Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:08.769940Z" + }, + "activity": { + "id": "01m7tfj0e7d43d71dc0cn963d7", + "type": "slow", + "class": "slow", + "queue": "rv307-f39c42", + "status": "running", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "attempt_id": "01M7TFJ701JSSCHXA8T2EJ54WW", + "created_at": "2026-11-19T18:02:02.055260Z", + "started_at": "2026-11-19T18:02:08.769739Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfj0e7d43d71dc0cn963d7" + }, + "sequence": 2, + "activity_type": "slow", + "activity_class": "slow", + "attempt_number": 1, + "parallel_group_id": "select-calls:1:2", + "activity_attempt_id": "01M7TFJ701JSSCHXA8T2EJ54WW", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "activity_execution_id": "01m7tfj0e7d43d71dc0cn963d7", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + }, + "workflow_task_id": "01m7tfj0edp9fwsb4ahn8rjk72", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:08.778249Z" + }, + { + "id": "01m7tfj74dn9wsdkq4sv6pk3jg", + "sequence": 12, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfj6x9vzrd4cfdwg8yarhb", + "type": "activity", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:08.899853Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:08.681145Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:08.900064Z" + }, + "activity": { + "id": "01m7tfj6x1hxdjh54ks46ysn4f", + "type": "m0", + "class": "m0", + "queue": "rv307-f39c42", + "status": "running", + "sequence": 4, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "attempt_id": "01M7TFJ7438T97B0R8W3QFT5ND", + "created_at": "2026-11-19T18:02:08.674048Z", + "started_at": "2026-11-19T18:02:08.899853Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfj6x1hxdjh54ks46ysn4f" + }, + "sequence": 4, + "activity_type": "m0", + "activity_class": "m0", + "attempt_number": 1, + "parallel_group_id": "parallel-activities:4:2", + "activity_attempt_id": "01M7TFJ7438T97B0R8W3QFT5ND", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfj6x1hxdjh54ks46ysn4f", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfj6x9vzrd4cfdwg8yarhb", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:08.908547Z" + }, + { + "id": "01m7tfj791jrvr12pggqz3j40n", + "sequence": 13, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfj6xec161vy6b32hnnnxm", + "type": "activity", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:09.051783Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:08.686369Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:09.051895Z" + }, + "activity": { + "id": "01m7tfj6xb9j38aqpyrbd9f1wr", + "type": "m1", + "class": "m1", + "queue": "rv307-f39c42", + "status": "running", + "sequence": 5, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "attempt_id": "01M7TFJ78VEQGP56SB5GFV2X3T", + "created_at": "2026-11-19T18:02:08.683384Z", + "started_at": "2026-11-19T18:02:09.051783Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfj6xb9j38aqpyrbd9f1wr" + }, + "sequence": 5, + "activity_type": "m1", + "activity_class": "m1", + "attempt_number": 1, + "parallel_group_id": "parallel-activities:4:2", + "activity_attempt_id": "01M7TFJ78VEQGP56SB5GFV2X3T", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfj6xb9j38aqpyrbd9f1wr", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfj6xec161vy6b32hnnnxm", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:09.056688Z" + }, + { + "id": "01m7tfj7cw2nkt972a6r7jf0rv", + "sequence": 14, + "event_type": "ActivityFailed", + "namespace": "default", + "payload": { + "code": 0, + "task": { + "id": "01m7tfj6xec161vy6b32hnnnxm", + "type": "activity", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:09.051783Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:08.686369Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:09.133724Z" + }, + "message": "fail-m1", + "activity": { + "id": "01m7tfj6xb9j38aqpyrbd9f1wr", + "type": "m1", + "class": "m1", + "queue": "rv307-f39c42", + "status": "failed", + "sequence": 5, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "closed_at": "2026-11-19T18:02:09.177030Z", + "exception": { + "code": 0, + "type": null, + "message": "fail-m1", + "non_retryable": true + }, + "attempt_id": "01M7TFJ78VEQGP56SB5GFV2X3T", + "created_at": "2026-11-19T18:02:08.683384Z", + "started_at": "2026-11-19T18:02:09.051783Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfj6xb9j38aqpyrbd9f1wr" + }, + "sequence": 5, + "exception": { + "type": null, + "message": "fail-m1", + "code": 0, + "non_retryable": true + }, + "failure_id": "01m7tfj7cr91jf20g5befgd5gt", + "activity_type": "m1", + "non_retryable": true, + "activity_class": "m1", + "attempt_number": 1, + "exception_type": null, + "exception_class": "RuntimeException", + "failure_category": "activity", + "parallel_group_id": "parallel-activities:4:2", + "activity_attempt_id": "01M7TFJ78VEQGP56SB5GFV2X3T", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfj6xb9j38aqpyrbd9f1wr", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfj6xec161vy6b32hnnnxm", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:09.179695Z" + }, + { + "id": "01m7tfj8p67khhm646dm2z939w", + "sequence": 15, + "event_type": "ActivityFailed", + "namespace": "default", + "payload": { + "code": 0, + "task": { + "id": "01m7tfj6x9vzrd4cfdwg8yarhb", + "type": "activity", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:08.899853Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:08.681145Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:09.007058Z" + }, + "message": "fail-m0", + "activity": { + "id": "01m7tfj6x1hxdjh54ks46ysn4f", + "type": "m0", + "class": "m0", + "queue": "rv307-f39c42", + "status": "failed", + "sequence": 4, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "closed_at": "2026-11-19T18:02:10.500387Z", + "exception": { + "code": 0, + "type": null, + "message": "fail-m0", + "non_retryable": true + }, + "attempt_id": "01M7TFJ7438T97B0R8W3QFT5ND", + "created_at": "2026-11-19T18:02:08.674048Z", + "started_at": "2026-11-19T18:02:08.899853Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfj6x1hxdjh54ks46ysn4f" + }, + "sequence": 4, + "exception": { + "type": null, + "message": "fail-m0", + "code": 0, + "non_retryable": true + }, + "failure_id": "01m7tfj8p375ctwss26hg6hpas", + "activity_type": "m0", + "non_retryable": true, + "activity_class": "m0", + "attempt_number": 1, + "exception_type": null, + "exception_class": "RuntimeException", + "failure_category": "activity", + "parallel_group_id": "parallel-activities:4:2", + "activity_attempt_id": "01M7TFJ7438T97B0R8W3QFT5ND", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfj6x1hxdjh54ks46ysn4f", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfj6x9vzrd4cfdwg8yarhb", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:10.501792Z" + }, + { + "id": "01m7tfj9sy5cdrrveet9chckqd", + "sequence": 16, + "event_type": "ActivityCompleted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfj0edp9fwsb4ahn8rjk72", + "type": "activity", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:08.769739Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:02.060445Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:08.840081Z" + }, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoEb2s=" + }, + "activity": { + "id": "01m7tfj0e7d43d71dc0cn963d7", + "type": "slow", + "class": "slow", + "queue": "rv307-f39c42", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoEb2s=" + }, + "status": "completed", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "closed_at": "2026-11-19T18:02:11.644584Z", + "attempt_id": "01M7TFJ701JSSCHXA8T2EJ54WW", + "created_at": "2026-11-19T18:02:02.055260Z", + "started_at": "2026-11-19T18:02:08.769739Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfj0e7d43d71dc0cn963d7" + }, + "sequence": 2, + "activity_type": "slow", + "payload_codec": "avro", + "activity_class": "slow", + "attempt_number": 1, + "parallel_group_id": "select-calls:1:2", + "activity_attempt_id": "01M7TFJ701JSSCHXA8T2EJ54WW", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "activity_execution_id": "01m7tfj0e7d43d71dc0cn963d7", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + }, + "workflow_task_id": "01m7tfj0edp9fwsb4ahn8rjk72", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:11.646013Z" + }, + { + "id": "01m7tfjad2fsrgfm3fxjjhzeh4", + "sequence": 17, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfj8pb9rxyy5wekr276xp6", + "type": "workflow", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:11.804295Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:10.507428Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:21.804236Z", + "last_dispatched_at": "2026-11-19T18:02:10.557382Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:10.557382Z" + }, + "activity": { + "id": "01m7tfjad0h5zvqbmbf7v51fca", + "type": "compensate", + "class": "compensate", + "queue": "rv307-f39c42", + "status": "pending", + "sequence": 6, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCCg5mYWlsLW0wAA==" + }, + "created_at": "2026-11-19T18:02:12.256587Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfjad0h5zvqbmbf7v51fca" + }, + "sequence": 6, + "activity_type": "compensate", + "activity_class": "compensate", + "activity_execution_id": "01m7tfjad0h5zvqbmbf7v51fca" + }, + "workflow_task_id": "01m7tfj8pb9rxyy5wekr276xp6", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:12.257854Z" + }, + { + "id": "01m7tfjchb7bmw31062f6ejg1j", + "sequence": 18, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfjad43s5v835shxn4cyj9", + "type": "activity", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:14.433591Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:12.260565Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:14.433724Z" + }, + "activity": { + "id": "01m7tfjad0h5zvqbmbf7v51fca", + "type": "compensate", + "class": "compensate", + "queue": "rv307-f39c42", + "status": "running", + "sequence": 6, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCCg5mYWlsLW0wAA==" + }, + "attempt_id": "01M7TFJCH1T7S3VBEXDPSJ19K3", + "created_at": "2026-11-19T18:02:12.256587Z", + "started_at": "2026-11-19T18:02:14.433591Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfjad0h5zvqbmbf7v51fca" + }, + "sequence": 6, + "activity_type": "compensate", + "activity_class": "compensate", + "attempt_number": 1, + "activity_attempt_id": "01M7TFJCH1T7S3VBEXDPSJ19K3", + "activity_execution_id": "01m7tfjad0h5zvqbmbf7v51fca" + }, + "workflow_task_id": "01m7tfjad43s5v835shxn4cyj9", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:14.442173Z" + }, + { + "id": "01m7tfjcnbsgfhx31cb3qevy4s", + "sequence": 19, + "event_type": "ActivityCompleted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfjad43s5v835shxn4cyj9", + "type": "activity", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:14.433591Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:12.260565Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:14.528281Z" + }, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoMdW5kb25l" + }, + "activity": { + "id": "01m7tfjad0h5zvqbmbf7v51fca", + "type": "compensate", + "class": "compensate", + "queue": "rv307-f39c42", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoMdW5kb25l" + }, + "status": "completed", + "sequence": 6, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCCg5mYWlsLW0wAA==" + }, + "closed_at": "2026-11-19T18:02:14.569395Z", + "attempt_id": "01M7TFJCH1T7S3VBEXDPSJ19K3", + "created_at": "2026-11-19T18:02:12.256587Z", + "started_at": "2026-11-19T18:02:14.433591Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfjad0h5zvqbmbf7v51fca" + }, + "sequence": 6, + "activity_type": "compensate", + "payload_codec": "avro", + "activity_class": "compensate", + "attempt_number": 1, + "activity_attempt_id": "01M7TFJCH1T7S3VBEXDPSJ19K3", + "activity_execution_id": "01m7tfjad0h5zvqbmbf7v51fca" + }, + "workflow_task_id": "01m7tfjad43s5v835shxn4cyj9", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:14.571009Z" + } + ], + "expected": { + "query_result": "fail-m0" + } +} diff --git a/tests/fixtures/replay_regressions/failure-order-released-zero-command-catch-complete.json b/tests/fixtures/replay_regressions/failure-order-released-zero-command-catch-complete.json new file mode 100644 index 00000000..98d020e7 --- /dev/null +++ b/tests/fixtures/replay_regressions/failure-order-released-zero-command-catch-complete.json @@ -0,0 +1,1292 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "failure-order-released-zero-command-catch-complete", + "protocol_version": "1.20", + "bindings": [ + "python" + ], + "parallel_failure_order": { + "issues": [ + "durable-workflow/sdk-python#307" + ], + "note": "The history the last task of the 2.19.1 run of failure-order-released-zero-command-catch received on a Server built from durable-workflow/server main 52e3ec00 (docker-compose.test.yml). Replay completes it with fail-m0." + }, + "workflow": { + "type": "rv348_handle", + "input": [], + "payload_codec": "avro", + "run_id": "01m7tfhzghsnm7xgs9xz0g5v2n", + "workflow_id": "rv307-rv348_handle-b54788" + }, + "history": [ + { + "id": "01m7tfhzh1505fxjhjgnac3ztn", + "sequence": 1, + "event_type": "StartAccepted", + "namespace": "default", + "payload": { + "memo": null, + "command": { + "id": "01m7tfhzgp2fnepq5he9jhes6q", + "type": "start", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "outcome": "started_new", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "sequence": 1, + "applied_at": "2026-11-19T18:02:01.106839Z", + "accepted_at": "2026-11-19T18:02:01.106831Z", + "auth_method": "token", + "auth_status": "authorized", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfhzghsnm7xgs9xz0g5v2n", + "request_route_name": "generated::FqJuJMl97iiXlIDz", + "request_fingerprint": "sha256:121ee40454411402a1c0d919aa0184dc9c7bbd7f3bb6d0a481223855f3306280" + }, + "outcome": "started_new", + "business_key": null, + "workflow_type": "rv348_handle", + "workflow_class": "rv348_handle", + "workflow_run_id": "01m7tfhzghsnm7xgs9xz0g5v2n", + "search_attributes": null, + "visibility_labels": null, + "workflow_command_id": "01m7tfhzgp2fnepq5he9jhes6q", + "workflow_instance_id": "rv307-rv348_handle-b54788" + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfhzgp2fnepq5he9jhes6q", + "recorded_at": "2026-11-19T18:02:01.117812Z" + }, + { + "id": "01m7tfhzhd4xbby9s36djxsedb", + "sequence": 2, + "event_type": "WorkflowStarted", + "namespace": "default", + "payload": { + "command": { + "id": "01m7tfhzgp2fnepq5he9jhes6q", + "type": "start", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "outcome": "started_new", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "sequence": 1, + "applied_at": "2026-11-19T18:02:01.106839Z", + "accepted_at": "2026-11-19T18:02:01.106831Z", + "auth_method": "token", + "auth_status": "authorized", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfhzghsnm7xgs9xz0g5v2n", + "request_route_name": "generated::FqJuJMl97iiXlIDz", + "request_fingerprint": "sha256:121ee40454411402a1c0d919aa0184dc9c7bbd7f3bb6d0a481223855f3306280" + }, + "workflow_type": "rv348_handle", + "workflow_class": "rv348_handle", + "run_deadline_at": "2026-11-19T18:12:01+00:00", + "workflow_run_id": "01m7tfhzghsnm7xgs9xz0g5v2n", + "declared_queries": [ + "caught" + ], + "declared_signals": [], + "declared_updates": [ + "report" + ], + "declared_entry_mode": "canonical", + "run_timeout_seconds": 600, + "workflow_command_id": "01m7tfhzgp2fnepq5he9jhes6q", + "workflow_instance_id": "rv307-rv348_handle-b54788", + "declared_entry_method": "handle", + "execution_deadline_at": "2026-11-19T19:02:01+00:00", + "declared_query_contracts": [ + { + "name": "caught", + "parameters": [] + } + ], + "declared_signal_contracts": [], + "declared_update_contracts": [ + { + "name": "report", + "parameters": [ + { + "name": "_", + "type": null, + "default": null, + "position": 0, + "required": false, + "variadic": true, + "allows_null": true, + "default_available": false + } + ] + } + ], + "execution_timeout_seconds": 3600, + "declared_update_validators": [], + "declared_entry_declaring_class": "external:rv348_handle", + "workflow_definition_fingerprint": "sha256:69ed762d44804878ebb703a2ad385119f6f776d0dae561c15f8f84e1707a704a", + "workflow_definition_fingerprint_source": "worker" + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfhzgp2fnepq5he9jhes6q", + "recorded_at": "2026-11-19T18:02:01.131946Z" + }, + { + "id": "01m7tfj0dz708dd35zb5hzs07a", + "sequence": 3, + "event_type": "TimerScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfhzhgz2rc1r2pw3w5aek0", + "type": "workflow", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:01.228018Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:01.136331Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:11.227929Z", + "last_dispatched_at": "2026-11-19T18:02:01.144200Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:01.144200Z" + }, + "fire_at": "2026-11-19T18:02:03.042591Z", + "sequence": 1, + "timer_id": "01m7tfj0dt86gabdw3gvr6x7ss", + "delay_seconds": 1, + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + }, + "workflow_task_id": "01m7tfhzhgz2rc1r2pw3w5aek0", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:02.046739Z" + }, + { + "id": "01m7tfj0ea70c9hrb4ct35kt96", + "sequence": 4, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfhzhgz2rc1r2pw3w5aek0", + "type": "workflow", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:01.228018Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:01.136331Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:11.227929Z", + "last_dispatched_at": "2026-11-19T18:02:01.144200Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:01.144200Z" + }, + "activity": { + "id": "01m7tfj0e7d43d71dc0cn963d7", + "type": "slow", + "class": "slow", + "queue": "rv307-f39c42", + "status": "pending", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "created_at": "2026-11-19T18:02:02.055260Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfj0e7d43d71dc0cn963d7" + }, + "sequence": 2, + "activity_type": "slow", + "activity_class": "slow", + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "activity_execution_id": "01m7tfj0e7d43d71dc0cn963d7", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + }, + "workflow_task_id": "01m7tfhzhgz2rc1r2pw3w5aek0", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:02.057530Z" + }, + { + "id": "01m7tfj328eczy13j1b3eyprgy", + "sequence": 5, + "event_type": "TimerFired", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfj0e3qf9j6pk5nmt7zxyf", + "type": "timer", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:04.697890Z", + "connection": "mysql", + "lease_owner": "01m7tfj0e3qf9j6pk5nmt7zxyf", + "available_at": "2026-11-19T18:02:03.042591Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:14.697901Z" + }, + "fire_at": "2026-11-19T18:02:03.042591Z", + "fired_at": "2026-11-19T18:02:04.740400Z", + "sequence": 1, + "timer_id": "01m7tfj0dt86gabdw3gvr6x7ss", + "delay_seconds": 1, + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + }, + "workflow_task_id": "01m7tfj0e3qf9j6pk5nmt7zxyf", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:04.743320Z" + }, + { + "id": "01m7tfj32g0xfzd6d8baneqfa5", + "sequence": 6, + "event_type": "SelectionResolved", + "namespace": "default", + "payload": { + "outcome": "completed", + "member_key": "t", + "member_size": 1, + "member_index": 0, + "operation_kind": "timer", + "operation_identity": "01m7tfj0dt86gabdw3gvr6x7ss", + "selection_group_id": "select-calls:1:2", + "resolution_event_id": "01m7tfj328eczy13j1b3eyprgy", + "member_base_sequence": 1, + "selection_group_size": 2, + "resolution_event_type": "TimerFired", + "selection_group_base_sequence": 1 + }, + "workflow_task_id": null, + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:04.752074Z" + }, + { + "id": "01m7tfj3nksvadck19h10819p2", + "sequence": 7, + "event_type": "TimerScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfj32phrtt1h33xf07rszm", + "type": "workflow", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:04.844857Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:04.757674Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:14.844761Z", + "last_dispatched_at": "2026-11-19T18:02:04.793958Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:04.793958Z" + }, + "fire_at": "2026-11-19T18:02:06.359934Z", + "sequence": 3, + "timer_id": "01m7tfj3ngq7bbbwn7pbph87j1", + "delay_seconds": 1 + }, + "workflow_task_id": "01m7tfj32phrtt1h33xf07rszm", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:05.362057Z" + }, + { + "id": "01m7tfj69wrkgnj84twwz575m1", + "sequence": 8, + "event_type": "TimerFired", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfj3nq6nr10x4sj3x9gz85", + "type": "timer", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:07.974448Z", + "connection": "mysql", + "lease_owner": "01m7tfj3nq6nr10x4sj3x9gz85", + "available_at": "2026-11-19T18:02:06.359934Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:17.974454Z" + }, + "fire_at": "2026-11-19T18:02:06.359934Z", + "fired_at": "2026-11-19T18:02:08.057448Z", + "sequence": 3, + "timer_id": "01m7tfj3ngq7bbbwn7pbph87j1", + "delay_seconds": 1 + }, + "workflow_task_id": "01m7tfj3nq6nr10x4sj3x9gz85", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:08.059422Z" + }, + { + "id": "01m7tfj6x51pmsjbshkcwgry97", + "sequence": 9, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfj6a55pj0bm6br3ryrwkf", + "type": "workflow", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:08.190103Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:08.068110Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:18.190054Z", + "last_dispatched_at": "2026-11-19T18:02:08.099972Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:08.099972Z" + }, + "activity": { + "id": "01m7tfj6x1hxdjh54ks46ysn4f", + "type": "m0", + "class": "m0", + "queue": "rv307-f39c42", + "status": "pending", + "sequence": 4, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "created_at": "2026-11-19T18:02:08.674048Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfj6x1hxdjh54ks46ysn4f" + }, + "sequence": 4, + "activity_type": "m0", + "activity_class": "m0", + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfj6x1hxdjh54ks46ysn4f", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfj6a55pj0bm6br3ryrwkf", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:08.676278Z" + }, + { + "id": "01m7tfj6xcsksft6xttf9jf471", + "sequence": 10, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfj6a55pj0bm6br3ryrwkf", + "type": "workflow", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:08.190103Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:08.068110Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:18.190054Z", + "last_dispatched_at": "2026-11-19T18:02:08.099972Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:08.099972Z" + }, + "activity": { + "id": "01m7tfj6xb9j38aqpyrbd9f1wr", + "type": "m1", + "class": "m1", + "queue": "rv307-f39c42", + "status": "pending", + "sequence": 5, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "created_at": "2026-11-19T18:02:08.683384Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfj6xb9j38aqpyrbd9f1wr" + }, + "sequence": 5, + "activity_type": "m1", + "activity_class": "m1", + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfj6xb9j38aqpyrbd9f1wr", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfj6a55pj0bm6br3ryrwkf", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:08.684419Z" + }, + { + "id": "01m7tfj70bh8q54z37pfbbn4em", + "sequence": 11, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfj0edp9fwsb4ahn8rjk72", + "type": "activity", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:08.769739Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:02.060445Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:08.769940Z" + }, + "activity": { + "id": "01m7tfj0e7d43d71dc0cn963d7", + "type": "slow", + "class": "slow", + "queue": "rv307-f39c42", + "status": "running", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "attempt_id": "01M7TFJ701JSSCHXA8T2EJ54WW", + "created_at": "2026-11-19T18:02:02.055260Z", + "started_at": "2026-11-19T18:02:08.769739Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfj0e7d43d71dc0cn963d7" + }, + "sequence": 2, + "activity_type": "slow", + "activity_class": "slow", + "attempt_number": 1, + "parallel_group_id": "select-calls:1:2", + "activity_attempt_id": "01M7TFJ701JSSCHXA8T2EJ54WW", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "activity_execution_id": "01m7tfj0e7d43d71dc0cn963d7", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + }, + "workflow_task_id": "01m7tfj0edp9fwsb4ahn8rjk72", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:08.778249Z" + }, + { + "id": "01m7tfj74dn9wsdkq4sv6pk3jg", + "sequence": 12, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfj6x9vzrd4cfdwg8yarhb", + "type": "activity", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:08.899853Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:08.681145Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:08.900064Z" + }, + "activity": { + "id": "01m7tfj6x1hxdjh54ks46ysn4f", + "type": "m0", + "class": "m0", + "queue": "rv307-f39c42", + "status": "running", + "sequence": 4, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "attempt_id": "01M7TFJ7438T97B0R8W3QFT5ND", + "created_at": "2026-11-19T18:02:08.674048Z", + "started_at": "2026-11-19T18:02:08.899853Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfj6x1hxdjh54ks46ysn4f" + }, + "sequence": 4, + "activity_type": "m0", + "activity_class": "m0", + "attempt_number": 1, + "parallel_group_id": "parallel-activities:4:2", + "activity_attempt_id": "01M7TFJ7438T97B0R8W3QFT5ND", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfj6x1hxdjh54ks46ysn4f", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfj6x9vzrd4cfdwg8yarhb", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:08.908547Z" + }, + { + "id": "01m7tfj791jrvr12pggqz3j40n", + "sequence": 13, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfj6xec161vy6b32hnnnxm", + "type": "activity", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:09.051783Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:08.686369Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:09.051895Z" + }, + "activity": { + "id": "01m7tfj6xb9j38aqpyrbd9f1wr", + "type": "m1", + "class": "m1", + "queue": "rv307-f39c42", + "status": "running", + "sequence": 5, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "attempt_id": "01M7TFJ78VEQGP56SB5GFV2X3T", + "created_at": "2026-11-19T18:02:08.683384Z", + "started_at": "2026-11-19T18:02:09.051783Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfj6xb9j38aqpyrbd9f1wr" + }, + "sequence": 5, + "activity_type": "m1", + "activity_class": "m1", + "attempt_number": 1, + "parallel_group_id": "parallel-activities:4:2", + "activity_attempt_id": "01M7TFJ78VEQGP56SB5GFV2X3T", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfj6xb9j38aqpyrbd9f1wr", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfj6xec161vy6b32hnnnxm", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:09.056688Z" + }, + { + "id": "01m7tfj7cw2nkt972a6r7jf0rv", + "sequence": 14, + "event_type": "ActivityFailed", + "namespace": "default", + "payload": { + "code": 0, + "task": { + "id": "01m7tfj6xec161vy6b32hnnnxm", + "type": "activity", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:09.051783Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:08.686369Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:09.133724Z" + }, + "message": "fail-m1", + "activity": { + "id": "01m7tfj6xb9j38aqpyrbd9f1wr", + "type": "m1", + "class": "m1", + "queue": "rv307-f39c42", + "status": "failed", + "sequence": 5, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "closed_at": "2026-11-19T18:02:09.177030Z", + "exception": { + "code": 0, + "type": null, + "message": "fail-m1", + "non_retryable": true + }, + "attempt_id": "01M7TFJ78VEQGP56SB5GFV2X3T", + "created_at": "2026-11-19T18:02:08.683384Z", + "started_at": "2026-11-19T18:02:09.051783Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfj6xb9j38aqpyrbd9f1wr" + }, + "sequence": 5, + "exception": { + "type": null, + "message": "fail-m1", + "code": 0, + "non_retryable": true + }, + "failure_id": "01m7tfj7cr91jf20g5befgd5gt", + "activity_type": "m1", + "non_retryable": true, + "activity_class": "m1", + "attempt_number": 1, + "exception_type": null, + "exception_class": "RuntimeException", + "failure_category": "activity", + "parallel_group_id": "parallel-activities:4:2", + "activity_attempt_id": "01M7TFJ78VEQGP56SB5GFV2X3T", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfj6xb9j38aqpyrbd9f1wr", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfj6xec161vy6b32hnnnxm", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:09.179695Z" + }, + { + "id": "01m7tfj8p67khhm646dm2z939w", + "sequence": 15, + "event_type": "ActivityFailed", + "namespace": "default", + "payload": { + "code": 0, + "task": { + "id": "01m7tfj6x9vzrd4cfdwg8yarhb", + "type": "activity", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:08.899853Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:08.681145Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:09.007058Z" + }, + "message": "fail-m0", + "activity": { + "id": "01m7tfj6x1hxdjh54ks46ysn4f", + "type": "m0", + "class": "m0", + "queue": "rv307-f39c42", + "status": "failed", + "sequence": 4, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "closed_at": "2026-11-19T18:02:10.500387Z", + "exception": { + "code": 0, + "type": null, + "message": "fail-m0", + "non_retryable": true + }, + "attempt_id": "01M7TFJ7438T97B0R8W3QFT5ND", + "created_at": "2026-11-19T18:02:08.674048Z", + "started_at": "2026-11-19T18:02:08.899853Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfj6x1hxdjh54ks46ysn4f" + }, + "sequence": 4, + "exception": { + "type": null, + "message": "fail-m0", + "code": 0, + "non_retryable": true + }, + "failure_id": "01m7tfj8p375ctwss26hg6hpas", + "activity_type": "m0", + "non_retryable": true, + "activity_class": "m0", + "attempt_number": 1, + "exception_type": null, + "exception_class": "RuntimeException", + "failure_category": "activity", + "parallel_group_id": "parallel-activities:4:2", + "activity_attempt_id": "01M7TFJ7438T97B0R8W3QFT5ND", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfj6x1hxdjh54ks46ysn4f", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfj6x9vzrd4cfdwg8yarhb", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:10.501792Z" + }, + { + "id": "01m7tfj9sy5cdrrveet9chckqd", + "sequence": 16, + "event_type": "ActivityCompleted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfj0edp9fwsb4ahn8rjk72", + "type": "activity", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:08.769739Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:02.060445Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:08.840081Z" + }, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoEb2s=" + }, + "activity": { + "id": "01m7tfj0e7d43d71dc0cn963d7", + "type": "slow", + "class": "slow", + "queue": "rv307-f39c42", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoEb2s=" + }, + "status": "completed", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "closed_at": "2026-11-19T18:02:11.644584Z", + "attempt_id": "01M7TFJ701JSSCHXA8T2EJ54WW", + "created_at": "2026-11-19T18:02:02.055260Z", + "started_at": "2026-11-19T18:02:08.769739Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfj0e7d43d71dc0cn963d7" + }, + "sequence": 2, + "activity_type": "slow", + "payload_codec": "avro", + "activity_class": "slow", + "attempt_number": 1, + "parallel_group_id": "select-calls:1:2", + "activity_attempt_id": "01M7TFJ701JSSCHXA8T2EJ54WW", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "activity_execution_id": "01m7tfj0e7d43d71dc0cn963d7", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + }, + "workflow_task_id": "01m7tfj0edp9fwsb4ahn8rjk72", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:11.646013Z" + }, + { + "id": "01m7tfjad2fsrgfm3fxjjhzeh4", + "sequence": 17, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfj8pb9rxyy5wekr276xp6", + "type": "workflow", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:11.804295Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:10.507428Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:21.804236Z", + "last_dispatched_at": "2026-11-19T18:02:10.557382Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:10.557382Z" + }, + "activity": { + "id": "01m7tfjad0h5zvqbmbf7v51fca", + "type": "compensate", + "class": "compensate", + "queue": "rv307-f39c42", + "status": "pending", + "sequence": 6, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCCg5mYWlsLW0wAA==" + }, + "created_at": "2026-11-19T18:02:12.256587Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfjad0h5zvqbmbf7v51fca" + }, + "sequence": 6, + "activity_type": "compensate", + "activity_class": "compensate", + "activity_execution_id": "01m7tfjad0h5zvqbmbf7v51fca" + }, + "workflow_task_id": "01m7tfj8pb9rxyy5wekr276xp6", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:12.257854Z" + }, + { + "id": "01m7tfjchb7bmw31062f6ejg1j", + "sequence": 18, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfjad43s5v835shxn4cyj9", + "type": "activity", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:14.433591Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:12.260565Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:14.433724Z" + }, + "activity": { + "id": "01m7tfjad0h5zvqbmbf7v51fca", + "type": "compensate", + "class": "compensate", + "queue": "rv307-f39c42", + "status": "running", + "sequence": 6, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCCg5mYWlsLW0wAA==" + }, + "attempt_id": "01M7TFJCH1T7S3VBEXDPSJ19K3", + "created_at": "2026-11-19T18:02:12.256587Z", + "started_at": "2026-11-19T18:02:14.433591Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfjad0h5zvqbmbf7v51fca" + }, + "sequence": 6, + "activity_type": "compensate", + "activity_class": "compensate", + "attempt_number": 1, + "activity_attempt_id": "01M7TFJCH1T7S3VBEXDPSJ19K3", + "activity_execution_id": "01m7tfjad0h5zvqbmbf7v51fca" + }, + "workflow_task_id": "01m7tfjad43s5v835shxn4cyj9", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:14.442173Z" + }, + { + "id": "01m7tfjcnbsgfhx31cb3qevy4s", + "sequence": 19, + "event_type": "ActivityCompleted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfjad43s5v835shxn4cyj9", + "type": "activity", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:14.433591Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:12.260565Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:14.528281Z" + }, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoMdW5kb25l" + }, + "activity": { + "id": "01m7tfjad0h5zvqbmbf7v51fca", + "type": "compensate", + "class": "compensate", + "queue": "rv307-f39c42", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoMdW5kb25l" + }, + "status": "completed", + "sequence": 6, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCCg5mYWlsLW0wAA==" + }, + "closed_at": "2026-11-19T18:02:14.569395Z", + "attempt_id": "01M7TFJCH1T7S3VBEXDPSJ19K3", + "created_at": "2026-11-19T18:02:12.256587Z", + "started_at": "2026-11-19T18:02:14.433591Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfjad0h5zvqbmbf7v51fca" + }, + "sequence": 6, + "activity_type": "compensate", + "payload_codec": "avro", + "activity_class": "compensate", + "attempt_number": 1, + "activity_attempt_id": "01M7TFJCH1T7S3VBEXDPSJ19K3", + "activity_execution_id": "01m7tfjad0h5zvqbmbf7v51fca" + }, + "workflow_task_id": "01m7tfjad43s5v835shxn4cyj9", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:14.571009Z" + } + ], + "expected": { + "command_sequence": [ + { + "type": "complete_workflow" + } + ] + } +} diff --git a/tests/fixtures/replay_regressions/failure-order-released-zero-command-catch-query.json b/tests/fixtures/replay_regressions/failure-order-released-zero-command-catch-query.json new file mode 100644 index 00000000..3b94624a --- /dev/null +++ b/tests/fixtures/replay_regressions/failure-order-released-zero-command-catch-query.json @@ -0,0 +1,1107 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "failure-order-released-zero-command-catch-query", + "protocol_version": "1.20", + "bindings": [ + "python" + ], + "parallel_failure_order": { + "issues": [ + "durable-workflow/sdk-python#307" + ], + "note": "Recorded by published 2.19.1 on a Server built from durable-workflow/server main 52e3ec00 (docker-compose.test.yml). The group (m0, m1) was scheduled without the failure-order marker. m1 failed, then m0. The task that saw the settled group caught fail-m0 (lowest position) and then awaited a select() handle, so it recorded no command. This is the history the next task receives after the handle completes: it keeps fail-m0 and compensates it, and does not take the empty frontier after the group for a new decision." + }, + "workflow": { + "type": "rv348_handle", + "input": [], + "payload_codec": "avro", + "run_id": "01m7tfhzghsnm7xgs9xz0g5v2n", + "workflow_id": "rv307-rv348_handle-b54788", + "query": { + "name": "caught", + "arguments": [] + } + }, + "history": [ + { + "id": "01m7tfhzh1505fxjhjgnac3ztn", + "sequence": 1, + "event_type": "StartAccepted", + "namespace": "default", + "payload": { + "memo": null, + "command": { + "id": "01m7tfhzgp2fnepq5he9jhes6q", + "type": "start", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "outcome": "started_new", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "sequence": 1, + "applied_at": "2026-11-19T18:02:01.106839Z", + "accepted_at": "2026-11-19T18:02:01.106831Z", + "auth_method": "token", + "auth_status": "authorized", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfhzghsnm7xgs9xz0g5v2n", + "request_route_name": "generated::FqJuJMl97iiXlIDz", + "request_fingerprint": "sha256:121ee40454411402a1c0d919aa0184dc9c7bbd7f3bb6d0a481223855f3306280" + }, + "outcome": "started_new", + "business_key": null, + "workflow_type": "rv348_handle", + "workflow_class": "rv348_handle", + "workflow_run_id": "01m7tfhzghsnm7xgs9xz0g5v2n", + "search_attributes": null, + "visibility_labels": null, + "workflow_command_id": "01m7tfhzgp2fnepq5he9jhes6q", + "workflow_instance_id": "rv307-rv348_handle-b54788" + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfhzgp2fnepq5he9jhes6q", + "recorded_at": "2026-11-19T18:02:01.117812Z" + }, + { + "id": "01m7tfhzhd4xbby9s36djxsedb", + "sequence": 2, + "event_type": "WorkflowStarted", + "namespace": "default", + "payload": { + "command": { + "id": "01m7tfhzgp2fnepq5he9jhes6q", + "type": "start", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "outcome": "started_new", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "sequence": 1, + "applied_at": "2026-11-19T18:02:01.106839Z", + "accepted_at": "2026-11-19T18:02:01.106831Z", + "auth_method": "token", + "auth_status": "authorized", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfhzghsnm7xgs9xz0g5v2n", + "request_route_name": "generated::FqJuJMl97iiXlIDz", + "request_fingerprint": "sha256:121ee40454411402a1c0d919aa0184dc9c7bbd7f3bb6d0a481223855f3306280" + }, + "workflow_type": "rv348_handle", + "workflow_class": "rv348_handle", + "run_deadline_at": "2026-11-19T18:12:01+00:00", + "workflow_run_id": "01m7tfhzghsnm7xgs9xz0g5v2n", + "declared_queries": [ + "caught" + ], + "declared_signals": [], + "declared_updates": [ + "report" + ], + "declared_entry_mode": "canonical", + "run_timeout_seconds": 600, + "workflow_command_id": "01m7tfhzgp2fnepq5he9jhes6q", + "workflow_instance_id": "rv307-rv348_handle-b54788", + "declared_entry_method": "handle", + "execution_deadline_at": "2026-11-19T19:02:01+00:00", + "declared_query_contracts": [ + { + "name": "caught", + "parameters": [] + } + ], + "declared_signal_contracts": [], + "declared_update_contracts": [ + { + "name": "report", + "parameters": [ + { + "name": "_", + "type": null, + "default": null, + "position": 0, + "required": false, + "variadic": true, + "allows_null": true, + "default_available": false + } + ] + } + ], + "execution_timeout_seconds": 3600, + "declared_update_validators": [], + "declared_entry_declaring_class": "external:rv348_handle", + "workflow_definition_fingerprint": "sha256:69ed762d44804878ebb703a2ad385119f6f776d0dae561c15f8f84e1707a704a", + "workflow_definition_fingerprint_source": "worker" + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfhzgp2fnepq5he9jhes6q", + "recorded_at": "2026-11-19T18:02:01.131946Z" + }, + { + "id": "01m7tfj0dz708dd35zb5hzs07a", + "sequence": 3, + "event_type": "TimerScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfhzhgz2rc1r2pw3w5aek0", + "type": "workflow", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:01.228018Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:01.136331Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:11.227929Z", + "last_dispatched_at": "2026-11-19T18:02:01.144200Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:01.144200Z" + }, + "fire_at": "2026-11-19T18:02:03.042591Z", + "sequence": 1, + "timer_id": "01m7tfj0dt86gabdw3gvr6x7ss", + "delay_seconds": 1, + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + }, + "workflow_task_id": "01m7tfhzhgz2rc1r2pw3w5aek0", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:02.046739Z" + }, + { + "id": "01m7tfj0ea70c9hrb4ct35kt96", + "sequence": 4, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfhzhgz2rc1r2pw3w5aek0", + "type": "workflow", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:01.228018Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:01.136331Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:11.227929Z", + "last_dispatched_at": "2026-11-19T18:02:01.144200Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:01.144200Z" + }, + "activity": { + "id": "01m7tfj0e7d43d71dc0cn963d7", + "type": "slow", + "class": "slow", + "queue": "rv307-f39c42", + "status": "pending", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "created_at": "2026-11-19T18:02:02.055260Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfj0e7d43d71dc0cn963d7" + }, + "sequence": 2, + "activity_type": "slow", + "activity_class": "slow", + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "activity_execution_id": "01m7tfj0e7d43d71dc0cn963d7", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + }, + "workflow_task_id": "01m7tfhzhgz2rc1r2pw3w5aek0", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:02.057530Z" + }, + { + "id": "01m7tfj328eczy13j1b3eyprgy", + "sequence": 5, + "event_type": "TimerFired", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfj0e3qf9j6pk5nmt7zxyf", + "type": "timer", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:04.697890Z", + "connection": "mysql", + "lease_owner": "01m7tfj0e3qf9j6pk5nmt7zxyf", + "available_at": "2026-11-19T18:02:03.042591Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:14.697901Z" + }, + "fire_at": "2026-11-19T18:02:03.042591Z", + "fired_at": "2026-11-19T18:02:04.740400Z", + "sequence": 1, + "timer_id": "01m7tfj0dt86gabdw3gvr6x7ss", + "delay_seconds": 1, + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + }, + "workflow_task_id": "01m7tfj0e3qf9j6pk5nmt7zxyf", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:04.743320Z" + }, + { + "id": "01m7tfj32g0xfzd6d8baneqfa5", + "sequence": 6, + "event_type": "SelectionResolved", + "namespace": "default", + "payload": { + "outcome": "completed", + "member_key": "t", + "member_size": 1, + "member_index": 0, + "operation_kind": "timer", + "operation_identity": "01m7tfj0dt86gabdw3gvr6x7ss", + "selection_group_id": "select-calls:1:2", + "resolution_event_id": "01m7tfj328eczy13j1b3eyprgy", + "member_base_sequence": 1, + "selection_group_size": 2, + "resolution_event_type": "TimerFired", + "selection_group_base_sequence": 1 + }, + "workflow_task_id": null, + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:04.752074Z" + }, + { + "id": "01m7tfj3nksvadck19h10819p2", + "sequence": 7, + "event_type": "TimerScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfj32phrtt1h33xf07rszm", + "type": "workflow", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:04.844857Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:04.757674Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:14.844761Z", + "last_dispatched_at": "2026-11-19T18:02:04.793958Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:04.793958Z" + }, + "fire_at": "2026-11-19T18:02:06.359934Z", + "sequence": 3, + "timer_id": "01m7tfj3ngq7bbbwn7pbph87j1", + "delay_seconds": 1 + }, + "workflow_task_id": "01m7tfj32phrtt1h33xf07rszm", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:05.362057Z" + }, + { + "id": "01m7tfj69wrkgnj84twwz575m1", + "sequence": 8, + "event_type": "TimerFired", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfj3nq6nr10x4sj3x9gz85", + "type": "timer", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:07.974448Z", + "connection": "mysql", + "lease_owner": "01m7tfj3nq6nr10x4sj3x9gz85", + "available_at": "2026-11-19T18:02:06.359934Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:17.974454Z" + }, + "fire_at": "2026-11-19T18:02:06.359934Z", + "fired_at": "2026-11-19T18:02:08.057448Z", + "sequence": 3, + "timer_id": "01m7tfj3ngq7bbbwn7pbph87j1", + "delay_seconds": 1 + }, + "workflow_task_id": "01m7tfj3nq6nr10x4sj3x9gz85", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:08.059422Z" + }, + { + "id": "01m7tfj6x51pmsjbshkcwgry97", + "sequence": 9, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfj6a55pj0bm6br3ryrwkf", + "type": "workflow", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:08.190103Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:08.068110Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:18.190054Z", + "last_dispatched_at": "2026-11-19T18:02:08.099972Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:08.099972Z" + }, + "activity": { + "id": "01m7tfj6x1hxdjh54ks46ysn4f", + "type": "m0", + "class": "m0", + "queue": "rv307-f39c42", + "status": "pending", + "sequence": 4, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "created_at": "2026-11-19T18:02:08.674048Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfj6x1hxdjh54ks46ysn4f" + }, + "sequence": 4, + "activity_type": "m0", + "activity_class": "m0", + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfj6x1hxdjh54ks46ysn4f", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfj6a55pj0bm6br3ryrwkf", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:08.676278Z" + }, + { + "id": "01m7tfj6xcsksft6xttf9jf471", + "sequence": 10, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfj6a55pj0bm6br3ryrwkf", + "type": "workflow", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:08.190103Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:08.068110Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:18.190054Z", + "last_dispatched_at": "2026-11-19T18:02:08.099972Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:08.099972Z" + }, + "activity": { + "id": "01m7tfj6xb9j38aqpyrbd9f1wr", + "type": "m1", + "class": "m1", + "queue": "rv307-f39c42", + "status": "pending", + "sequence": 5, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "created_at": "2026-11-19T18:02:08.683384Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfj6xb9j38aqpyrbd9f1wr" + }, + "sequence": 5, + "activity_type": "m1", + "activity_class": "m1", + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfj6xb9j38aqpyrbd9f1wr", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfj6a55pj0bm6br3ryrwkf", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:08.684419Z" + }, + { + "id": "01m7tfj70bh8q54z37pfbbn4em", + "sequence": 11, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfj0edp9fwsb4ahn8rjk72", + "type": "activity", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:08.769739Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:02.060445Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:08.769940Z" + }, + "activity": { + "id": "01m7tfj0e7d43d71dc0cn963d7", + "type": "slow", + "class": "slow", + "queue": "rv307-f39c42", + "status": "running", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "attempt_id": "01M7TFJ701JSSCHXA8T2EJ54WW", + "created_at": "2026-11-19T18:02:02.055260Z", + "started_at": "2026-11-19T18:02:08.769739Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfj0e7d43d71dc0cn963d7" + }, + "sequence": 2, + "activity_type": "slow", + "activity_class": "slow", + "attempt_number": 1, + "parallel_group_id": "select-calls:1:2", + "activity_attempt_id": "01M7TFJ701JSSCHXA8T2EJ54WW", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "activity_execution_id": "01m7tfj0e7d43d71dc0cn963d7", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + }, + "workflow_task_id": "01m7tfj0edp9fwsb4ahn8rjk72", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:08.778249Z" + }, + { + "id": "01m7tfj74dn9wsdkq4sv6pk3jg", + "sequence": 12, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfj6x9vzrd4cfdwg8yarhb", + "type": "activity", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:08.899853Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:08.681145Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:08.900064Z" + }, + "activity": { + "id": "01m7tfj6x1hxdjh54ks46ysn4f", + "type": "m0", + "class": "m0", + "queue": "rv307-f39c42", + "status": "running", + "sequence": 4, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "attempt_id": "01M7TFJ7438T97B0R8W3QFT5ND", + "created_at": "2026-11-19T18:02:08.674048Z", + "started_at": "2026-11-19T18:02:08.899853Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfj6x1hxdjh54ks46ysn4f" + }, + "sequence": 4, + "activity_type": "m0", + "activity_class": "m0", + "attempt_number": 1, + "parallel_group_id": "parallel-activities:4:2", + "activity_attempt_id": "01M7TFJ7438T97B0R8W3QFT5ND", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfj6x1hxdjh54ks46ysn4f", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfj6x9vzrd4cfdwg8yarhb", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:08.908547Z" + }, + { + "id": "01m7tfj791jrvr12pggqz3j40n", + "sequence": 13, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfj6xec161vy6b32hnnnxm", + "type": "activity", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:09.051783Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:08.686369Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:09.051895Z" + }, + "activity": { + "id": "01m7tfj6xb9j38aqpyrbd9f1wr", + "type": "m1", + "class": "m1", + "queue": "rv307-f39c42", + "status": "running", + "sequence": 5, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "attempt_id": "01M7TFJ78VEQGP56SB5GFV2X3T", + "created_at": "2026-11-19T18:02:08.683384Z", + "started_at": "2026-11-19T18:02:09.051783Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfj6xb9j38aqpyrbd9f1wr" + }, + "sequence": 5, + "activity_type": "m1", + "activity_class": "m1", + "attempt_number": 1, + "parallel_group_id": "parallel-activities:4:2", + "activity_attempt_id": "01M7TFJ78VEQGP56SB5GFV2X3T", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfj6xb9j38aqpyrbd9f1wr", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfj6xec161vy6b32hnnnxm", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:09.056688Z" + }, + { + "id": "01m7tfj7cw2nkt972a6r7jf0rv", + "sequence": 14, + "event_type": "ActivityFailed", + "namespace": "default", + "payload": { + "code": 0, + "task": { + "id": "01m7tfj6xec161vy6b32hnnnxm", + "type": "activity", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:09.051783Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:08.686369Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:09.133724Z" + }, + "message": "fail-m1", + "activity": { + "id": "01m7tfj6xb9j38aqpyrbd9f1wr", + "type": "m1", + "class": "m1", + "queue": "rv307-f39c42", + "status": "failed", + "sequence": 5, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "closed_at": "2026-11-19T18:02:09.177030Z", + "exception": { + "code": 0, + "type": null, + "message": "fail-m1", + "non_retryable": true + }, + "attempt_id": "01M7TFJ78VEQGP56SB5GFV2X3T", + "created_at": "2026-11-19T18:02:08.683384Z", + "started_at": "2026-11-19T18:02:09.051783Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfj6xb9j38aqpyrbd9f1wr" + }, + "sequence": 5, + "exception": { + "type": null, + "message": "fail-m1", + "code": 0, + "non_retryable": true + }, + "failure_id": "01m7tfj7cr91jf20g5befgd5gt", + "activity_type": "m1", + "non_retryable": true, + "activity_class": "m1", + "attempt_number": 1, + "exception_type": null, + "exception_class": "RuntimeException", + "failure_category": "activity", + "parallel_group_id": "parallel-activities:4:2", + "activity_attempt_id": "01M7TFJ78VEQGP56SB5GFV2X3T", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfj6xb9j38aqpyrbd9f1wr", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfj6xec161vy6b32hnnnxm", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:09.179695Z" + }, + { + "id": "01m7tfj8p67khhm646dm2z939w", + "sequence": 15, + "event_type": "ActivityFailed", + "namespace": "default", + "payload": { + "code": 0, + "task": { + "id": "01m7tfj6x9vzrd4cfdwg8yarhb", + "type": "activity", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:08.899853Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:08.681145Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:09.007058Z" + }, + "message": "fail-m0", + "activity": { + "id": "01m7tfj6x1hxdjh54ks46ysn4f", + "type": "m0", + "class": "m0", + "queue": "rv307-f39c42", + "status": "failed", + "sequence": 4, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "closed_at": "2026-11-19T18:02:10.500387Z", + "exception": { + "code": 0, + "type": null, + "message": "fail-m0", + "non_retryable": true + }, + "attempt_id": "01M7TFJ7438T97B0R8W3QFT5ND", + "created_at": "2026-11-19T18:02:08.674048Z", + "started_at": "2026-11-19T18:02:08.899853Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfj6x1hxdjh54ks46ysn4f" + }, + "sequence": 4, + "exception": { + "type": null, + "message": "fail-m0", + "code": 0, + "non_retryable": true + }, + "failure_id": "01m7tfj8p375ctwss26hg6hpas", + "activity_type": "m0", + "non_retryable": true, + "activity_class": "m0", + "attempt_number": 1, + "exception_type": null, + "exception_class": "RuntimeException", + "failure_category": "activity", + "parallel_group_id": "parallel-activities:4:2", + "activity_attempt_id": "01M7TFJ7438T97B0R8W3QFT5ND", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfj6x1hxdjh54ks46ysn4f", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfj6x9vzrd4cfdwg8yarhb", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:10.501792Z" + }, + { + "id": "01m7tfj9sy5cdrrveet9chckqd", + "sequence": 16, + "event_type": "ActivityCompleted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfj0edp9fwsb4ahn8rjk72", + "type": "activity", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:08.769739Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:02.060445Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:08.840081Z" + }, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoEb2s=" + }, + "activity": { + "id": "01m7tfj0e7d43d71dc0cn963d7", + "type": "slow", + "class": "slow", + "queue": "rv307-f39c42", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoEb2s=" + }, + "status": "completed", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "closed_at": "2026-11-19T18:02:11.644584Z", + "attempt_id": "01M7TFJ701JSSCHXA8T2EJ54WW", + "created_at": "2026-11-19T18:02:02.055260Z", + "started_at": "2026-11-19T18:02:08.769739Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfj0e7d43d71dc0cn963d7" + }, + "sequence": 2, + "activity_type": "slow", + "payload_codec": "avro", + "activity_class": "slow", + "attempt_number": 1, + "parallel_group_id": "select-calls:1:2", + "activity_attempt_id": "01M7TFJ701JSSCHXA8T2EJ54WW", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "activity_execution_id": "01m7tfj0e7d43d71dc0cn963d7", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + }, + "workflow_task_id": "01m7tfj0edp9fwsb4ahn8rjk72", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:11.646013Z" + } + ], + "expected": { + "query_result": "fail-m0" + } +} diff --git a/tests/fixtures/replay_regressions/failure-order-released-zero-command-catch.json b/tests/fixtures/replay_regressions/failure-order-released-zero-command-catch.json new file mode 100644 index 00000000..b0116476 --- /dev/null +++ b/tests/fixtures/replay_regressions/failure-order-released-zero-command-catch.json @@ -0,0 +1,1111 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "failure-order-released-zero-command-catch", + "protocol_version": "1.20", + "bindings": [ + "python" + ], + "parallel_failure_order": { + "issues": [ + "durable-workflow/sdk-python#307" + ], + "note": "Recorded by published 2.19.1 on a Server built from durable-workflow/server main 52e3ec00 (docker-compose.test.yml). The group (m0, m1) was scheduled without the failure-order marker. m1 failed, then m0. The task that saw the settled group caught fail-m0 (lowest position) and then awaited a select() handle, so it recorded no command. This is the history the next task receives after the handle completes: it keeps fail-m0 and compensates it, and does not take the empty frontier after the group for a new decision." + }, + "workflow": { + "type": "rv348_handle", + "input": [], + "payload_codec": "avro", + "run_id": "01m7tfhzghsnm7xgs9xz0g5v2n", + "workflow_id": "rv307-rv348_handle-b54788" + }, + "history": [ + { + "id": "01m7tfhzh1505fxjhjgnac3ztn", + "sequence": 1, + "event_type": "StartAccepted", + "namespace": "default", + "payload": { + "memo": null, + "command": { + "id": "01m7tfhzgp2fnepq5he9jhes6q", + "type": "start", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "outcome": "started_new", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "sequence": 1, + "applied_at": "2026-11-19T18:02:01.106839Z", + "accepted_at": "2026-11-19T18:02:01.106831Z", + "auth_method": "token", + "auth_status": "authorized", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfhzghsnm7xgs9xz0g5v2n", + "request_route_name": "generated::FqJuJMl97iiXlIDz", + "request_fingerprint": "sha256:121ee40454411402a1c0d919aa0184dc9c7bbd7f3bb6d0a481223855f3306280" + }, + "outcome": "started_new", + "business_key": null, + "workflow_type": "rv348_handle", + "workflow_class": "rv348_handle", + "workflow_run_id": "01m7tfhzghsnm7xgs9xz0g5v2n", + "search_attributes": null, + "visibility_labels": null, + "workflow_command_id": "01m7tfhzgp2fnepq5he9jhes6q", + "workflow_instance_id": "rv307-rv348_handle-b54788" + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfhzgp2fnepq5he9jhes6q", + "recorded_at": "2026-11-19T18:02:01.117812Z" + }, + { + "id": "01m7tfhzhd4xbby9s36djxsedb", + "sequence": 2, + "event_type": "WorkflowStarted", + "namespace": "default", + "payload": { + "command": { + "id": "01m7tfhzgp2fnepq5he9jhes6q", + "type": "start", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "outcome": "started_new", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "sequence": 1, + "applied_at": "2026-11-19T18:02:01.106839Z", + "accepted_at": "2026-11-19T18:02:01.106831Z", + "auth_method": "token", + "auth_status": "authorized", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfhzghsnm7xgs9xz0g5v2n", + "request_route_name": "generated::FqJuJMl97iiXlIDz", + "request_fingerprint": "sha256:121ee40454411402a1c0d919aa0184dc9c7bbd7f3bb6d0a481223855f3306280" + }, + "workflow_type": "rv348_handle", + "workflow_class": "rv348_handle", + "run_deadline_at": "2026-11-19T18:12:01+00:00", + "workflow_run_id": "01m7tfhzghsnm7xgs9xz0g5v2n", + "declared_queries": [ + "caught" + ], + "declared_signals": [], + "declared_updates": [ + "report" + ], + "declared_entry_mode": "canonical", + "run_timeout_seconds": 600, + "workflow_command_id": "01m7tfhzgp2fnepq5he9jhes6q", + "workflow_instance_id": "rv307-rv348_handle-b54788", + "declared_entry_method": "handle", + "execution_deadline_at": "2026-11-19T19:02:01+00:00", + "declared_query_contracts": [ + { + "name": "caught", + "parameters": [] + } + ], + "declared_signal_contracts": [], + "declared_update_contracts": [ + { + "name": "report", + "parameters": [ + { + "name": "_", + "type": null, + "default": null, + "position": 0, + "required": false, + "variadic": true, + "allows_null": true, + "default_available": false + } + ] + } + ], + "execution_timeout_seconds": 3600, + "declared_update_validators": [], + "declared_entry_declaring_class": "external:rv348_handle", + "workflow_definition_fingerprint": "sha256:69ed762d44804878ebb703a2ad385119f6f776d0dae561c15f8f84e1707a704a", + "workflow_definition_fingerprint_source": "worker" + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfhzgp2fnepq5he9jhes6q", + "recorded_at": "2026-11-19T18:02:01.131946Z" + }, + { + "id": "01m7tfj0dz708dd35zb5hzs07a", + "sequence": 3, + "event_type": "TimerScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfhzhgz2rc1r2pw3w5aek0", + "type": "workflow", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:01.228018Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:01.136331Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:11.227929Z", + "last_dispatched_at": "2026-11-19T18:02:01.144200Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:01.144200Z" + }, + "fire_at": "2026-11-19T18:02:03.042591Z", + "sequence": 1, + "timer_id": "01m7tfj0dt86gabdw3gvr6x7ss", + "delay_seconds": 1, + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + }, + "workflow_task_id": "01m7tfhzhgz2rc1r2pw3w5aek0", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:02.046739Z" + }, + { + "id": "01m7tfj0ea70c9hrb4ct35kt96", + "sequence": 4, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfhzhgz2rc1r2pw3w5aek0", + "type": "workflow", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:01.228018Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:01.136331Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:11.227929Z", + "last_dispatched_at": "2026-11-19T18:02:01.144200Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:01.144200Z" + }, + "activity": { + "id": "01m7tfj0e7d43d71dc0cn963d7", + "type": "slow", + "class": "slow", + "queue": "rv307-f39c42", + "status": "pending", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "created_at": "2026-11-19T18:02:02.055260Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfj0e7d43d71dc0cn963d7" + }, + "sequence": 2, + "activity_type": "slow", + "activity_class": "slow", + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "activity_execution_id": "01m7tfj0e7d43d71dc0cn963d7", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + }, + "workflow_task_id": "01m7tfhzhgz2rc1r2pw3w5aek0", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:02.057530Z" + }, + { + "id": "01m7tfj328eczy13j1b3eyprgy", + "sequence": 5, + "event_type": "TimerFired", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfj0e3qf9j6pk5nmt7zxyf", + "type": "timer", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:04.697890Z", + "connection": "mysql", + "lease_owner": "01m7tfj0e3qf9j6pk5nmt7zxyf", + "available_at": "2026-11-19T18:02:03.042591Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:14.697901Z" + }, + "fire_at": "2026-11-19T18:02:03.042591Z", + "fired_at": "2026-11-19T18:02:04.740400Z", + "sequence": 1, + "timer_id": "01m7tfj0dt86gabdw3gvr6x7ss", + "delay_seconds": 1, + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "selection_member_key": "t", + "selection_member_kind": "timer", + "selection_member_size": 1, + "selection_member_index": 0, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 1 + }, + "workflow_task_id": "01m7tfj0e3qf9j6pk5nmt7zxyf", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:04.743320Z" + }, + { + "id": "01m7tfj32g0xfzd6d8baneqfa5", + "sequence": 6, + "event_type": "SelectionResolved", + "namespace": "default", + "payload": { + "outcome": "completed", + "member_key": "t", + "member_size": 1, + "member_index": 0, + "operation_kind": "timer", + "operation_identity": "01m7tfj0dt86gabdw3gvr6x7ss", + "selection_group_id": "select-calls:1:2", + "resolution_event_id": "01m7tfj328eczy13j1b3eyprgy", + "member_base_sequence": 1, + "selection_group_size": 2, + "resolution_event_type": "TimerFired", + "selection_group_base_sequence": 1 + }, + "workflow_task_id": null, + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:04.752074Z" + }, + { + "id": "01m7tfj3nksvadck19h10819p2", + "sequence": 7, + "event_type": "TimerScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfj32phrtt1h33xf07rszm", + "type": "workflow", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:04.844857Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:04.757674Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:14.844761Z", + "last_dispatched_at": "2026-11-19T18:02:04.793958Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:04.793958Z" + }, + "fire_at": "2026-11-19T18:02:06.359934Z", + "sequence": 3, + "timer_id": "01m7tfj3ngq7bbbwn7pbph87j1", + "delay_seconds": 1 + }, + "workflow_task_id": "01m7tfj32phrtt1h33xf07rszm", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:05.362057Z" + }, + { + "id": "01m7tfj69wrkgnj84twwz575m1", + "sequence": 8, + "event_type": "TimerFired", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfj3nq6nr10x4sj3x9gz85", + "type": "timer", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:07.974448Z", + "connection": "mysql", + "lease_owner": "01m7tfj3nq6nr10x4sj3x9gz85", + "available_at": "2026-11-19T18:02:06.359934Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:17.974454Z" + }, + "fire_at": "2026-11-19T18:02:06.359934Z", + "fired_at": "2026-11-19T18:02:08.057448Z", + "sequence": 3, + "timer_id": "01m7tfj3ngq7bbbwn7pbph87j1", + "delay_seconds": 1 + }, + "workflow_task_id": "01m7tfj3nq6nr10x4sj3x9gz85", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:08.059422Z" + }, + { + "id": "01m7tfj6x51pmsjbshkcwgry97", + "sequence": 9, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfj6a55pj0bm6br3ryrwkf", + "type": "workflow", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:08.190103Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:08.068110Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:18.190054Z", + "last_dispatched_at": "2026-11-19T18:02:08.099972Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:08.099972Z" + }, + "activity": { + "id": "01m7tfj6x1hxdjh54ks46ysn4f", + "type": "m0", + "class": "m0", + "queue": "rv307-f39c42", + "status": "pending", + "sequence": 4, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "created_at": "2026-11-19T18:02:08.674048Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfj6x1hxdjh54ks46ysn4f" + }, + "sequence": 4, + "activity_type": "m0", + "activity_class": "m0", + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfj6x1hxdjh54ks46ysn4f", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfj6a55pj0bm6br3ryrwkf", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:08.676278Z" + }, + { + "id": "01m7tfj6xcsksft6xttf9jf471", + "sequence": 10, + "event_type": "ActivityScheduled", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfj6a55pj0bm6br3ryrwkf", + "type": "workflow", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:08.190103Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:08.068110Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:02:18.190054Z", + "last_dispatched_at": "2026-11-19T18:02:08.099972Z", + "last_dispatch_attempt_at": "2026-11-19T18:02:08.099972Z" + }, + "activity": { + "id": "01m7tfj6xb9j38aqpyrbd9f1wr", + "type": "m1", + "class": "m1", + "queue": "rv307-f39c42", + "status": "pending", + "sequence": 5, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "created_at": "2026-11-19T18:02:08.683384Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 0, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfj6xb9j38aqpyrbd9f1wr" + }, + "sequence": 5, + "activity_type": "m1", + "activity_class": "m1", + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfj6xb9j38aqpyrbd9f1wr", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfj6a55pj0bm6br3ryrwkf", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:08.684419Z" + }, + { + "id": "01m7tfj70bh8q54z37pfbbn4em", + "sequence": 11, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfj0edp9fwsb4ahn8rjk72", + "type": "activity", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:08.769739Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:02.060445Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:08.769940Z" + }, + "activity": { + "id": "01m7tfj0e7d43d71dc0cn963d7", + "type": "slow", + "class": "slow", + "queue": "rv307-f39c42", + "status": "running", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "attempt_id": "01M7TFJ701JSSCHXA8T2EJ54WW", + "created_at": "2026-11-19T18:02:02.055260Z", + "started_at": "2026-11-19T18:02:08.769739Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfj0e7d43d71dc0cn963d7" + }, + "sequence": 2, + "activity_type": "slow", + "activity_class": "slow", + "attempt_number": 1, + "parallel_group_id": "select-calls:1:2", + "activity_attempt_id": "01M7TFJ701JSSCHXA8T2EJ54WW", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "activity_execution_id": "01m7tfj0e7d43d71dc0cn963d7", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + }, + "workflow_task_id": "01m7tfj0edp9fwsb4ahn8rjk72", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:08.778249Z" + }, + { + "id": "01m7tfj74dn9wsdkq4sv6pk3jg", + "sequence": 12, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfj6x9vzrd4cfdwg8yarhb", + "type": "activity", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:08.899853Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:08.681145Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:08.900064Z" + }, + "activity": { + "id": "01m7tfj6x1hxdjh54ks46ysn4f", + "type": "m0", + "class": "m0", + "queue": "rv307-f39c42", + "status": "running", + "sequence": 4, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "attempt_id": "01M7TFJ7438T97B0R8W3QFT5ND", + "created_at": "2026-11-19T18:02:08.674048Z", + "started_at": "2026-11-19T18:02:08.899853Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfj6x1hxdjh54ks46ysn4f" + }, + "sequence": 4, + "activity_type": "m0", + "activity_class": "m0", + "attempt_number": 1, + "parallel_group_id": "parallel-activities:4:2", + "activity_attempt_id": "01M7TFJ7438T97B0R8W3QFT5ND", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfj6x1hxdjh54ks46ysn4f", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfj6x9vzrd4cfdwg8yarhb", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:08.908547Z" + }, + { + "id": "01m7tfj791jrvr12pggqz3j40n", + "sequence": 13, + "event_type": "ActivityStarted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfj6xec161vy6b32hnnnxm", + "type": "activity", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:09.051783Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:08.686369Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:09.051895Z" + }, + "activity": { + "id": "01m7tfj6xb9j38aqpyrbd9f1wr", + "type": "m1", + "class": "m1", + "queue": "rv307-f39c42", + "status": "running", + "sequence": 5, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "attempt_id": "01M7TFJ78VEQGP56SB5GFV2X3T", + "created_at": "2026-11-19T18:02:08.683384Z", + "started_at": "2026-11-19T18:02:09.051783Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfj6xb9j38aqpyrbd9f1wr" + }, + "sequence": 5, + "activity_type": "m1", + "activity_class": "m1", + "attempt_number": 1, + "parallel_group_id": "parallel-activities:4:2", + "activity_attempt_id": "01M7TFJ78VEQGP56SB5GFV2X3T", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfj6xb9j38aqpyrbd9f1wr", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfj6xec161vy6b32hnnnxm", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:09.056688Z" + }, + { + "id": "01m7tfj7cw2nkt972a6r7jf0rv", + "sequence": 14, + "event_type": "ActivityFailed", + "namespace": "default", + "payload": { + "code": 0, + "task": { + "id": "01m7tfj6xec161vy6b32hnnnxm", + "type": "activity", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:09.051783Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:08.686369Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:09.133724Z" + }, + "message": "fail-m1", + "activity": { + "id": "01m7tfj6xb9j38aqpyrbd9f1wr", + "type": "m1", + "class": "m1", + "queue": "rv307-f39c42", + "status": "failed", + "sequence": 5, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAIA" + }, + "closed_at": "2026-11-19T18:02:09.177030Z", + "exception": { + "code": 0, + "type": null, + "message": "fail-m1", + "non_retryable": true + }, + "attempt_id": "01M7TFJ78VEQGP56SB5GFV2X3T", + "created_at": "2026-11-19T18:02:08.683384Z", + "started_at": "2026-11-19T18:02:09.051783Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfj6xb9j38aqpyrbd9f1wr" + }, + "sequence": 5, + "exception": { + "type": null, + "message": "fail-m1", + "code": 0, + "non_retryable": true + }, + "failure_id": "01m7tfj7cr91jf20g5befgd5gt", + "activity_type": "m1", + "non_retryable": true, + "activity_class": "m1", + "attempt_number": 1, + "exception_type": null, + "exception_class": "RuntimeException", + "failure_category": "activity", + "parallel_group_id": "parallel-activities:4:2", + "activity_attempt_id": "01M7TFJ78VEQGP56SB5GFV2X3T", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "activity_execution_id": "01m7tfj6xb9j38aqpyrbd9f1wr", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfj6xec161vy6b32hnnnxm", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:09.179695Z" + }, + { + "id": "01m7tfj8p67khhm646dm2z939w", + "sequence": 15, + "event_type": "ActivityFailed", + "namespace": "default", + "payload": { + "code": 0, + "task": { + "id": "01m7tfj6x9vzrd4cfdwg8yarhb", + "type": "activity", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:08.899853Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:08.681145Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:09.007058Z" + }, + "message": "fail-m0", + "activity": { + "id": "01m7tfj6x1hxdjh54ks46ysn4f", + "type": "m0", + "class": "m0", + "queue": "rv307-f39c42", + "status": "failed", + "sequence": 4, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwCBAAA" + }, + "closed_at": "2026-11-19T18:02:10.500387Z", + "exception": { + "code": 0, + "type": null, + "message": "fail-m0", + "non_retryable": true + }, + "attempt_id": "01M7TFJ7438T97B0R8W3QFT5ND", + "created_at": "2026-11-19T18:02:08.674048Z", + "started_at": "2026-11-19T18:02:08.899853Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfj6x1hxdjh54ks46ysn4f" + }, + "sequence": 4, + "exception": { + "type": null, + "message": "fail-m0", + "code": 0, + "non_retryable": true + }, + "failure_id": "01m7tfj8p375ctwss26hg6hpas", + "activity_type": "m0", + "non_retryable": true, + "activity_class": "m0", + "attempt_number": 1, + "exception_type": null, + "exception_class": "RuntimeException", + "failure_category": "activity", + "parallel_group_id": "parallel-activities:4:2", + "activity_attempt_id": "01M7TFJ7438T97B0R8W3QFT5ND", + "parallel_group_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-activities:4:2", + "parallel_group_kind": "activity", + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_base_sequence": 4 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 0, + "activity_execution_id": "01m7tfj6x1hxdjh54ks46ysn4f", + "parallel_group_base_sequence": 4 + }, + "workflow_task_id": "01m7tfj6x9vzrd4cfdwg8yarhb", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:10.501792Z" + }, + { + "id": "01m7tfj9sy5cdrrveet9chckqd", + "sequence": 16, + "event_type": "ActivityCompleted", + "namespace": "default", + "payload": { + "task": { + "id": "01m7tfj0edp9fwsb4ahn8rjk72", + "type": "activity", + "queue": "rv307-f39c42", + "status": "leased", + "leased_at": "2026-11-19T18:02:08.769739Z", + "connection": "mysql", + "lease_owner": "rv307-w-e62bb0", + "available_at": "2026-11-19T18:02:02.060445Z", + "repair_count": 0, + "attempt_count": 1, + "lease_expires_at": "2026-11-19T18:07:08.840081Z" + }, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoEb2s=" + }, + "activity": { + "id": "01m7tfj0e7d43d71dc0cn963d7", + "type": "slow", + "class": "slow", + "queue": "rv307-f39c42", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoEb2s=" + }, + "status": "completed", + "sequence": 2, + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "closed_at": "2026-11-19T18:02:11.644584Z", + "attempt_id": "01M7TFJ701JSSCHXA8T2EJ54WW", + "created_at": "2026-11-19T18:02:02.055260Z", + "started_at": "2026-11-19T18:02:08.769739Z", + "retry_policy": { + "max_attempts": 1, + "backoff_seconds": [], + "snapshot_version": 1, + "heartbeat_timeout": null, + "start_to_close_timeout": null, + "non_retryable_error_types": [], + "schedule_to_close_timeout": null, + "schedule_to_start_timeout": null + }, + "attempt_count": 1, + "payload_codec": "avro", + "local_activity": false, + "idempotency_key": "01m7tfj0e7d43d71dc0cn963d7" + }, + "sequence": 2, + "activity_type": "slow", + "payload_codec": "avro", + "activity_class": "slow", + "attempt_number": 1, + "parallel_group_id": "select-calls:1:2", + "activity_attempt_id": "01M7TFJ701JSSCHXA8T2EJ54WW", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:1:2", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + } + ], + "parallel_group_size": 2, + "parallel_group_index": 1, + "selection_member_key": "slow", + "activity_execution_id": "01m7tfj0e7d43d71dc0cn963d7", + "selection_member_kind": "activity", + "selection_member_size": 1, + "selection_member_index": 1, + "parallel_group_base_sequence": 1, + "selection_member_base_sequence": 2 + }, + "workflow_task_id": "01m7tfj0edp9fwsb4ahn8rjk72", + "workflow_command_id": null, + "recorded_at": "2026-11-19T18:02:11.646013Z" + } + ], + "expected": { + "command_sequence": [ + { + "type": "schedule_activity", + "activity_type": "compensate", + "arguments": [ + "fail-m0" + ] + } + ] + } +} diff --git a/tests/fixtures/replay_regressions/parallel-group-metadata-emission-after-failure-order-marker.json b/tests/fixtures/replay_regressions/parallel-group-metadata-emission-after-failure-order-marker.json new file mode 100644 index 00000000..3e124cc3 --- /dev/null +++ b/tests/fixtures/replay_regressions/parallel-group-metadata-emission-after-failure-order-marker.json @@ -0,0 +1,93 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "parallel-group-metadata-emission-after-failure-order-marker", + "protocol_version": "1.20", + "bindings": [ + "python" + ], + "supersedes": [ + "parallel-group-metadata-emission" + ], + "parallel_failure_order": { + "issues": [ + "durable-workflow/sdk-python#307" + ], + "note": "The new run of parallel-group-metadata-emission. A run now records the parallel failure-order version marker directly before its first all-group that can fail, so the group's metadata starts at sequence 2. Histories recorded without the marker keep their sequences and their failure selection." + }, + "workflow": { + "type": "tests.replay.parallel-metadata-producer", + "input": [], + "payload_codec": "avro" + }, + "history": [ + { + "event_type": "WorkflowStarted", + "payload": { + "workflow_type": "tests.replay.parallel-metadata-producer" + } + } + ], + "expected": { + "command_sequence": [ + { + "type": "record_version_marker", + "change_id": "durable_workflow.parallel_failure_order", + "version": 1, + "min_supported": 1, + "max_supported": 1 + }, + { + "type": "schedule_activity", + "parallel_group_id": "parallel-calls:2:3", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 2, + "parallel_group_size": 3, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:2:3", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 2, + "parallel_group_size": 3, + "parallel_group_index": 0 + } + ] + }, + { + "type": "start_child_workflow", + "parallel_group_id": "parallel-calls:2:3", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 2, + "parallel_group_size": 3, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:2:3", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 2, + "parallel_group_size": 3, + "parallel_group_index": 1 + } + ] + }, + { + "type": "start_timer", + "parallel_group_id": "parallel-calls:2:3", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 2, + "parallel_group_size": 3, + "parallel_group_index": 2, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:2:3", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 2, + "parallel_group_size": 3, + "parallel_group_index": 2 + } + ] + } + ] + } +} diff --git a/tests/fixtures/replay_regressions/settled-group-first-task-records-decision.json b/tests/fixtures/replay_regressions/settled-group-first-task-records-decision.json deleted file mode 100644 index bb0d1d32..00000000 --- a/tests/fixtures/replay_regressions/settled-group-first-task-records-decision.json +++ /dev/null @@ -1,1047 +0,0 @@ -{ - "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", - "fixture_schema": "durable-workflow.replay-regression/v1", - "id": "settled-group-first-task-records-decision", - "protocol_version": "1.20", - "bindings": [ - "python" - ], - "settled_group_first_recorded_failure": { - "issues": [ - "durable-workflow/sdk-python#307" - ], - "note": "The history the deciding task received (native Server built from durable-workflow/server main 52e3ec00): every member settled, member 1 failed first. The task records which member it raised and schedules the compensation for it." - }, - "workflow": { - "type": "py307.caught", - "input": [], - "payload_codec": "avro" - }, - "history": [ - { - "sequence": 1, - "event_type": "StartAccepted", - "timestamp": "2026-11-19T16:26:12.027000Z", - "principal": { - "type": "auth:token", - "id": "legacy-token", - "label": "Admin" - }, - "payload": { - "workflow_instance_id": "py307-caught-1795105571", - "workflow_run_id": "01M7TA2H5T1GTXE76ZS15AM165", - "workflow_type": "py307.caught", - "workflow_class": "py307.caught", - "workflow_command_id": "01M7TA2H5T1GTXE76ZS15AM166", - "memo": null, - "business_key": null, - "visibility_labels": null, - "search_attributes": null, - "outcome": "started_new", - "command": { - "id": "01M7TA2H5T1GTXE76ZS15AM166", - "sequence": 1, - "type": "start", - "target_scope": "instance", - "resolved_run_id": "01M7TA2H5T1GTXE76ZS15AM165", - "payload_codec": "avro", - "payload": { - "codec": "avro", - "blob": "wwHioz3/VYAiNwwA" - }, - "source": "control_plane", - "context": { - "principal": { - "type": "auth:token", - "id": "legacy-token", - "label": "Admin" - } - }, - "caller_label": "Standalone Server", - "principal_type": "auth:token", - "principal_id": "legacy-token", - "principal_label": "Admin", - "auth_status": "authorized", - "auth_method": "token", - "request_method": "POST", - "request_path": "/api/workflows", - "request_fingerprint": "sha256:47edc9a9a51d6470d6a390ed2559a1fcda1699404c126e1e34ac6345d8076f0f", - "status": "accepted", - "outcome": "started_new", - "accepted_at": "2026-11-19T16:26:12.026000Z", - "applied_at": "2026-11-19T16:26:12.026000Z" - } - } - }, - { - "sequence": 2, - "event_type": "WorkflowStarted", - "timestamp": "2026-11-19T16:26:12.028000Z", - "principal": { - "type": "auth:token", - "id": "legacy-token", - "label": "Admin" - }, - "payload": { - "workflow_instance_id": "py307-caught-1795105571", - "workflow_run_id": "01M7TA2H5T1GTXE76ZS15AM165", - "workflow_type": "py307.caught", - "workflow_class": "py307.caught", - "workflow_command_id": "01M7TA2H5T1GTXE76ZS15AM166", - "execution_timeout_seconds": 3600, - "run_timeout_seconds": 600, - "execution_deadline_at": "2026-11-19T17:26:12.026Z", - "run_deadline_at": "2026-11-19T16:36:12.026Z", - "command": { - "id": "01M7TA2H5T1GTXE76ZS15AM166", - "sequence": 1, - "type": "start", - "target_scope": "instance", - "resolved_run_id": "01M7TA2H5T1GTXE76ZS15AM165", - "payload_codec": "avro", - "payload": { - "codec": "avro", - "blob": "wwHioz3/VYAiNwwA" - }, - "source": "control_plane", - "context": { - "principal": { - "type": "auth:token", - "id": "legacy-token", - "label": "Admin" - } - }, - "caller_label": "Standalone Server", - "principal_type": "auth:token", - "principal_id": "legacy-token", - "principal_label": "Admin", - "auth_status": "authorized", - "auth_method": "token", - "request_method": "POST", - "request_path": "/api/workflows", - "request_fingerprint": "sha256:47edc9a9a51d6470d6a390ed2559a1fcda1699404c126e1e34ac6345d8076f0f", - "status": "accepted", - "outcome": "started_new", - "accepted_at": "2026-11-19T16:26:12.026000Z", - "applied_at": "2026-11-19T16:26:12.026000Z" - } - } - }, - { - "sequence": 3, - "event_type": "ActivityScheduled", - "timestamp": "2026-11-19T16:26:12.164000Z", - "principal": null, - "payload": { - "activity_execution_id": "01M7TA2HA40T3VYAMYBJSGH18V", - "activity_type": "py307.fail", - "activity_class": "py307.fail", - "sequence": 1, - "activity": { - "id": "01M7TA2HA40T3VYAMYBJSGH18V", - "idempotency_key": "01M7TA2HA40T3VYAMYBJSGH18V", - "sequence": 1, - "type": "py307.fail", - "class": "py307.fail", - "local_activity": false, - "activity_type": "py307.fail", - "arguments": { - "codec": "avro", - "blob": "wwHioz3/VYAiNwwCBgAAAAAAAPg/AA==" - }, - "retry_policy": { - "snapshot_version": 1, - "max_attempts": 1, - "backoff_seconds": [], - "non_retryable_error_types": [], - "start_to_close_timeout": 60, - "schedule_to_start_timeout": null, - "schedule_to_close_timeout": null, - "heartbeat_timeout": null - }, - "status": "pending", - "attempt_count": 0, - "closed_at": null, - "attempt_id": null, - "exception": null, - "payload_codec": "avro", - "queue": "py307-activities" - }, - "parallel_group_id": "parallel-activities:1:4", - "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, - "parallel_group_size": 4, - "parallel_group_index": 0, - "parallel_group_path": [ - { - "parallel_group_id": "parallel-activities:1:4", - "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, - "parallel_group_size": 4, - "parallel_group_index": 0 - } - ], - "task": { - "id": "01M7TA2H5WNWMXXSWJ6JJ4M0GS", - "type": "workflow", - "status": "leased", - "attempt_count": 1, - "repair_count": 0, - "queue": "py307-workflows", - "connection": "sqlite", - "available_at": "2026-11-19T16:26:12.028000Z", - "leased_at": "2026-11-19T16:26:12.086000Z", - "lease_expires_at": "2026-11-19T16:27:12.086000Z", - "last_dispatch_attempt_at": "2026-11-19T16:26:12.028000Z", - "last_dispatched_at": "2026-11-19T16:26:12.028000Z", - "lease_owner": "py-worker-89b000d4" - } - } - }, - { - "sequence": 4, - "event_type": "ActivityScheduled", - "timestamp": "2026-11-19T16:26:12.165000Z", - "principal": null, - "payload": { - "activity_execution_id": "01M7TA2HA54J8KXYXJEJ7S4S5T", - "activity_type": "py307.fail", - "activity_class": "py307.fail", - "sequence": 2, - "activity": { - "id": "01M7TA2HA54J8KXYXJEJ7S4S5T", - "idempotency_key": "01M7TA2HA54J8KXYXJEJ7S4S5T", - "sequence": 2, - "type": "py307.fail", - "class": "py307.fail", - "local_activity": false, - "activity_type": "py307.fail", - "arguments": { - "codec": "avro", - "blob": "wwHioz3/VYAiNwwCBAAA" - }, - "retry_policy": { - "snapshot_version": 1, - "max_attempts": 1, - "backoff_seconds": [], - "non_retryable_error_types": [], - "start_to_close_timeout": 60, - "schedule_to_start_timeout": null, - "schedule_to_close_timeout": null, - "heartbeat_timeout": null - }, - "status": "pending", - "attempt_count": 0, - "closed_at": null, - "attempt_id": null, - "exception": null, - "payload_codec": "avro", - "queue": "py307-activities" - }, - "parallel_group_id": "parallel-activities:1:4", - "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, - "parallel_group_size": 4, - "parallel_group_index": 1, - "parallel_group_path": [ - { - "parallel_group_id": "parallel-activities:1:4", - "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, - "parallel_group_size": 4, - "parallel_group_index": 1 - } - ], - "task": { - "id": "01M7TA2H5WNWMXXSWJ6JJ4M0GS", - "type": "workflow", - "status": "leased", - "attempt_count": 1, - "repair_count": 0, - "queue": "py307-workflows", - "connection": "sqlite", - "available_at": "2026-11-19T16:26:12.028000Z", - "leased_at": "2026-11-19T16:26:12.086000Z", - "lease_expires_at": "2026-11-19T16:27:12.086000Z", - "last_dispatch_attempt_at": "2026-11-19T16:26:12.028000Z", - "last_dispatched_at": "2026-11-19T16:26:12.028000Z", - "lease_owner": "py-worker-89b000d4" - } - } - }, - { - "sequence": 5, - "event_type": "ActivityScheduled", - "timestamp": "2026-11-19T16:26:12.166000Z", - "principal": null, - "payload": { - "activity_execution_id": "01M7TA2HA66FRY17YJN6HC6S52", - "activity_type": "py307.ok", - "activity_class": "py307.ok", - "sequence": 3, - "activity": { - "id": "01M7TA2HA66FRY17YJN6HC6S52", - "idempotency_key": "01M7TA2HA66FRY17YJN6HC6S52", - "sequence": 3, - "type": "py307.ok", - "class": "py307.ok", - "local_activity": false, - "activity_type": "py307.ok", - "arguments": { - "codec": "avro", - "blob": "wwHioz3/VYAiNwwCBAAA" - }, - "retry_policy": { - "snapshot_version": 1, - "max_attempts": 1, - "backoff_seconds": [], - "non_retryable_error_types": [], - "start_to_close_timeout": 60, - "schedule_to_start_timeout": null, - "schedule_to_close_timeout": null, - "heartbeat_timeout": null - }, - "status": "pending", - "attempt_count": 0, - "closed_at": null, - "attempt_id": null, - "exception": null, - "payload_codec": "avro", - "queue": "py307-activities" - }, - "parallel_group_id": "parallel-activities:1:4", - "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, - "parallel_group_size": 4, - "parallel_group_index": 2, - "parallel_group_path": [ - { - "parallel_group_id": "parallel-activities:1:4", - "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, - "parallel_group_size": 4, - "parallel_group_index": 2 - } - ], - "task": { - "id": "01M7TA2H5WNWMXXSWJ6JJ4M0GS", - "type": "workflow", - "status": "leased", - "attempt_count": 1, - "repair_count": 0, - "queue": "py307-workflows", - "connection": "sqlite", - "available_at": "2026-11-19T16:26:12.028000Z", - "leased_at": "2026-11-19T16:26:12.086000Z", - "lease_expires_at": "2026-11-19T16:27:12.086000Z", - "last_dispatch_attempt_at": "2026-11-19T16:26:12.028000Z", - "last_dispatched_at": "2026-11-19T16:26:12.028000Z", - "lease_owner": "py-worker-89b000d4" - } - } - }, - { - "sequence": 6, - "event_type": "ActivityScheduled", - "timestamp": "2026-11-19T16:26:12.166000Z", - "principal": null, - "payload": { - "activity_execution_id": "01M7TA2HA66FRY17YJN6HC6S55", - "activity_type": "py307.ok", - "activity_class": "py307.ok", - "sequence": 4, - "activity": { - "id": "01M7TA2HA66FRY17YJN6HC6S55", - "idempotency_key": "01M7TA2HA66FRY17YJN6HC6S55", - "sequence": 4, - "type": "py307.ok", - "class": "py307.ok", - "local_activity": false, - "activity_type": "py307.ok", - "arguments": { - "codec": "avro", - "blob": "wwHioz3/VYAiNwwCBgAAAAAAAOA/AA==" - }, - "retry_policy": { - "snapshot_version": 1, - "max_attempts": 1, - "backoff_seconds": [], - "non_retryable_error_types": [], - "start_to_close_timeout": 60, - "schedule_to_start_timeout": null, - "schedule_to_close_timeout": null, - "heartbeat_timeout": null - }, - "status": "pending", - "attempt_count": 0, - "closed_at": null, - "attempt_id": null, - "exception": null, - "payload_codec": "avro", - "queue": "py307-activities" - }, - "parallel_group_id": "parallel-activities:1:4", - "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, - "parallel_group_size": 4, - "parallel_group_index": 3, - "parallel_group_path": [ - { - "parallel_group_id": "parallel-activities:1:4", - "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, - "parallel_group_size": 4, - "parallel_group_index": 3 - } - ], - "task": { - "id": "01M7TA2H5WNWMXXSWJ6JJ4M0GS", - "type": "workflow", - "status": "leased", - "attempt_count": 1, - "repair_count": 0, - "queue": "py307-workflows", - "connection": "sqlite", - "available_at": "2026-11-19T16:26:12.028000Z", - "leased_at": "2026-11-19T16:26:12.086000Z", - "lease_expires_at": "2026-11-19T16:27:12.086000Z", - "last_dispatch_attempt_at": "2026-11-19T16:26:12.028000Z", - "last_dispatched_at": "2026-11-19T16:26:12.028000Z", - "lease_owner": "py-worker-89b000d4" - } - } - }, - { - "sequence": 7, - "event_type": "ActivityStarted", - "timestamp": "2026-11-19T16:26:12.449000Z", - "principal": null, - "payload": { - "activity_execution_id": "01M7TA2HA40T3VYAMYBJSGH18V", - "activity_type": "py307.fail", - "activity_class": "py307.fail", - "sequence": 1, - "activity": { - "id": "01M7TA2HA40T3VYAMYBJSGH18V", - "idempotency_key": "01M7TA2HA40T3VYAMYBJSGH18V", - "sequence": 1, - "type": "py307.fail", - "class": "py307.fail", - "local_activity": false, - "activity_type": "py307.fail", - "arguments": { - "codec": "avro", - "blob": "wwHioz3/VYAiNwwCBgAAAAAAAPg/AA==" - }, - "retry_policy": { - "snapshot_version": 1, - "max_attempts": 1, - "backoff_seconds": [], - "non_retryable_error_types": [], - "start_to_close_timeout": 60, - "schedule_to_start_timeout": null, - "schedule_to_close_timeout": null, - "heartbeat_timeout": null - }, - "status": "running", - "attempt_count": 1, - "closed_at": null, - "attempt_id": "01M7TA2HK1H2Y7GACGE77Z49TH", - "exception": null, - "payload_codec": "avro", - "queue": "py307-activities", - "started_at": "2026-11-19T16:26:12.449Z" - }, - "parallel_group_id": "parallel-activities:1:4", - "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, - "parallel_group_size": 4, - "parallel_group_index": 0, - "parallel_group_path": [ - { - "parallel_group_id": "parallel-activities:1:4", - "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, - "parallel_group_size": 4, - "parallel_group_index": 0 - } - ], - "activity_attempt_id": "01M7TA2HK1H2Y7GACGE77Z49TH", - "attempt_number": 1, - "task": { - "id": "01M7TA2HA40T3VYAMYBJSGH18X", - "type": "activity", - "status": "leased", - "repair_count": 0, - "available_at": "2026-11-19T16:26:12.164Z", - "leased_at": "2026-11-19T16:26:12.448Z", - "lease_owner": "py-worker-9b247caa", - "lease_expires_at": "2026-11-19T16:31:12.448Z", - "attempt_count": 1, - "queue": "py307-activities", - "connection": "sqlite" - } - } - }, - { - "sequence": 8, - "event_type": "ActivityStarted", - "timestamp": "2026-11-19T16:26:12.497000Z", - "principal": null, - "payload": { - "activity_execution_id": "01M7TA2HA54J8KXYXJEJ7S4S5T", - "activity_type": "py307.fail", - "activity_class": "py307.fail", - "sequence": 2, - "activity": { - "id": "01M7TA2HA54J8KXYXJEJ7S4S5T", - "idempotency_key": "01M7TA2HA54J8KXYXJEJ7S4S5T", - "sequence": 2, - "type": "py307.fail", - "class": "py307.fail", - "local_activity": false, - "activity_type": "py307.fail", - "arguments": { - "codec": "avro", - "blob": "wwHioz3/VYAiNwwCBAAA" - }, - "retry_policy": { - "snapshot_version": 1, - "max_attempts": 1, - "backoff_seconds": [], - "non_retryable_error_types": [], - "start_to_close_timeout": 60, - "schedule_to_start_timeout": null, - "schedule_to_close_timeout": null, - "heartbeat_timeout": null - }, - "status": "running", - "attempt_count": 1, - "closed_at": null, - "attempt_id": "01M7TA2HMHNJVVHNEW678ZZ1KW", - "exception": null, - "payload_codec": "avro", - "queue": "py307-activities", - "started_at": "2026-11-19T16:26:12.497Z" - }, - "parallel_group_id": "parallel-activities:1:4", - "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, - "parallel_group_size": 4, - "parallel_group_index": 1, - "parallel_group_path": [ - { - "parallel_group_id": "parallel-activities:1:4", - "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, - "parallel_group_size": 4, - "parallel_group_index": 1 - } - ], - "activity_attempt_id": "01M7TA2HMHNJVVHNEW678ZZ1KW", - "attempt_number": 1, - "task": { - "id": "01M7TA2HA54J8KXYXJEJ7S4S5W", - "type": "activity", - "status": "leased", - "repair_count": 0, - "available_at": "2026-11-19T16:26:12.165Z", - "leased_at": "2026-11-19T16:26:12.496Z", - "lease_owner": "py-worker-9b247caa", - "lease_expires_at": "2026-11-19T16:31:12.496Z", - "attempt_count": 1, - "queue": "py307-activities", - "connection": "sqlite" - } - } - }, - { - "sequence": 9, - "event_type": "ActivityStarted", - "timestamp": "2026-11-19T16:26:12.567000Z", - "principal": null, - "payload": { - "activity_execution_id": "01M7TA2HA66FRY17YJN6HC6S52", - "activity_type": "py307.ok", - "activity_class": "py307.ok", - "sequence": 3, - "activity": { - "id": "01M7TA2HA66FRY17YJN6HC6S52", - "idempotency_key": "01M7TA2HA66FRY17YJN6HC6S52", - "sequence": 3, - "type": "py307.ok", - "class": "py307.ok", - "local_activity": false, - "activity_type": "py307.ok", - "arguments": { - "codec": "avro", - "blob": "wwHioz3/VYAiNwwCBAAA" - }, - "retry_policy": { - "snapshot_version": 1, - "max_attempts": 1, - "backoff_seconds": [], - "non_retryable_error_types": [], - "start_to_close_timeout": 60, - "schedule_to_start_timeout": null, - "schedule_to_close_timeout": null, - "heartbeat_timeout": null - }, - "status": "running", - "attempt_count": 1, - "closed_at": null, - "attempt_id": "01M7TA2HPQRHFPFHKK2D6JVY07", - "exception": null, - "payload_codec": "avro", - "queue": "py307-activities", - "started_at": "2026-11-19T16:26:12.567Z" - }, - "parallel_group_id": "parallel-activities:1:4", - "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, - "parallel_group_size": 4, - "parallel_group_index": 2, - "parallel_group_path": [ - { - "parallel_group_id": "parallel-activities:1:4", - "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, - "parallel_group_size": 4, - "parallel_group_index": 2 - } - ], - "activity_attempt_id": "01M7TA2HPQRHFPFHKK2D6JVY07", - "attempt_number": 1, - "task": { - "id": "01M7TA2HA66FRY17YJN6HC6S54", - "type": "activity", - "status": "leased", - "repair_count": 0, - "available_at": "2026-11-19T16:26:12.166Z", - "leased_at": "2026-11-19T16:26:12.567Z", - "lease_owner": "py-worker-9b247caa", - "lease_expires_at": "2026-11-19T16:31:12.567Z", - "attempt_count": 1, - "queue": "py307-activities", - "connection": "sqlite" - } - } - }, - { - "sequence": 10, - "event_type": "ActivityFailed", - "timestamp": "2026-11-19T16:26:12.580000Z", - "principal": null, - "payload": { - "activity_execution_id": "01M7TA2HA54J8KXYXJEJ7S4S5T", - "activity_type": "py307.fail", - "activity_class": "py307.fail", - "sequence": 2, - "activity": { - "id": "01M7TA2HA54J8KXYXJEJ7S4S5T", - "idempotency_key": "01M7TA2HA54J8KXYXJEJ7S4S5T", - "sequence": 2, - "type": "py307.fail", - "class": "py307.fail", - "local_activity": false, - "activity_type": "py307.fail", - "arguments": { - "codec": "avro", - "blob": "wwHioz3/VYAiNwwCBAAA" - }, - "retry_policy": { - "snapshot_version": 1, - "max_attempts": 1, - "backoff_seconds": [], - "non_retryable_error_types": [], - "start_to_close_timeout": 60, - "schedule_to_start_timeout": null, - "schedule_to_close_timeout": null, - "heartbeat_timeout": null - }, - "status": "failed", - "attempt_count": 1, - "closed_at": "2026-11-19T16:26:12.579Z", - "attempt_id": "01M7TA2HMHNJVVHNEW678ZZ1KW", - "exception": { - "code": 0, - "type": "RuntimeError", - "message": "py307 failed after 0" - }, - "payload_codec": "avro", - "queue": "py307-activities", - "started_at": "2026-11-19T16:26:12.497Z" - }, - "parallel_group_id": "parallel-activities:1:4", - "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, - "parallel_group_size": 4, - "parallel_group_index": 1, - "parallel_group_path": [ - { - "parallel_group_id": "parallel-activities:1:4", - "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, - "parallel_group_size": 4, - "parallel_group_index": 1 - } - ], - "activity_attempt_id": "01M7TA2HMHNJVVHNEW678ZZ1KW", - "attempt_number": 1, - "failure_id": "01M7TA2HQ4AZ38WZ2BVBJKHQKX", - "failure_category": "activity", - "non_retryable": false, - "exception_type": "RuntimeError", - "exception_class": "builtins.RuntimeError", - "message": "py307 failed after 0", - "code": 0, - "exception": { - "type": "RuntimeError", - "message": "py307 failed after 0", - "code": 0 - }, - "task": { - "id": "01M7TA2HA54J8KXYXJEJ7S4S5W", - "type": "activity", - "status": "leased", - "available_at": "2026-11-19T16:26:12.165Z", - "leased_at": "2026-11-19T16:26:12.496Z", - "lease_owner": "py-worker-9b247caa", - "lease_expires_at": "2026-11-19T16:31:12.524Z", - "attempt_count": 1, - "repair_count": 0, - "queue": "py307-activities", - "connection": "sqlite" - } - } - }, - { - "sequence": 11, - "event_type": "ActivityCompleted", - "timestamp": "2026-11-19T16:26:12.638000Z", - "principal": null, - "payload": { - "activity_execution_id": "01M7TA2HA66FRY17YJN6HC6S52", - "activity_type": "py307.ok", - "activity_class": "py307.ok", - "sequence": 3, - "activity": { - "id": "01M7TA2HA66FRY17YJN6HC6S52", - "idempotency_key": "01M7TA2HA66FRY17YJN6HC6S52", - "sequence": 3, - "type": "py307.ok", - "class": "py307.ok", - "local_activity": false, - "activity_type": "py307.ok", - "arguments": { - "codec": "avro", - "blob": "wwHioz3/VYAiNwwCBAAA" - }, - "retry_policy": { - "snapshot_version": 1, - "max_attempts": 1, - "backoff_seconds": [], - "non_retryable_error_types": [], - "start_to_close_timeout": 60, - "schedule_to_start_timeout": null, - "schedule_to_close_timeout": null, - "heartbeat_timeout": null - }, - "status": "completed", - "attempt_count": 1, - "closed_at": "2026-11-19T16:26:12.636Z", - "attempt_id": "01M7TA2HPQRHFPFHKK2D6JVY07", - "exception": null, - "payload_codec": "avro", - "queue": "py307-activities", - "started_at": "2026-11-19T16:26:12.567Z" - }, - "parallel_group_id": "parallel-activities:1:4", - "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, - "parallel_group_size": 4, - "parallel_group_index": 2, - "parallel_group_path": [ - { - "parallel_group_id": "parallel-activities:1:4", - "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, - "parallel_group_size": 4, - "parallel_group_index": 2 - } - ], - "activity_attempt_id": "01M7TA2HPQRHFPFHKK2D6JVY07", - "attempt_number": 1, - "result": { - "codec": "avro", - "blob": "wwHioz3/VYAiNwoEb2s=" - }, - "payload_codec": "avro", - "task": { - "id": "01M7TA2HA66FRY17YJN6HC6S54", - "type": "activity", - "status": "leased", - "repair_count": 0, - "available_at": "2026-11-19T16:26:12.166Z", - "leased_at": "2026-11-19T16:26:12.567Z", - "lease_owner": "py-worker-9b247caa", - "lease_expires_at": "2026-11-19T16:31:12.615Z", - "attempt_count": 1, - "queue": "py307-activities", - "connection": "sqlite" - } - } - }, - { - "sequence": 12, - "event_type": "ActivityStarted", - "timestamp": "2026-11-19T16:26:12.662000Z", - "principal": null, - "payload": { - "activity_execution_id": "01M7TA2HA66FRY17YJN6HC6S55", - "activity_type": "py307.ok", - "activity_class": "py307.ok", - "sequence": 4, - "activity": { - "id": "01M7TA2HA66FRY17YJN6HC6S55", - "idempotency_key": "01M7TA2HA66FRY17YJN6HC6S55", - "sequence": 4, - "type": "py307.ok", - "class": "py307.ok", - "local_activity": false, - "activity_type": "py307.ok", - "arguments": { - "codec": "avro", - "blob": "wwHioz3/VYAiNwwCBgAAAAAAAOA/AA==" - }, - "retry_policy": { - "snapshot_version": 1, - "max_attempts": 1, - "backoff_seconds": [], - "non_retryable_error_types": [], - "start_to_close_timeout": 60, - "schedule_to_start_timeout": null, - "schedule_to_close_timeout": null, - "heartbeat_timeout": null - }, - "status": "running", - "attempt_count": 1, - "closed_at": null, - "attempt_id": "01M7TA2HSPDGABA249E1XKK6EM", - "exception": null, - "payload_codec": "avro", - "queue": "py307-activities", - "started_at": "2026-11-19T16:26:12.662Z" - }, - "parallel_group_id": "parallel-activities:1:4", - "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, - "parallel_group_size": 4, - "parallel_group_index": 3, - "parallel_group_path": [ - { - "parallel_group_id": "parallel-activities:1:4", - "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, - "parallel_group_size": 4, - "parallel_group_index": 3 - } - ], - "activity_attempt_id": "01M7TA2HSPDGABA249E1XKK6EM", - "attempt_number": 1, - "task": { - "id": "01M7TA2HA77XPRQ2H05A6SRV55", - "type": "activity", - "status": "leased", - "repair_count": 0, - "available_at": "2026-11-19T16:26:12.167Z", - "leased_at": "2026-11-19T16:26:12.662Z", - "lease_owner": "py-worker-9b247caa", - "lease_expires_at": "2026-11-19T16:31:12.662Z", - "attempt_count": 1, - "queue": "py307-activities", - "connection": "sqlite" - } - } - }, - { - "sequence": 13, - "event_type": "ActivityCompleted", - "timestamp": "2026-11-19T16:26:13.261000Z", - "principal": null, - "payload": { - "activity_execution_id": "01M7TA2HA66FRY17YJN6HC6S55", - "activity_type": "py307.ok", - "activity_class": "py307.ok", - "sequence": 4, - "activity": { - "id": "01M7TA2HA66FRY17YJN6HC6S55", - "idempotency_key": "01M7TA2HA66FRY17YJN6HC6S55", - "sequence": 4, - "type": "py307.ok", - "class": "py307.ok", - "local_activity": false, - "activity_type": "py307.ok", - "arguments": { - "codec": "avro", - "blob": "wwHioz3/VYAiNwwCBgAAAAAAAOA/AA==" - }, - "retry_policy": { - "snapshot_version": 1, - "max_attempts": 1, - "backoff_seconds": [], - "non_retryable_error_types": [], - "start_to_close_timeout": 60, - "schedule_to_start_timeout": null, - "schedule_to_close_timeout": null, - "heartbeat_timeout": null - }, - "status": "completed", - "attempt_count": 1, - "closed_at": "2026-11-19T16:26:13.261Z", - "attempt_id": "01M7TA2HSPDGABA249E1XKK6EM", - "exception": null, - "payload_codec": "avro", - "queue": "py307-activities", - "started_at": "2026-11-19T16:26:12.662Z" - }, - "parallel_group_id": "parallel-activities:1:4", - "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, - "parallel_group_size": 4, - "parallel_group_index": 3, - "parallel_group_path": [ - { - "parallel_group_id": "parallel-activities:1:4", - "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, - "parallel_group_size": 4, - "parallel_group_index": 3 - } - ], - "activity_attempt_id": "01M7TA2HSPDGABA249E1XKK6EM", - "attempt_number": 1, - "result": { - "codec": "avro", - "blob": "wwHioz3/VYAiNwoEb2s=" - }, - "payload_codec": "avro", - "task": { - "id": "01M7TA2HA77XPRQ2H05A6SRV55", - "type": "activity", - "status": "leased", - "repair_count": 0, - "available_at": "2026-11-19T16:26:12.167Z", - "leased_at": "2026-11-19T16:26:12.662Z", - "lease_owner": "py-worker-9b247caa", - "lease_expires_at": "2026-11-19T16:31:12.724Z", - "attempt_count": 1, - "queue": "py307-activities", - "connection": "sqlite" - } - } - }, - { - "sequence": 14, - "event_type": "ActivityFailed", - "timestamp": "2026-11-19T16:26:14.004000Z", - "principal": null, - "payload": { - "activity_execution_id": "01M7TA2HA40T3VYAMYBJSGH18V", - "activity_type": "py307.fail", - "activity_class": "py307.fail", - "sequence": 1, - "activity": { - "id": "01M7TA2HA40T3VYAMYBJSGH18V", - "idempotency_key": "01M7TA2HA40T3VYAMYBJSGH18V", - "sequence": 1, - "type": "py307.fail", - "class": "py307.fail", - "local_activity": false, - "activity_type": "py307.fail", - "arguments": { - "codec": "avro", - "blob": "wwHioz3/VYAiNwwCBgAAAAAAAPg/AA==" - }, - "retry_policy": { - "snapshot_version": 1, - "max_attempts": 1, - "backoff_seconds": [], - "non_retryable_error_types": [], - "start_to_close_timeout": 60, - "schedule_to_start_timeout": null, - "schedule_to_close_timeout": null, - "heartbeat_timeout": null - }, - "status": "failed", - "attempt_count": 1, - "closed_at": "2026-11-19T16:26:14.003Z", - "attempt_id": "01M7TA2HK1H2Y7GACGE77Z49TH", - "exception": { - "code": 0, - "type": "RuntimeError", - "message": "py307 failed after 1.5" - }, - "payload_codec": "avro", - "queue": "py307-activities", - "started_at": "2026-11-19T16:26:12.449Z" - }, - "parallel_group_id": "parallel-activities:1:4", - "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, - "parallel_group_size": 4, - "parallel_group_index": 0, - "parallel_group_path": [ - { - "parallel_group_id": "parallel-activities:1:4", - "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, - "parallel_group_size": 4, - "parallel_group_index": 0 - } - ], - "activity_attempt_id": "01M7TA2HK1H2Y7GACGE77Z49TH", - "attempt_number": 1, - "failure_id": "01M7TA2K3KSSXE8NCV02N8BP2J", - "failure_category": "activity", - "non_retryable": false, - "exception_type": "RuntimeError", - "exception_class": "builtins.RuntimeError", - "message": "py307 failed after 1.5", - "code": 0, - "exception": { - "type": "RuntimeError", - "message": "py307 failed after 1.5", - "code": 0 - }, - "task": { - "id": "01M7TA2HA40T3VYAMYBJSGH18X", - "type": "activity", - "status": "leased", - "available_at": "2026-11-19T16:26:12.164Z", - "leased_at": "2026-11-19T16:26:12.448Z", - "lease_owner": "py-worker-9b247caa", - "lease_expires_at": "2026-11-19T16:31:12.471Z", - "attempt_count": 1, - "repair_count": 0, - "queue": "py307-activities", - "connection": "sqlite" - } - } - } - ], - "expected": { - "command_sequence": [ - { - "type": "record_side_effect" - }, - { - "type": "schedule_activity", - "activity_type": "py307.compensate", - "arguments": [ - "py307.fail#2" - ] - } - ] - } -} diff --git a/tests/fixtures/replay_regressions/settled-group-records-first-recorded-failure.json b/tests/fixtures/replay_regressions/settled-group-new-run-raises-first-recorded-failure.json similarity index 65% rename from tests/fixtures/replay_regressions/settled-group-records-first-recorded-failure.json rename to tests/fixtures/replay_regressions/settled-group-new-run-raises-first-recorded-failure.json index ac3f020e..a5c7eab5 100644 --- a/tests/fixtures/replay_regressions/settled-group-records-first-recorded-failure.json +++ b/tests/fixtures/replay_regressions/settled-group-new-run-raises-first-recorded-failure.json @@ -1,7 +1,7 @@ { "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", "fixture_schema": "durable-workflow.replay-regression/v1", - "id": "settled-group-records-first-recorded-failure", + "id": "settled-group-new-run-raises-first-recorded-failure", "protocol_version": "1.20", "bindings": [ "python" @@ -10,7 +10,7 @@ "issues": [ "durable-workflow/sdk-python#307" ], - "note": "Recorded by this version on native Server built from durable-workflow/server main 52e3ec00, same scenario. The task raised member 1, whose failure was recorded first, as Workflow's embedded engine does, recorded that decision at sequence 5 and scheduled the compensation at 6." + "note": "Recorded by this version on native Server built from durable-workflow/server main 52e3ec00, in the same scenario as settled-group-released-worker-keeps-position-rule. The run recorded the parallel failure-order marker at sequence 1 and its group at 2-5. Member 1 (sequence 3) failed first and member 0 later; every member had settled before the workflow task ran. The task raised member 1, the first recorded failure, recorded no decision and scheduled the compensation at sequence 6." }, "workflow": { "type": "py307.caught", @@ -25,29 +25,29 @@ { "sequence": 1, "event_type": "StartAccepted", - "timestamp": "2026-11-19T16:26:12.027000Z", + "timestamp": "2026-11-19T18:05:09.061000Z", "principal": { "type": "auth:token", "id": "legacy-token", "label": "Admin" }, "payload": { - "workflow_instance_id": "py307-caught-1795105571", - "workflow_run_id": "01M7TA2H5T1GTXE76ZS15AM165", + "workflow_instance_id": "py307-caught-1795111509", + "workflow_run_id": "01M7TFQQ24QMR4STW8W0DPZSY0", "workflow_type": "py307.caught", "workflow_class": "py307.caught", - "workflow_command_id": "01M7TA2H5T1GTXE76ZS15AM166", + "workflow_command_id": "01M7TFQQ24QMR4STW8W0DPZSY1", "memo": null, "business_key": null, "visibility_labels": null, "search_attributes": null, "outcome": "started_new", "command": { - "id": "01M7TA2H5T1GTXE76ZS15AM166", + "id": "01M7TFQQ24QMR4STW8W0DPZSY1", "sequence": 1, "type": "start", "target_scope": "instance", - "resolved_run_id": "01M7TA2H5T1GTXE76ZS15AM165", + "resolved_run_id": "01M7TFQQ24QMR4STW8W0DPZSY0", "payload_codec": "avro", "payload": { "codec": "avro", @@ -69,39 +69,39 @@ "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", - "request_fingerprint": "sha256:47edc9a9a51d6470d6a390ed2559a1fcda1699404c126e1e34ac6345d8076f0f", + "request_fingerprint": "sha256:b256f11f13e17899852f71ddb288b9b9b554465552359274dc1086461ebbd6c0", "status": "accepted", "outcome": "started_new", - "accepted_at": "2026-11-19T16:26:12.026000Z", - "applied_at": "2026-11-19T16:26:12.026000Z" + "accepted_at": "2026-11-19T18:05:09.060000Z", + "applied_at": "2026-11-19T18:05:09.060000Z" } } }, { "sequence": 2, "event_type": "WorkflowStarted", - "timestamp": "2026-11-19T16:26:12.028000Z", + "timestamp": "2026-11-19T18:05:09.064000Z", "principal": { "type": "auth:token", "id": "legacy-token", "label": "Admin" }, "payload": { - "workflow_instance_id": "py307-caught-1795105571", - "workflow_run_id": "01M7TA2H5T1GTXE76ZS15AM165", + "workflow_instance_id": "py307-caught-1795111509", + "workflow_run_id": "01M7TFQQ24QMR4STW8W0DPZSY0", "workflow_type": "py307.caught", "workflow_class": "py307.caught", - "workflow_command_id": "01M7TA2H5T1GTXE76ZS15AM166", + "workflow_command_id": "01M7TFQQ24QMR4STW8W0DPZSY1", "execution_timeout_seconds": 3600, "run_timeout_seconds": 600, - "execution_deadline_at": "2026-11-19T17:26:12.026Z", - "run_deadline_at": "2026-11-19T16:36:12.026Z", + "execution_deadline_at": "2026-11-19T19:05:09.060Z", + "run_deadline_at": "2026-11-19T18:15:09.060Z", "command": { - "id": "01M7TA2H5T1GTXE76ZS15AM166", + "id": "01M7TFQQ24QMR4STW8W0DPZSY1", "sequence": 1, "type": "start", "target_scope": "instance", - "resolved_run_id": "01M7TA2H5T1GTXE76ZS15AM165", + "resolved_run_id": "01M7TFQQ24QMR4STW8W0DPZSY0", "payload_codec": "avro", "payload": { "codec": "avro", @@ -123,28 +123,56 @@ "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", - "request_fingerprint": "sha256:47edc9a9a51d6470d6a390ed2559a1fcda1699404c126e1e34ac6345d8076f0f", + "request_fingerprint": "sha256:b256f11f13e17899852f71ddb288b9b9b554465552359274dc1086461ebbd6c0", "status": "accepted", "outcome": "started_new", - "accepted_at": "2026-11-19T16:26:12.026000Z", - "applied_at": "2026-11-19T16:26:12.026000Z" + "accepted_at": "2026-11-19T18:05:09.060000Z", + "applied_at": "2026-11-19T18:05:09.060000Z" } } }, { "sequence": 3, + "event_type": "VersionMarkerRecorded", + "timestamp": "2026-11-19T18:05:09.149000Z", + "principal": null, + "payload": { + "change_id": "durable_workflow.parallel_failure_order", + "version": 1, + "min_supported": 1, + "max_supported": 1, + "sequence": 1, + "task": { + "id": "01M7TFQQ28ZRZ8HR1R5D8GV6H7", + "type": "workflow", + "status": "leased", + "attempt_count": 1, + "repair_count": 0, + "queue": "py307-workflows", + "connection": "sqlite", + "available_at": "2026-11-19T18:05:09.064000Z", + "leased_at": "2026-11-19T18:05:09.094000Z", + "lease_expires_at": "2026-11-19T18:06:09.094000Z", + "last_dispatch_attempt_at": "2026-11-19T18:05:09.064000Z", + "last_dispatched_at": "2026-11-19T18:05:09.064000Z", + "lease_owner": "py-worker-7f4ce288" + } + } + }, + { + "sequence": 4, "event_type": "ActivityScheduled", - "timestamp": "2026-11-19T16:26:12.164000Z", + "timestamp": "2026-11-19T18:05:09.150000Z", "principal": null, "payload": { - "activity_execution_id": "01M7TA2HA40T3VYAMYBJSGH18V", + "activity_execution_id": "01M7TFQQ4XKB5ZFN4ETAA5G51E", "activity_type": "py307.fail", "activity_class": "py307.fail", - "sequence": 1, + "sequence": 2, "activity": { - "id": "01M7TA2HA40T3VYAMYBJSGH18V", - "idempotency_key": "01M7TA2HA40T3VYAMYBJSGH18V", - "sequence": 1, + "id": "01M7TFQQ4XKB5ZFN4ETAA5G51E", + "idempotency_key": "01M7TFQQ4XKB5ZFN4ETAA5G51E", + "sequence": 2, "type": "py307.fail", "class": "py307.fail", "local_activity": false, @@ -171,51 +199,51 @@ "payload_codec": "avro", "queue": "py307-activities" }, - "parallel_group_id": "parallel-activities:1:4", + "parallel_group_id": "parallel-activities:2:4", "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, + "parallel_group_base_sequence": 2, "parallel_group_size": 4, "parallel_group_index": 0, "parallel_group_path": [ { - "parallel_group_id": "parallel-activities:1:4", + "parallel_group_id": "parallel-activities:2:4", "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, + "parallel_group_base_sequence": 2, "parallel_group_size": 4, "parallel_group_index": 0 } ], "task": { - "id": "01M7TA2H5WNWMXXSWJ6JJ4M0GS", + "id": "01M7TFQQ28ZRZ8HR1R5D8GV6H7", "type": "workflow", "status": "leased", "attempt_count": 1, "repair_count": 0, "queue": "py307-workflows", "connection": "sqlite", - "available_at": "2026-11-19T16:26:12.028000Z", - "leased_at": "2026-11-19T16:26:12.086000Z", - "lease_expires_at": "2026-11-19T16:27:12.086000Z", - "last_dispatch_attempt_at": "2026-11-19T16:26:12.028000Z", - "last_dispatched_at": "2026-11-19T16:26:12.028000Z", - "lease_owner": "py-worker-89b000d4" + "available_at": "2026-11-19T18:05:09.064000Z", + "leased_at": "2026-11-19T18:05:09.094000Z", + "lease_expires_at": "2026-11-19T18:06:09.094000Z", + "last_dispatch_attempt_at": "2026-11-19T18:05:09.064000Z", + "last_dispatched_at": "2026-11-19T18:05:09.064000Z", + "lease_owner": "py-worker-7f4ce288" } } }, { - "sequence": 4, + "sequence": 5, "event_type": "ActivityScheduled", - "timestamp": "2026-11-19T16:26:12.165000Z", + "timestamp": "2026-11-19T18:05:09.151000Z", "principal": null, "payload": { - "activity_execution_id": "01M7TA2HA54J8KXYXJEJ7S4S5T", + "activity_execution_id": "01M7TFQQ4Z2ERNYZ3Z901FMZ2M", "activity_type": "py307.fail", "activity_class": "py307.fail", - "sequence": 2, + "sequence": 3, "activity": { - "id": "01M7TA2HA54J8KXYXJEJ7S4S5T", - "idempotency_key": "01M7TA2HA54J8KXYXJEJ7S4S5T", - "sequence": 2, + "id": "01M7TFQQ4Z2ERNYZ3Z901FMZ2M", + "idempotency_key": "01M7TFQQ4Z2ERNYZ3Z901FMZ2M", + "sequence": 3, "type": "py307.fail", "class": "py307.fail", "local_activity": false, @@ -242,51 +270,51 @@ "payload_codec": "avro", "queue": "py307-activities" }, - "parallel_group_id": "parallel-activities:1:4", + "parallel_group_id": "parallel-activities:2:4", "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, + "parallel_group_base_sequence": 2, "parallel_group_size": 4, "parallel_group_index": 1, "parallel_group_path": [ { - "parallel_group_id": "parallel-activities:1:4", + "parallel_group_id": "parallel-activities:2:4", "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, + "parallel_group_base_sequence": 2, "parallel_group_size": 4, "parallel_group_index": 1 } ], "task": { - "id": "01M7TA2H5WNWMXXSWJ6JJ4M0GS", + "id": "01M7TFQQ28ZRZ8HR1R5D8GV6H7", "type": "workflow", "status": "leased", "attempt_count": 1, "repair_count": 0, "queue": "py307-workflows", "connection": "sqlite", - "available_at": "2026-11-19T16:26:12.028000Z", - "leased_at": "2026-11-19T16:26:12.086000Z", - "lease_expires_at": "2026-11-19T16:27:12.086000Z", - "last_dispatch_attempt_at": "2026-11-19T16:26:12.028000Z", - "last_dispatched_at": "2026-11-19T16:26:12.028000Z", - "lease_owner": "py-worker-89b000d4" + "available_at": "2026-11-19T18:05:09.064000Z", + "leased_at": "2026-11-19T18:05:09.094000Z", + "lease_expires_at": "2026-11-19T18:06:09.094000Z", + "last_dispatch_attempt_at": "2026-11-19T18:05:09.064000Z", + "last_dispatched_at": "2026-11-19T18:05:09.064000Z", + "lease_owner": "py-worker-7f4ce288" } } }, { - "sequence": 5, + "sequence": 6, "event_type": "ActivityScheduled", - "timestamp": "2026-11-19T16:26:12.166000Z", + "timestamp": "2026-11-19T18:05:09.152000Z", "principal": null, "payload": { - "activity_execution_id": "01M7TA2HA66FRY17YJN6HC6S52", + "activity_execution_id": "01M7TFQQ50ZJS87NSM808PCJTC", "activity_type": "py307.ok", "activity_class": "py307.ok", - "sequence": 3, + "sequence": 4, "activity": { - "id": "01M7TA2HA66FRY17YJN6HC6S52", - "idempotency_key": "01M7TA2HA66FRY17YJN6HC6S52", - "sequence": 3, + "id": "01M7TFQQ50ZJS87NSM808PCJTC", + "idempotency_key": "01M7TFQQ50ZJS87NSM808PCJTC", + "sequence": 4, "type": "py307.ok", "class": "py307.ok", "local_activity": false, @@ -313,51 +341,51 @@ "payload_codec": "avro", "queue": "py307-activities" }, - "parallel_group_id": "parallel-activities:1:4", + "parallel_group_id": "parallel-activities:2:4", "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, + "parallel_group_base_sequence": 2, "parallel_group_size": 4, "parallel_group_index": 2, "parallel_group_path": [ { - "parallel_group_id": "parallel-activities:1:4", + "parallel_group_id": "parallel-activities:2:4", "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, + "parallel_group_base_sequence": 2, "parallel_group_size": 4, "parallel_group_index": 2 } ], "task": { - "id": "01M7TA2H5WNWMXXSWJ6JJ4M0GS", + "id": "01M7TFQQ28ZRZ8HR1R5D8GV6H7", "type": "workflow", "status": "leased", "attempt_count": 1, "repair_count": 0, "queue": "py307-workflows", "connection": "sqlite", - "available_at": "2026-11-19T16:26:12.028000Z", - "leased_at": "2026-11-19T16:26:12.086000Z", - "lease_expires_at": "2026-11-19T16:27:12.086000Z", - "last_dispatch_attempt_at": "2026-11-19T16:26:12.028000Z", - "last_dispatched_at": "2026-11-19T16:26:12.028000Z", - "lease_owner": "py-worker-89b000d4" + "available_at": "2026-11-19T18:05:09.064000Z", + "leased_at": "2026-11-19T18:05:09.094000Z", + "lease_expires_at": "2026-11-19T18:06:09.094000Z", + "last_dispatch_attempt_at": "2026-11-19T18:05:09.064000Z", + "last_dispatched_at": "2026-11-19T18:05:09.064000Z", + "lease_owner": "py-worker-7f4ce288" } } }, { - "sequence": 6, + "sequence": 7, "event_type": "ActivityScheduled", - "timestamp": "2026-11-19T16:26:12.166000Z", + "timestamp": "2026-11-19T18:05:09.153000Z", "principal": null, "payload": { - "activity_execution_id": "01M7TA2HA66FRY17YJN6HC6S55", + "activity_execution_id": "01M7TFQQ50ZJS87NSM808PCJTF", "activity_type": "py307.ok", "activity_class": "py307.ok", - "sequence": 4, + "sequence": 5, "activity": { - "id": "01M7TA2HA66FRY17YJN6HC6S55", - "idempotency_key": "01M7TA2HA66FRY17YJN6HC6S55", - "sequence": 4, + "id": "01M7TFQQ50ZJS87NSM808PCJTF", + "idempotency_key": "01M7TFQQ50ZJS87NSM808PCJTF", + "sequence": 5, "type": "py307.ok", "class": "py307.ok", "local_activity": false, @@ -384,51 +412,51 @@ "payload_codec": "avro", "queue": "py307-activities" }, - "parallel_group_id": "parallel-activities:1:4", + "parallel_group_id": "parallel-activities:2:4", "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, + "parallel_group_base_sequence": 2, "parallel_group_size": 4, "parallel_group_index": 3, "parallel_group_path": [ { - "parallel_group_id": "parallel-activities:1:4", + "parallel_group_id": "parallel-activities:2:4", "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, + "parallel_group_base_sequence": 2, "parallel_group_size": 4, "parallel_group_index": 3 } ], "task": { - "id": "01M7TA2H5WNWMXXSWJ6JJ4M0GS", + "id": "01M7TFQQ28ZRZ8HR1R5D8GV6H7", "type": "workflow", "status": "leased", "attempt_count": 1, "repair_count": 0, "queue": "py307-workflows", "connection": "sqlite", - "available_at": "2026-11-19T16:26:12.028000Z", - "leased_at": "2026-11-19T16:26:12.086000Z", - "lease_expires_at": "2026-11-19T16:27:12.086000Z", - "last_dispatch_attempt_at": "2026-11-19T16:26:12.028000Z", - "last_dispatched_at": "2026-11-19T16:26:12.028000Z", - "lease_owner": "py-worker-89b000d4" + "available_at": "2026-11-19T18:05:09.064000Z", + "leased_at": "2026-11-19T18:05:09.094000Z", + "lease_expires_at": "2026-11-19T18:06:09.094000Z", + "last_dispatch_attempt_at": "2026-11-19T18:05:09.064000Z", + "last_dispatched_at": "2026-11-19T18:05:09.064000Z", + "lease_owner": "py-worker-7f4ce288" } } }, { - "sequence": 7, + "sequence": 8, "event_type": "ActivityStarted", - "timestamp": "2026-11-19T16:26:12.449000Z", + "timestamp": "2026-11-19T18:05:09.442000Z", "principal": null, "payload": { - "activity_execution_id": "01M7TA2HA40T3VYAMYBJSGH18V", + "activity_execution_id": "01M7TFQQ4XKB5ZFN4ETAA5G51E", "activity_type": "py307.fail", "activity_class": "py307.fail", - "sequence": 1, + "sequence": 2, "activity": { - "id": "01M7TA2HA40T3VYAMYBJSGH18V", - "idempotency_key": "01M7TA2HA40T3VYAMYBJSGH18V", - "sequence": 1, + "id": "01M7TFQQ4XKB5ZFN4ETAA5G51E", + "idempotency_key": "01M7TFQQ4XKB5ZFN4ETAA5G51E", + "sequence": 2, "type": "py307.fail", "class": "py307.fail", "local_activity": false, @@ -450,37 +478,37 @@ "status": "running", "attempt_count": 1, "closed_at": null, - "attempt_id": "01M7TA2HK1H2Y7GACGE77Z49TH", + "attempt_id": "01M7TFQQE2YCP4Q0MBS339PJMD", "exception": null, "payload_codec": "avro", "queue": "py307-activities", - "started_at": "2026-11-19T16:26:12.449Z" + "started_at": "2026-11-19T18:05:09.442Z" }, - "parallel_group_id": "parallel-activities:1:4", + "parallel_group_id": "parallel-activities:2:4", "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, + "parallel_group_base_sequence": 2, "parallel_group_size": 4, "parallel_group_index": 0, "parallel_group_path": [ { - "parallel_group_id": "parallel-activities:1:4", + "parallel_group_id": "parallel-activities:2:4", "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, + "parallel_group_base_sequence": 2, "parallel_group_size": 4, "parallel_group_index": 0 } ], - "activity_attempt_id": "01M7TA2HK1H2Y7GACGE77Z49TH", + "activity_attempt_id": "01M7TFQQE2YCP4Q0MBS339PJMD", "attempt_number": 1, "task": { - "id": "01M7TA2HA40T3VYAMYBJSGH18X", + "id": "01M7TFQQ4YC26P3QVJ6E43ZSG9", "type": "activity", "status": "leased", "repair_count": 0, - "available_at": "2026-11-19T16:26:12.164Z", - "leased_at": "2026-11-19T16:26:12.448Z", - "lease_owner": "py-worker-9b247caa", - "lease_expires_at": "2026-11-19T16:31:12.448Z", + "available_at": "2026-11-19T18:05:09.150Z", + "leased_at": "2026-11-19T18:05:09.442Z", + "lease_owner": "py-worker-ffbfe129", + "lease_expires_at": "2026-11-19T18:10:09.442Z", "attempt_count": 1, "queue": "py307-activities", "connection": "sqlite" @@ -488,19 +516,19 @@ } }, { - "sequence": 8, + "sequence": 9, "event_type": "ActivityStarted", - "timestamp": "2026-11-19T16:26:12.497000Z", + "timestamp": "2026-11-19T18:05:09.485000Z", "principal": null, "payload": { - "activity_execution_id": "01M7TA2HA54J8KXYXJEJ7S4S5T", + "activity_execution_id": "01M7TFQQ4Z2ERNYZ3Z901FMZ2M", "activity_type": "py307.fail", "activity_class": "py307.fail", - "sequence": 2, + "sequence": 3, "activity": { - "id": "01M7TA2HA54J8KXYXJEJ7S4S5T", - "idempotency_key": "01M7TA2HA54J8KXYXJEJ7S4S5T", - "sequence": 2, + "id": "01M7TFQQ4Z2ERNYZ3Z901FMZ2M", + "idempotency_key": "01M7TFQQ4Z2ERNYZ3Z901FMZ2M", + "sequence": 3, "type": "py307.fail", "class": "py307.fail", "local_activity": false, @@ -522,37 +550,37 @@ "status": "running", "attempt_count": 1, "closed_at": null, - "attempt_id": "01M7TA2HMHNJVVHNEW678ZZ1KW", + "attempt_id": "01M7TFQQFDN00MCBYMY7WECF2J", "exception": null, "payload_codec": "avro", "queue": "py307-activities", - "started_at": "2026-11-19T16:26:12.497Z" + "started_at": "2026-11-19T18:05:09.485Z" }, - "parallel_group_id": "parallel-activities:1:4", + "parallel_group_id": "parallel-activities:2:4", "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, + "parallel_group_base_sequence": 2, "parallel_group_size": 4, "parallel_group_index": 1, "parallel_group_path": [ { - "parallel_group_id": "parallel-activities:1:4", + "parallel_group_id": "parallel-activities:2:4", "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, + "parallel_group_base_sequence": 2, "parallel_group_size": 4, "parallel_group_index": 1 } ], - "activity_attempt_id": "01M7TA2HMHNJVVHNEW678ZZ1KW", + "activity_attempt_id": "01M7TFQQFDN00MCBYMY7WECF2J", "attempt_number": 1, "task": { - "id": "01M7TA2HA54J8KXYXJEJ7S4S5W", + "id": "01M7TFQQ4Z2ERNYZ3Z901FMZ2P", "type": "activity", "status": "leased", "repair_count": 0, - "available_at": "2026-11-19T16:26:12.165Z", - "leased_at": "2026-11-19T16:26:12.496Z", - "lease_owner": "py-worker-9b247caa", - "lease_expires_at": "2026-11-19T16:31:12.496Z", + "available_at": "2026-11-19T18:05:09.151Z", + "leased_at": "2026-11-19T18:05:09.485Z", + "lease_owner": "py-worker-ffbfe129", + "lease_expires_at": "2026-11-19T18:10:09.485Z", "attempt_count": 1, "queue": "py307-activities", "connection": "sqlite" @@ -560,23 +588,23 @@ } }, { - "sequence": 9, - "event_type": "ActivityStarted", - "timestamp": "2026-11-19T16:26:12.567000Z", + "sequence": 10, + "event_type": "ActivityFailed", + "timestamp": "2026-11-19T18:05:09.517000Z", "principal": null, "payload": { - "activity_execution_id": "01M7TA2HA66FRY17YJN6HC6S52", - "activity_type": "py307.ok", - "activity_class": "py307.ok", + "activity_execution_id": "01M7TFQQ4Z2ERNYZ3Z901FMZ2M", + "activity_type": "py307.fail", + "activity_class": "py307.fail", "sequence": 3, "activity": { - "id": "01M7TA2HA66FRY17YJN6HC6S52", - "idempotency_key": "01M7TA2HA66FRY17YJN6HC6S52", + "id": "01M7TFQQ4Z2ERNYZ3Z901FMZ2M", + "idempotency_key": "01M7TFQQ4Z2ERNYZ3Z901FMZ2M", "sequence": 3, - "type": "py307.ok", - "class": "py307.ok", + "type": "py307.fail", + "class": "py307.fail", "local_activity": false, - "activity_type": "py307.ok", + "activity_type": "py307.fail", "arguments": { "codec": "avro", "blob": "wwHioz3/VYAiNwwCBAAA" @@ -591,64 +619,80 @@ "schedule_to_close_timeout": null, "heartbeat_timeout": null }, - "status": "running", + "status": "failed", "attempt_count": 1, - "closed_at": null, - "attempt_id": "01M7TA2HPQRHFPFHKK2D6JVY07", - "exception": null, + "closed_at": "2026-11-19T18:05:09.516Z", + "attempt_id": "01M7TFQQFDN00MCBYMY7WECF2J", + "exception": { + "code": 0, + "type": "RuntimeError", + "message": "py307 failed after 0" + }, "payload_codec": "avro", "queue": "py307-activities", - "started_at": "2026-11-19T16:26:12.567Z" + "started_at": "2026-11-19T18:05:09.485Z" }, - "parallel_group_id": "parallel-activities:1:4", + "parallel_group_id": "parallel-activities:2:4", "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, + "parallel_group_base_sequence": 2, "parallel_group_size": 4, - "parallel_group_index": 2, + "parallel_group_index": 1, "parallel_group_path": [ { - "parallel_group_id": "parallel-activities:1:4", + "parallel_group_id": "parallel-activities:2:4", "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, + "parallel_group_base_sequence": 2, "parallel_group_size": 4, - "parallel_group_index": 2 + "parallel_group_index": 1 } ], - "activity_attempt_id": "01M7TA2HPQRHFPFHKK2D6JVY07", + "activity_attempt_id": "01M7TFQQFDN00MCBYMY7WECF2J", "attempt_number": 1, + "failure_id": "01M7TFQQGC9VCYFHNHDGF51CFD", + "failure_category": "activity", + "non_retryable": false, + "exception_type": "RuntimeError", + "exception_class": "builtins.RuntimeError", + "message": "py307 failed after 0", + "code": 0, + "exception": { + "type": "RuntimeError", + "message": "py307 failed after 0", + "code": 0 + }, "task": { - "id": "01M7TA2HA66FRY17YJN6HC6S54", + "id": "01M7TFQQ4Z2ERNYZ3Z901FMZ2P", "type": "activity", "status": "leased", - "repair_count": 0, - "available_at": "2026-11-19T16:26:12.166Z", - "leased_at": "2026-11-19T16:26:12.567Z", - "lease_owner": "py-worker-9b247caa", - "lease_expires_at": "2026-11-19T16:31:12.567Z", + "available_at": "2026-11-19T18:05:09.151Z", + "leased_at": "2026-11-19T18:05:09.485Z", + "lease_owner": "py-worker-ffbfe129", + "lease_expires_at": "2026-11-19T18:10:09.500Z", "attempt_count": 1, + "repair_count": 0, "queue": "py307-activities", "connection": "sqlite" } } }, { - "sequence": 10, - "event_type": "ActivityFailed", - "timestamp": "2026-11-19T16:26:12.580000Z", + "sequence": 11, + "event_type": "ActivityStarted", + "timestamp": "2026-11-19T18:05:09.535000Z", "principal": null, "payload": { - "activity_execution_id": "01M7TA2HA54J8KXYXJEJ7S4S5T", - "activity_type": "py307.fail", - "activity_class": "py307.fail", - "sequence": 2, + "activity_execution_id": "01M7TFQQ50ZJS87NSM808PCJTC", + "activity_type": "py307.ok", + "activity_class": "py307.ok", + "sequence": 4, "activity": { - "id": "01M7TA2HA54J8KXYXJEJ7S4S5T", - "idempotency_key": "01M7TA2HA54J8KXYXJEJ7S4S5T", - "sequence": 2, - "type": "py307.fail", - "class": "py307.fail", + "id": "01M7TFQQ50ZJS87NSM808PCJTC", + "idempotency_key": "01M7TFQQ50ZJS87NSM808PCJTC", + "sequence": 4, + "type": "py307.ok", + "class": "py307.ok", "local_activity": false, - "activity_type": "py307.fail", + "activity_type": "py307.ok", "arguments": { "codec": "avro", "blob": "wwHioz3/VYAiNwwCBAAA" @@ -663,76 +707,60 @@ "schedule_to_close_timeout": null, "heartbeat_timeout": null }, - "status": "failed", + "status": "running", "attempt_count": 1, - "closed_at": "2026-11-19T16:26:12.579Z", - "attempt_id": "01M7TA2HMHNJVVHNEW678ZZ1KW", - "exception": { - "code": 0, - "type": "RuntimeError", - "message": "py307 failed after 0" - }, + "closed_at": null, + "attempt_id": "01M7TFQQGZQXY27HECMPV3V5K2", + "exception": null, "payload_codec": "avro", "queue": "py307-activities", - "started_at": "2026-11-19T16:26:12.497Z" + "started_at": "2026-11-19T18:05:09.535Z" }, - "parallel_group_id": "parallel-activities:1:4", + "parallel_group_id": "parallel-activities:2:4", "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, + "parallel_group_base_sequence": 2, "parallel_group_size": 4, - "parallel_group_index": 1, + "parallel_group_index": 2, "parallel_group_path": [ { - "parallel_group_id": "parallel-activities:1:4", + "parallel_group_id": "parallel-activities:2:4", "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, + "parallel_group_base_sequence": 2, "parallel_group_size": 4, - "parallel_group_index": 1 + "parallel_group_index": 2 } ], - "activity_attempt_id": "01M7TA2HMHNJVVHNEW678ZZ1KW", + "activity_attempt_id": "01M7TFQQGZQXY27HECMPV3V5K2", "attempt_number": 1, - "failure_id": "01M7TA2HQ4AZ38WZ2BVBJKHQKX", - "failure_category": "activity", - "non_retryable": false, - "exception_type": "RuntimeError", - "exception_class": "builtins.RuntimeError", - "message": "py307 failed after 0", - "code": 0, - "exception": { - "type": "RuntimeError", - "message": "py307 failed after 0", - "code": 0 - }, "task": { - "id": "01M7TA2HA54J8KXYXJEJ7S4S5W", + "id": "01M7TFQQ50ZJS87NSM808PCJTE", "type": "activity", "status": "leased", - "available_at": "2026-11-19T16:26:12.165Z", - "leased_at": "2026-11-19T16:26:12.496Z", - "lease_owner": "py-worker-9b247caa", - "lease_expires_at": "2026-11-19T16:31:12.524Z", - "attempt_count": 1, "repair_count": 0, + "available_at": "2026-11-19T18:05:09.152Z", + "leased_at": "2026-11-19T18:05:09.535Z", + "lease_owner": "py-worker-ffbfe129", + "lease_expires_at": "2026-11-19T18:10:09.535Z", + "attempt_count": 1, "queue": "py307-activities", "connection": "sqlite" } } }, { - "sequence": 11, + "sequence": 12, "event_type": "ActivityCompleted", - "timestamp": "2026-11-19T16:26:12.638000Z", + "timestamp": "2026-11-19T18:05:09.571000Z", "principal": null, "payload": { - "activity_execution_id": "01M7TA2HA66FRY17YJN6HC6S52", + "activity_execution_id": "01M7TFQQ50ZJS87NSM808PCJTC", "activity_type": "py307.ok", "activity_class": "py307.ok", - "sequence": 3, + "sequence": 4, "activity": { - "id": "01M7TA2HA66FRY17YJN6HC6S52", - "idempotency_key": "01M7TA2HA66FRY17YJN6HC6S52", - "sequence": 3, + "id": "01M7TFQQ50ZJS87NSM808PCJTC", + "idempotency_key": "01M7TFQQ50ZJS87NSM808PCJTC", + "sequence": 4, "type": "py307.ok", "class": "py307.ok", "local_activity": false, @@ -753,28 +781,28 @@ }, "status": "completed", "attempt_count": 1, - "closed_at": "2026-11-19T16:26:12.636Z", - "attempt_id": "01M7TA2HPQRHFPFHKK2D6JVY07", + "closed_at": "2026-11-19T18:05:09.570Z", + "attempt_id": "01M7TFQQGZQXY27HECMPV3V5K2", "exception": null, "payload_codec": "avro", "queue": "py307-activities", - "started_at": "2026-11-19T16:26:12.567Z" + "started_at": "2026-11-19T18:05:09.535Z" }, - "parallel_group_id": "parallel-activities:1:4", + "parallel_group_id": "parallel-activities:2:4", "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, + "parallel_group_base_sequence": 2, "parallel_group_size": 4, "parallel_group_index": 2, "parallel_group_path": [ { - "parallel_group_id": "parallel-activities:1:4", + "parallel_group_id": "parallel-activities:2:4", "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, + "parallel_group_base_sequence": 2, "parallel_group_size": 4, "parallel_group_index": 2 } ], - "activity_attempt_id": "01M7TA2HPQRHFPFHKK2D6JVY07", + "activity_attempt_id": "01M7TFQQGZQXY27HECMPV3V5K2", "attempt_number": 1, "result": { "codec": "avro", @@ -782,14 +810,14 @@ }, "payload_codec": "avro", "task": { - "id": "01M7TA2HA66FRY17YJN6HC6S54", + "id": "01M7TFQQ50ZJS87NSM808PCJTE", "type": "activity", "status": "leased", "repair_count": 0, - "available_at": "2026-11-19T16:26:12.166Z", - "leased_at": "2026-11-19T16:26:12.567Z", - "lease_owner": "py-worker-9b247caa", - "lease_expires_at": "2026-11-19T16:31:12.615Z", + "available_at": "2026-11-19T18:05:09.152Z", + "leased_at": "2026-11-19T18:05:09.535Z", + "lease_owner": "py-worker-ffbfe129", + "lease_expires_at": "2026-11-19T18:10:09.554Z", "attempt_count": 1, "queue": "py307-activities", "connection": "sqlite" @@ -797,19 +825,19 @@ } }, { - "sequence": 12, + "sequence": 13, "event_type": "ActivityStarted", - "timestamp": "2026-11-19T16:26:12.662000Z", + "timestamp": "2026-11-19T18:05:09.583000Z", "principal": null, "payload": { - "activity_execution_id": "01M7TA2HA66FRY17YJN6HC6S55", + "activity_execution_id": "01M7TFQQ50ZJS87NSM808PCJTF", "activity_type": "py307.ok", "activity_class": "py307.ok", - "sequence": 4, + "sequence": 5, "activity": { - "id": "01M7TA2HA66FRY17YJN6HC6S55", - "idempotency_key": "01M7TA2HA66FRY17YJN6HC6S55", - "sequence": 4, + "id": "01M7TFQQ50ZJS87NSM808PCJTF", + "idempotency_key": "01M7TFQQ50ZJS87NSM808PCJTF", + "sequence": 5, "type": "py307.ok", "class": "py307.ok", "local_activity": false, @@ -831,37 +859,37 @@ "status": "running", "attempt_count": 1, "closed_at": null, - "attempt_id": "01M7TA2HSPDGABA249E1XKK6EM", + "attempt_id": "01M7TFQQJFPY9V8YWKPYR2956N", "exception": null, "payload_codec": "avro", "queue": "py307-activities", - "started_at": "2026-11-19T16:26:12.662Z" + "started_at": "2026-11-19T18:05:09.583Z" }, - "parallel_group_id": "parallel-activities:1:4", + "parallel_group_id": "parallel-activities:2:4", "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, + "parallel_group_base_sequence": 2, "parallel_group_size": 4, "parallel_group_index": 3, "parallel_group_path": [ { - "parallel_group_id": "parallel-activities:1:4", + "parallel_group_id": "parallel-activities:2:4", "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, + "parallel_group_base_sequence": 2, "parallel_group_size": 4, "parallel_group_index": 3 } ], - "activity_attempt_id": "01M7TA2HSPDGABA249E1XKK6EM", + "activity_attempt_id": "01M7TFQQJFPY9V8YWKPYR2956N", "attempt_number": 1, "task": { - "id": "01M7TA2HA77XPRQ2H05A6SRV55", + "id": "01M7TFQQ51E1J6XTDBJ8FZN2YK", "type": "activity", "status": "leased", "repair_count": 0, - "available_at": "2026-11-19T16:26:12.167Z", - "leased_at": "2026-11-19T16:26:12.662Z", - "lease_owner": "py-worker-9b247caa", - "lease_expires_at": "2026-11-19T16:31:12.662Z", + "available_at": "2026-11-19T18:05:09.153Z", + "leased_at": "2026-11-19T18:05:09.583Z", + "lease_owner": "py-worker-ffbfe129", + "lease_expires_at": "2026-11-19T18:10:09.583Z", "attempt_count": 1, "queue": "py307-activities", "connection": "sqlite" @@ -869,19 +897,19 @@ } }, { - "sequence": 13, + "sequence": 14, "event_type": "ActivityCompleted", - "timestamp": "2026-11-19T16:26:13.261000Z", + "timestamp": "2026-11-19T18:05:10.130000Z", "principal": null, "payload": { - "activity_execution_id": "01M7TA2HA66FRY17YJN6HC6S55", + "activity_execution_id": "01M7TFQQ50ZJS87NSM808PCJTF", "activity_type": "py307.ok", "activity_class": "py307.ok", - "sequence": 4, + "sequence": 5, "activity": { - "id": "01M7TA2HA66FRY17YJN6HC6S55", - "idempotency_key": "01M7TA2HA66FRY17YJN6HC6S55", - "sequence": 4, + "id": "01M7TFQQ50ZJS87NSM808PCJTF", + "idempotency_key": "01M7TFQQ50ZJS87NSM808PCJTF", + "sequence": 5, "type": "py307.ok", "class": "py307.ok", "local_activity": false, @@ -902,28 +930,28 @@ }, "status": "completed", "attempt_count": 1, - "closed_at": "2026-11-19T16:26:13.261Z", - "attempt_id": "01M7TA2HSPDGABA249E1XKK6EM", + "closed_at": "2026-11-19T18:05:10.129Z", + "attempt_id": "01M7TFQQJFPY9V8YWKPYR2956N", "exception": null, "payload_codec": "avro", "queue": "py307-activities", - "started_at": "2026-11-19T16:26:12.662Z" + "started_at": "2026-11-19T18:05:09.583Z" }, - "parallel_group_id": "parallel-activities:1:4", + "parallel_group_id": "parallel-activities:2:4", "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, + "parallel_group_base_sequence": 2, "parallel_group_size": 4, "parallel_group_index": 3, "parallel_group_path": [ { - "parallel_group_id": "parallel-activities:1:4", + "parallel_group_id": "parallel-activities:2:4", "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, + "parallel_group_base_sequence": 2, "parallel_group_size": 4, "parallel_group_index": 3 } ], - "activity_attempt_id": "01M7TA2HSPDGABA249E1XKK6EM", + "activity_attempt_id": "01M7TFQQJFPY9V8YWKPYR2956N", "attempt_number": 1, "result": { "codec": "avro", @@ -931,14 +959,14 @@ }, "payload_codec": "avro", "task": { - "id": "01M7TA2HA77XPRQ2H05A6SRV55", + "id": "01M7TFQQ51E1J6XTDBJ8FZN2YK", "type": "activity", "status": "leased", "repair_count": 0, - "available_at": "2026-11-19T16:26:12.167Z", - "leased_at": "2026-11-19T16:26:12.662Z", - "lease_owner": "py-worker-9b247caa", - "lease_expires_at": "2026-11-19T16:31:12.724Z", + "available_at": "2026-11-19T18:05:09.153Z", + "leased_at": "2026-11-19T18:05:09.583Z", + "lease_owner": "py-worker-ffbfe129", + "lease_expires_at": "2026-11-19T18:10:09.614Z", "attempt_count": 1, "queue": "py307-activities", "connection": "sqlite" @@ -946,19 +974,19 @@ } }, { - "sequence": 14, + "sequence": 15, "event_type": "ActivityFailed", - "timestamp": "2026-11-19T16:26:14.004000Z", + "timestamp": "2026-11-19T18:05:10.986000Z", "principal": null, "payload": { - "activity_execution_id": "01M7TA2HA40T3VYAMYBJSGH18V", + "activity_execution_id": "01M7TFQQ4XKB5ZFN4ETAA5G51E", "activity_type": "py307.fail", "activity_class": "py307.fail", - "sequence": 1, + "sequence": 2, "activity": { - "id": "01M7TA2HA40T3VYAMYBJSGH18V", - "idempotency_key": "01M7TA2HA40T3VYAMYBJSGH18V", - "sequence": 1, + "id": "01M7TFQQ4XKB5ZFN4ETAA5G51E", + "idempotency_key": "01M7TFQQ4XKB5ZFN4ETAA5G51E", + "sequence": 2, "type": "py307.fail", "class": "py307.fail", "local_activity": false, @@ -979,8 +1007,8 @@ }, "status": "failed", "attempt_count": 1, - "closed_at": "2026-11-19T16:26:14.003Z", - "attempt_id": "01M7TA2HK1H2Y7GACGE77Z49TH", + "closed_at": "2026-11-19T18:05:10.986Z", + "attempt_id": "01M7TFQQE2YCP4Q0MBS339PJMD", "exception": { "code": 0, "type": "RuntimeError", @@ -988,25 +1016,25 @@ }, "payload_codec": "avro", "queue": "py307-activities", - "started_at": "2026-11-19T16:26:12.449Z" + "started_at": "2026-11-19T18:05:09.442Z" }, - "parallel_group_id": "parallel-activities:1:4", + "parallel_group_id": "parallel-activities:2:4", "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, + "parallel_group_base_sequence": 2, "parallel_group_size": 4, "parallel_group_index": 0, "parallel_group_path": [ { - "parallel_group_id": "parallel-activities:1:4", + "parallel_group_id": "parallel-activities:2:4", "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, + "parallel_group_base_sequence": 2, "parallel_group_size": 4, "parallel_group_index": 0 } ], - "activity_attempt_id": "01M7TA2HK1H2Y7GACGE77Z49TH", + "activity_attempt_id": "01M7TFQQE2YCP4Q0MBS339PJMD", "attempt_number": 1, - "failure_id": "01M7TA2K3KSSXE8NCV02N8BP2J", + "failure_id": "01M7TFQRYADR26Y4NPG6BSV5AB", "failure_category": "activity", "non_retryable": false, "exception_type": "RuntimeError", @@ -1019,13 +1047,13 @@ "code": 0 }, "task": { - "id": "01M7TA2HA40T3VYAMYBJSGH18X", + "id": "01M7TFQQ4YC26P3QVJ6E43ZSG9", "type": "activity", "status": "leased", - "available_at": "2026-11-19T16:26:12.164Z", - "leased_at": "2026-11-19T16:26:12.448Z", - "lease_owner": "py-worker-9b247caa", - "lease_expires_at": "2026-11-19T16:31:12.471Z", + "available_at": "2026-11-19T18:05:09.150Z", + "leased_at": "2026-11-19T18:05:09.442Z", + "lease_owner": "py-worker-ffbfe129", + "lease_expires_at": "2026-11-19T18:10:09.472Z", "attempt_count": 1, "repair_count": 0, "queue": "py307-activities", @@ -1033,47 +1061,19 @@ } } }, - { - "sequence": 15, - "event_type": "SideEffectRecorded", - "timestamp": "2026-11-19T16:26:15.406000Z", - "principal": null, - "payload": { - "sequence": 5, - "result": { - "codec": "avro", - "blob": "wwHioz3/VYAiNw4CVGR1cmFibGVfd29ya2Zsb3cucGFyYWxsZWxfZmFpbHVyZV9kZWNpc2lvbg4IInBhcmFsbGVsX2dyb3VwX2lkCi5wYXJhbGxlbC1hY3Rpdml0aWVzOjE6NDhwYXJhbGxlbF9ncm91cF9iYXNlX3NlcXVlbmNlBAImcGFyYWxsZWxfZ3JvdXBfc2l6ZQQIHm1lbWJlcl9zZXF1ZW5jZQQEAAA=" - }, - "task": { - "id": "01M7TA2HQ5FCX8G18Y9FJDHP25", - "type": "workflow", - "status": "leased", - "attempt_count": 1, - "repair_count": 0, - "queue": "py307-workflows", - "connection": "sqlite", - "available_at": "2026-11-19T16:26:12.581000Z", - "leased_at": "2026-11-19T16:26:15.354000Z", - "lease_expires_at": "2026-11-19T16:27:15.354000Z", - "last_dispatch_attempt_at": "2026-11-19T16:26:12.581000Z", - "last_dispatched_at": "2026-11-19T16:26:12.581000Z", - "lease_owner": "py-worker-4a68eb6b" - } - } - }, { "sequence": 16, "event_type": "ActivityScheduled", - "timestamp": "2026-11-19T16:26:15.408000Z", + "timestamp": "2026-11-19T18:05:12.343000Z", "principal": null, "payload": { - "activity_execution_id": "01M7TA2MFF9GAY7D9X6VPGKEFX", + "activity_execution_id": "01M7TFQT8Q0J3NVF3ZG93C0JY4", "activity_type": "py307.compensate", "activity_class": "py307.compensate", "sequence": 6, "activity": { - "id": "01M7TA2MFF9GAY7D9X6VPGKEFX", - "idempotency_key": "01M7TA2MFF9GAY7D9X6VPGKEFX", + "id": "01M7TFQT8Q0J3NVF3ZG93C0JY4", + "idempotency_key": "01M7TFQT8Q0J3NVF3ZG93C0JY4", "sequence": 6, "type": "py307.compensate", "class": "py307.compensate", @@ -1081,7 +1081,7 @@ "activity_type": "py307.compensate", "arguments": { "codec": "avro", - "blob": "wwHioz3/VYAiNwwCChhweTMwNy5mYWlsIzIA" + "blob": "wwHioz3/VYAiNwwCChhweTMwNy5mYWlsIzMA" }, "retry_policy": { "snapshot_version": 1, @@ -1102,35 +1102,35 @@ "queue": "py307-activities" }, "task": { - "id": "01M7TA2HQ5FCX8G18Y9FJDHP25", + "id": "01M7TFQQGDT7HPMJ16KQEZYT92", "type": "workflow", "status": "leased", "attempt_count": 1, "repair_count": 0, "queue": "py307-workflows", "connection": "sqlite", - "available_at": "2026-11-19T16:26:12.581000Z", - "leased_at": "2026-11-19T16:26:15.354000Z", - "lease_expires_at": "2026-11-19T16:27:15.354000Z", - "last_dispatch_attempt_at": "2026-11-19T16:26:12.581000Z", - "last_dispatched_at": "2026-11-19T16:26:12.581000Z", - "lease_owner": "py-worker-4a68eb6b" + "available_at": "2026-11-19T18:05:09.517000Z", + "leased_at": "2026-11-19T18:05:12.328000Z", + "lease_expires_at": "2026-11-19T18:06:12.328000Z", + "last_dispatch_attempt_at": "2026-11-19T18:05:09.517000Z", + "last_dispatched_at": "2026-11-19T18:05:09.517000Z", + "lease_owner": "py-worker-f5b4afa0" } } }, { "sequence": 17, "event_type": "ActivityStarted", - "timestamp": "2026-11-19T16:26:15.459000Z", + "timestamp": "2026-11-19T18:05:12.374000Z", "principal": null, "payload": { - "activity_execution_id": "01M7TA2MFF9GAY7D9X6VPGKEFX", + "activity_execution_id": "01M7TFQT8Q0J3NVF3ZG93C0JY4", "activity_type": "py307.compensate", "activity_class": "py307.compensate", "sequence": 6, "activity": { - "id": "01M7TA2MFF9GAY7D9X6VPGKEFX", - "idempotency_key": "01M7TA2MFF9GAY7D9X6VPGKEFX", + "id": "01M7TFQT8Q0J3NVF3ZG93C0JY4", + "idempotency_key": "01M7TFQT8Q0J3NVF3ZG93C0JY4", "sequence": 6, "type": "py307.compensate", "class": "py307.compensate", @@ -1138,7 +1138,7 @@ "activity_type": "py307.compensate", "arguments": { "codec": "avro", - "blob": "wwHioz3/VYAiNwwCChhweTMwNy5mYWlsIzIA" + "blob": "wwHioz3/VYAiNwwCChhweTMwNy5mYWlsIzMA" }, "retry_policy": { "snapshot_version": 1, @@ -1153,23 +1153,23 @@ "status": "running", "attempt_count": 1, "closed_at": null, - "attempt_id": "01M7TA2MH3ZS1380G4XZQY3RJA", + "attempt_id": "01M7TFQT9P86V8FW7A5JTZKE55", "exception": null, "payload_codec": "avro", "queue": "py307-activities", - "started_at": "2026-11-19T16:26:15.459Z" + "started_at": "2026-11-19T18:05:12.374Z" }, - "activity_attempt_id": "01M7TA2MH3ZS1380G4XZQY3RJA", + "activity_attempt_id": "01M7TFQT9P86V8FW7A5JTZKE55", "attempt_number": 1, "task": { - "id": "01M7TA2MFG4KJAR1B08YR7ZMNW", + "id": "01M7TFQT8Q0J3NVF3ZG93C0JY6", "type": "activity", "status": "leased", "repair_count": 0, - "available_at": "2026-11-19T16:26:15.408Z", - "leased_at": "2026-11-19T16:26:15.458Z", - "lease_owner": "py-worker-9b247caa", - "lease_expires_at": "2026-11-19T16:31:15.458Z", + "available_at": "2026-11-19T18:05:12.343Z", + "leased_at": "2026-11-19T18:05:12.374Z", + "lease_owner": "py-worker-ffbfe129", + "lease_expires_at": "2026-11-19T18:10:12.373Z", "attempt_count": 1, "queue": "py307-activities", "connection": "sqlite" @@ -1179,16 +1179,16 @@ { "sequence": 18, "event_type": "ActivityCompleted", - "timestamp": "2026-11-19T16:26:15.520000Z", + "timestamp": "2026-11-19T18:05:12.401000Z", "principal": null, "payload": { - "activity_execution_id": "01M7TA2MFF9GAY7D9X6VPGKEFX", + "activity_execution_id": "01M7TFQT8Q0J3NVF3ZG93C0JY4", "activity_type": "py307.compensate", "activity_class": "py307.compensate", "sequence": 6, "activity": { - "id": "01M7TA2MFF9GAY7D9X6VPGKEFX", - "idempotency_key": "01M7TA2MFF9GAY7D9X6VPGKEFX", + "id": "01M7TFQT8Q0J3NVF3ZG93C0JY4", + "idempotency_key": "01M7TFQT8Q0J3NVF3ZG93C0JY4", "sequence": 6, "type": "py307.compensate", "class": "py307.compensate", @@ -1196,7 +1196,7 @@ "activity_type": "py307.compensate", "arguments": { "codec": "avro", - "blob": "wwHioz3/VYAiNwwCChhweTMwNy5mYWlsIzIA" + "blob": "wwHioz3/VYAiNwwCChhweTMwNy5mYWlsIzMA" }, "retry_policy": { "snapshot_version": 1, @@ -1210,29 +1210,29 @@ }, "status": "completed", "attempt_count": 1, - "closed_at": "2026-11-19T16:26:15.520Z", - "attempt_id": "01M7TA2MH3ZS1380G4XZQY3RJA", + "closed_at": "2026-11-19T18:05:12.401Z", + "attempt_id": "01M7TFQT9P86V8FW7A5JTZKE55", "exception": null, "payload_codec": "avro", "queue": "py307-activities", - "started_at": "2026-11-19T16:26:15.459Z" + "started_at": "2026-11-19T18:05:12.374Z" }, - "activity_attempt_id": "01M7TA2MH3ZS1380G4XZQY3RJA", + "activity_attempt_id": "01M7TFQT9P86V8FW7A5JTZKE55", "attempt_number": 1, "result": { "codec": "avro", - "blob": "wwHioz3/VYAiNwomdW5kb25lOnB5MzA3LmZhaWwjMg==" + "blob": "wwHioz3/VYAiNwomdW5kb25lOnB5MzA3LmZhaWwjMw==" }, "payload_codec": "avro", "task": { - "id": "01M7TA2MFG4KJAR1B08YR7ZMNW", + "id": "01M7TFQT8Q0J3NVF3ZG93C0JY6", "type": "activity", "status": "leased", "repair_count": 0, - "available_at": "2026-11-19T16:26:15.408Z", - "leased_at": "2026-11-19T16:26:15.458Z", - "lease_owner": "py-worker-9b247caa", - "lease_expires_at": "2026-11-19T16:31:15.493Z", + "available_at": "2026-11-19T18:05:12.343Z", + "leased_at": "2026-11-19T18:05:12.374Z", + "lease_owner": "py-worker-ffbfe129", + "lease_expires_at": "2026-11-19T18:10:12.388Z", "attempt_count": 1, "queue": "py307-activities", "connection": "sqlite" @@ -1242,7 +1242,7 @@ { "sequence": 19, "event_type": "WorkflowCompleted", - "timestamp": "2026-11-19T16:26:15.577000Z", + "timestamp": "2026-11-19T18:05:12.455000Z", "principal": { "type": "auth:token", "id": "legacy-token", @@ -1251,23 +1251,23 @@ "payload": { "output": { "codec": "avro", - "blob": "wwHioz3/VYAiNwomdW5kb25lOnB5MzA3LmZhaWwjMg==" + "blob": "wwHioz3/VYAiNwomdW5kb25lOnB5MzA3LmZhaWwjMw==" }, "payload_codec": "avro", "task": { - "id": "01M7TA2MK03BFHYXNRNDTRSBEE", + "id": "01M7TFQTAJNPVR5A2NEED7QHGZ", "type": "workflow", "status": "leased", "attempt_count": 1, "repair_count": 0, "queue": "py307-workflows", "connection": "sqlite", - "available_at": "2026-11-19T16:26:15.520000Z", - "leased_at": "2026-11-19T16:26:15.538000Z", - "lease_expires_at": "2026-11-19T16:27:15.538000Z", - "last_dispatch_attempt_at": "2026-11-19T16:26:15.520000Z", - "last_dispatched_at": "2026-11-19T16:26:15.520000Z", - "lease_owner": "py-worker-4a68eb6b" + "available_at": "2026-11-19T18:05:12.402000Z", + "leased_at": "2026-11-19T18:05:12.424000Z", + "lease_expires_at": "2026-11-19T18:06:12.424000Z", + "last_dispatch_attempt_at": "2026-11-19T18:05:12.402000Z", + "last_dispatched_at": "2026-11-19T18:05:12.402000Z", + "lease_owner": "py-worker-f5b4afa0" }, "command": { "type": "complete_workflow", @@ -1280,6 +1280,6 @@ } ], "expected": { - "query_result": "py307.fail#2" + "query_result": "py307.fail#3" } } diff --git a/tests/fixtures/replay_regressions/settled-group-raises-first-recorded-failure.json b/tests/fixtures/replay_regressions/settled-group-raises-first-recorded-failure.json deleted file mode 100644 index a704200b..00000000 --- a/tests/fixtures/replay_regressions/settled-group-raises-first-recorded-failure.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", - "fixture_schema": "durable-workflow.replay-regression/v1", - "id": "settled-group-raises-first-recorded-failure", - "protocol_version": "1.20", - "bindings": [ - "python" - ], - "settled_group_first_recorded_failure": { - "issues": [ - "durable-workflow/sdk-python#307" - ], - "note": "The history of business-clock-parallel-failure-selection-gap: both members failed before a task reached the group, member 2 first. That fixture expected the lower position (member 1) with no clock, because member 1's failure was not the earliest. Like Workflow's embedded engine, the task now raises the first recorded failure, with that failure's clock, and records which member it raised because released workers would have chosen another member. A run that a released worker already moved past keeps the position rule." - }, - "workflow": { - "type": "tests.replay.business-clock-parallel-failure-gap", - "input": [], - "payload_codec": "avro", - "workflow_id": "business-clock-parallel-failure-selection-gap", - "run_id": "business-clock-parallel-failure-selection-gap-run" - }, - "history": [ - { - "event_type": "WorkflowStarted", - "recorded_at": "2026-01-01T00:00:00.000000+00:00", - "payload": {}, - "sequence": 1 - }, - { - "event_type": "ActivityScheduled", - "recorded_at": "2026-01-01T00:00:00.000000+00:00", - "payload": { - "sequence": 1, - "activity_execution_id": "group-execution-0", - "activity_type": "group.0", - "parallel_group_id": "parallel-activities:1:2", - "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, - "parallel_group_size": 2, - "parallel_group_index": 0, - "parallel_group_path": [ - { - "parallel_group_id": "parallel-activities:1:2", - "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, - "parallel_group_size": 2, - "parallel_group_index": 0 - } - ] - }, - "sequence": 2 - }, - { - "event_type": "ActivityScheduled", - "recorded_at": "2026-01-01T00:00:00.000000+00:00", - "payload": { - "sequence": 2, - "activity_execution_id": "group-execution-1", - "activity_type": "group.1", - "parallel_group_id": "parallel-activities:1:2", - "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, - "parallel_group_size": 2, - "parallel_group_index": 1, - "parallel_group_path": [ - { - "parallel_group_id": "parallel-activities:1:2", - "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, - "parallel_group_size": 2, - "parallel_group_index": 1 - } - ] - }, - "sequence": 3 - }, - { - "event_type": "ActivityFailed", - "recorded_at": "2026-01-01T00:00:01Z", - "payload": { - "sequence": 2, - "activity_execution_id": "group-execution-1", - "activity_type": "group.1", - "parallel_group_id": "parallel-activities:1:2", - "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, - "parallel_group_size": 2, - "parallel_group_index": 1, - "parallel_group_path": [ - { - "parallel_group_id": "parallel-activities:1:2", - "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, - "parallel_group_size": 2, - "parallel_group_index": 1 - } - ], - "message": "failure-1" - }, - "sequence": 4 - }, - { - "event_type": "ActivityFailed", - "recorded_at": "2026-01-01T00:00:03.123100+00:00", - "payload": { - "sequence": 1, - "activity_execution_id": "group-execution-0", - "activity_type": "group.0", - "parallel_group_id": "parallel-activities:1:2", - "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, - "parallel_group_size": 2, - "parallel_group_index": 0, - "parallel_group_path": [ - { - "parallel_group_id": "parallel-activities:1:2", - "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, - "parallel_group_size": 2, - "parallel_group_index": 0 - } - ], - "message": "failure-0" - }, - "sequence": 5 - } - ], - "expected": { - "command_sequence": [ - { - "type": "record_side_effect" - }, - { - "type": "complete_workflow", - "result": { - "activity_type": "group.1", - "message": "failure-1", - "clock": "2026-01-01T00:00:01.000000+00:00", - "legacy": "2026-01-01T00:00:00.000000+00:00" - } - } - ] - } -} diff --git a/tests/test_activity_cancellation_policies.py b/tests/test_activity_cancellation_policies.py index f08dfdb3..c5c667c3 100644 --- a/tests/test_activity_cancellation_policies.py +++ b/tests/test_activity_cancellation_policies.py @@ -15,6 +15,7 @@ commands_to_server_commands, replay, ) +from tests.failure_order import failure_order_marker_event, without_failure_order_marker from tests.test_cooperative_cancellation import marker, request from tests.test_cooperative_cancellation_worker import ClaimServer, claimed_task @@ -78,9 +79,13 @@ def test_omitted_policy_preserves_wire_and_historical_try_default() -> None: def test_changed_policy_fails_group_matching_and_cancellation_delivery(mode: str, delivered: bool) -> None: options = {"cancellation_policy": CancellationPolicy.WAIT_CANCELLATION_COMPLETED} commands = commands_to_server_commands(replay(workflow(options, mode), [], []).commands, "queue") - history = [{"event_type": "ActivityScheduled", "payload": {"sequence": 1, **command}} for command in commands] + # A new run records its failure-order marker before its first all-group. + first = 2 if mode == "parallel" else 1 + history = [failure_order_marker_event()] if mode == "parallel" else [] + history += [{"event_type": "ActivityScheduled", "payload": {"sequence": first, **command}} + for command in without_failure_order_marker(commands)] if delivered: - history += [request(), marker(1, "activity" if mode == "sequential" else "parallel", sequence_span=1)] + history += [request(), marker(first, "activity" if mode == "sequential" else "parallel", sequence_span=1)] assert replay(workflow(options, mode), history, [], run_id="run-1").commands == [CompleteWorkflow("request-1")] with pytest.raises(NonDeterministicReplayError, match="activity_cancellation_policy_changed"): replay(workflow({"cancellation_policy": CancellationPolicy.TRY_CANCEL}, mode), history, [], run_id="run-1") diff --git a/tests/test_business_replay_clock_parallel.py b/tests/test_business_replay_clock_parallel.py index 3821a1f1..cd0f6ce0 100644 --- a/tests/test_business_replay_clock_parallel.py +++ b/tests/test_business_replay_clock_parallel.py @@ -13,13 +13,7 @@ from durable_workflow import serializer, workflow from durable_workflow.errors import ActivityFailed, NonDeterministicReplayError, TypedValueError from durable_workflow.replay_clock_math import canonical_milliseconds, from_canonical_milliseconds -from durable_workflow.workflow import ( - CompleteWorkflow, - RecordSideEffect, - WorkflowContext, - query_state, - replay, -) +from durable_workflow.workflow import WorkflowContext, query_state, replay from tests.business_clock_workflows import clock_text from tests.test_business_replay_clock import result @@ -80,37 +74,12 @@ def test_canonical_v1_authored_comparison_and_selected_failure_vectors(case: dic value = result(history(case["times"], order=(1, 0)), ParallelProbe) expected = datetime.fromisoformat(case["times"][case["success_member"]].replace("Z", "+00:00")) assert value["clock"] == expected.isoformat(timespec="microseconds") - rows = history(case["times"], failures={0, 1}, order=(1, 0)) - failure = result(closed(rows), ParallelProbe) + failure = result(history(case["times"], failures={0, 1}, order=(1, 0)), ParallelProbe) assert failure["failure"] == ("group.0", "failure-0") time = case["failure_time"] assert failure["clock"] == (datetime.fromisoformat(time.replace("Z", "+00:00")).isoformat( timespec="microseconds", ) if time is not None else None) - now = frontier_result(rows) - assert now["failure"] == ("group.1", "failure-1") - assert now["clock"] == canonical_text(case["times"][1]) - - -def closed(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: - """The history as a released worker left it: the run closed after the group.""" - return [*rows, {"event_type": "WorkflowCompleted", "recorded_at": "2099-01-01T00:00:00Z", "payload": {}}] - - -def frontier_result(rows: list[dict[str, Any]]) -> Any: - """A task reaching a settled group whose first recorded failure is not the - lowest position raises that failure and records it (sdk-python#307).""" - outcome = replay(ParallelProbe, rows, [], payload_codec="avro") - try: - assert [type(command) for command in outcome.commands] == [RecordSideEffect, CompleteWorkflow] - return outcome.commands[1].result - finally: - outcome.close() - - -def canonical_text(time: str) -> str | None: - instant = from_canonical_milliseconds(canonical_milliseconds(datetime.fromisoformat(time.replace("Z", "+00:00")))) - return instant.isoformat(timespec="microseconds") if instant is not None else None def test_reconstruction_overflow_refuses_time_without_changing_the_selected_failure() -> None: @@ -226,13 +195,9 @@ def test_caught_failure_ignores_successful_sibling_time_and_reconstructs_canonic def test_multiple_failures_preserve_legacy_winner_and_refuse_a_different_canonical_winner( times: list[str], expected: str | None, ) -> None: - rows = history(times, failures={0, 1}, order=(1, 0)) - value = result(closed(rows), ParallelProbe) + value = result(history(times, failures={0, 1}, order=(1, 0)), ParallelProbe) assert value["failure"] == ("group.0", "failure-0") assert value["clock"] == expected - now = frontier_result(rows) - assert now["failure"] == ("group.1", "failure-1") - assert now["clock"] == canonical_text(times[1]) @pytest.mark.parametrize("order", list(permutations(range(3)))) diff --git a/tests/test_cancellation_scope_authoring.py b/tests/test_cancellation_scope_authoring.py index 02c30a77..4d15388e 100644 --- a/tests/test_cancellation_scope_authoring.py +++ b/tests/test_cancellation_scope_authoring.py @@ -12,6 +12,7 @@ from durable_workflow.errors import NonDeterministicReplayError from durable_workflow.worker import Worker from durable_workflow.workflow import LocalActivityExecutionAborted +from tests.failure_order import is_failure_order_marker from tests.test_cooperative_cancellation import request @@ -147,10 +148,12 @@ def body(): # type: ignore[no-untyped-def] return (yield from ctx.cancellation_scope(body)) outcome = run(Probe, [started(), opening()]) - wire = workflow.commands_to_server_commands(outcome.commands, "queue", size_warning=None) + marker, *wire = workflow.commands_to_server_commands(outcome.commands, "queue", size_warning=None) + # The run's failure-order marker takes sequence 2, before its first all-group. + assert is_failure_order_marker(marker) assert len(wire) == 2 assert [command["cancellation_scope_id"] for command in wire] == ["scope-one", "scope-one"] - assert [command["parallel_group_base_sequence"] for command in wire] == [2, 2] + assert [command["parallel_group_base_sequence"] for command in wire] == [3, 3] assert [command["parallel_group_index"] for command in wire] == [0, 1] diff --git a/tests/test_child_workflow_policies.py b/tests/test_child_workflow_policies.py index 014bca24..a23eae2d 100644 --- a/tests/test_child_workflow_policies.py +++ b/tests/test_child_workflow_policies.py @@ -15,6 +15,7 @@ commands_to_server_commands, replay, ) +from tests.failure_order import failure_order_marker_event, is_failure_order_marker from tests.test_cooperative_cancellation import marker, request from tests.test_cooperative_cancellation_worker import ClaimServer, claimed_task @@ -46,7 +47,10 @@ def run(self, ctx: WorkflowContext): # type: ignore[no-untyped-def] def scheduled(commands: list[Any]) -> list[dict[str, Any]]: + """History for these commands. A new run's failure-order marker, recorded + before its first all-group, keeps its own sequence (sdk-python#307).""" return [ + failure_order_marker_event(index + 1) if is_failure_order_marker(command) else {"event_type": "ChildWorkflowScheduled", "payload": { "sequence": index + 1, "child_workflow_type": command["workflow_type"], "child_workflow_run_id": f"child-run-{index + 1}", **command, @@ -114,8 +118,10 @@ def test_existing_positional_constructor_and_omitted_wire_fields_are_preserved() def test_changed_policies_fail_cold_replay_and_cancellation_delivery(mode: str, field: str, delivered: bool) -> None: policies = options() history = scheduled(replay(workflow(mode, policies), [], []).commands) + first = 2 if mode == "parallel" else 1 if delivered: - history += [request(), marker(1, "child" if mode == "sequential" else "parallel", sequence_span=len(history))] + history += [request(), marker(first, "child" if mode == "sequential" else "parallel", + sequence_span=len(history) - first + 1)] assert replay(workflow(mode, policies), history, [], run_id="run-1").commands == [CompleteWorkflow("request-1")] else: cold = replay(workflow(mode, policies), history, []) @@ -123,7 +129,7 @@ def test_changed_policies_fail_cold_replay_and_cancellation_delivery(mode: str, policies[field] = "abandon" with pytest.raises(NonDeterministicReplayError, match="child_workflow_policy_changed") as captured: replay(workflow(mode, policies), history, [], run_id="run-1") - assert captured.value.workflow_sequence == 1 + assert captured.value.workflow_sequence == first @pytest.mark.parametrize("mode", ["sequential", "parallel", "selection"]) diff --git a/tests/test_completion_size_limit.py b/tests/test_completion_size_limit.py index cb2829d3..4b61be84 100644 --- a/tests/test_completion_size_limit.py +++ b/tests/test_completion_size_limit.py @@ -119,7 +119,10 @@ async def test_typed_413_fails_the_run_once_without_resending_the_completion() - "/api/worker/workflow-tasks/task-413/complete", ] oversized, failure = (body(request) for request in requests) - assert [command["type"] for command in oversized["commands"]] == ["schedule_activity"] * 4 + # A new run records its failure-order marker before its first all-group (#307). + assert [command["type"] for command in oversized["commands"]] == [ + "record_version_marker", *["schedule_activity"] * 4, + ] assert (failure["lease_owner"], failure["workflow_task_attempt"]) == ("worker-413", 3) assert len(failure["commands"]) == 1 command = failure["commands"][0] @@ -277,7 +280,7 @@ async def test_typed_409_fails_the_run_once_without_resending_or_failing_the_tas "/api/worker/workflow-tasks/task-413/complete", ] refused, failure = (body(request) for request in requests) - assert [command["type"] for command in refused["commands"]] == ["schedule_activity"] * 4 + assert [command["type"] for command in refused["commands"]] == ["record_version_marker", *["schedule_activity"] * 4] assert (failure["lease_owner"], failure["workflow_task_attempt"]) == ("worker-413", 3) command = failure["commands"][0] assert (command["type"], command["non_retryable"]) == ("fail_workflow", True) diff --git a/tests/test_parallel_completion_order_replay.py b/tests/test_parallel_completion_order_replay.py index a3ea8370..7fd3f363 100644 --- a/tests/test_parallel_completion_order_replay.py +++ b/tests/test_parallel_completion_order_replay.py @@ -66,23 +66,6 @@ def _completed_result(workflow_type: type, history: list[dict[str, Any]]) -> Any return command.result -def _failure_results(workflow_type: type, history: list[dict[str, Any]]) -> tuple[Any, Any]: - """The result a released worker recorded, and the one a task picks now. - - A history whose run already closed replays the failure released workers - raised: the lowest workflow position. A task reaching the settled group - for the first time raises the first recorded failure, as Workflow's - embedded engine does (sdk-python#307), recording the member it raised - when the two rules differ. - """ - closed = [*history, {"event_type": "WorkflowCompleted", "payload": {}}] - legacy = _completed_result(workflow_type, closed) - outcome = replay(workflow_type, history, [], payload_codec=serializer.AVRO_CODEC) - assert isinstance(outcome.commands[-1], CompleteWorkflow) - assert len(outcome.commands) in (1, 2) - return legacy, outcome.commands[-1].result - - @workflow.defn(name="parallel-completion-order-activities") class ParallelCompletionOrderActivities: def run(self, ctx: WorkflowContext): # type: ignore[no-untyped-def] @@ -374,18 +357,13 @@ def test_activity_failure_selection_follows_workflow_position(completion_order: ), ] - legacy, now = _failure_results( + assert _completed_result( ParallelCompletionOrderActivityFailures, _history_in_completion_order(events, completion_order), - ) - assert legacy == { + ) == { "activity_type": "activity-fails-first", "message": "first positional activity failure", } - first = 1 if completion_order.index(1) < completion_order.index(2) else 2 - assert now == ({"activity_type": "activity-fails-first", "message": "first positional activity failure"} - if first == 1 else - {"activity_type": "activity-fails-second", "message": "second positional activity failure"}) @pytest.mark.parametrize("completion_order", THREE_SIBLING_ORDERS) @@ -417,18 +395,13 @@ def test_child_failure_selection_follows_workflow_position(completion_order: tup ), ] - legacy, now = _failure_results( + assert _completed_result( ParallelCompletionOrderChildFailures, _history_in_completion_order(events, completion_order), - ) - assert legacy == { + ) == { "child_workflow_type": "child-fails-first", "message": "first positional child failure", } - first = 1 if completion_order.index(1) < completion_order.index(2) else 2 - assert now == ({"child_workflow_type": "child-fails-first", "message": "first positional child failure"} - if first == 1 else - {"child_workflow_type": "child-fails-second", "message": "second positional child failure"}) @pytest.mark.parametrize("completion_order", THREE_SIBLING_ORDERS) @@ -453,18 +426,13 @@ def test_mixed_failure_selection_follows_workflow_position(completion_order: tup _terminal_event("TimerFired", "mixed", 3, 2, timer_kind="sleep"), ] - legacy, now = _failure_results( + assert _completed_result( ParallelCompletionOrderMixedFailures, _history_in_completion_order(events, completion_order), - ) - assert legacy == { + ) == { "failure_type": "ChildWorkflowFailed", "message": "first positional mixed failure", } - first = 0 if completion_order.index(0) < completion_order.index(1) else 1 - assert now == ({"failure_type": "ChildWorkflowFailed", "message": "first positional mixed failure"} - if first == 0 else - {"failure_type": "ActivityFailed", "message": "second positional mixed failure"}) @pytest.mark.parametrize("completion_order", [(0, 1), (1, 0)]) @@ -483,9 +451,10 @@ def test_repeated_identical_failures_stay_bound_to_yielded_positions( for index in range(2) ] - legacy, now = _failure_results( - ParallelCompletionOrderRepeatedFailures, - _history_in_completion_order(events, completion_order), + assert ( + _completed_result( + ParallelCompletionOrderRepeatedFailures, + _history_in_completion_order(events, completion_order), + ) + == "position-0-failed" ) - assert legacy == "position-0-failed" - assert now == f"position-{completion_order[0]}-failed" diff --git a/tests/test_parallel_failure_order.py b/tests/test_parallel_failure_order.py new file mode 100644 index 00000000..e59dcc21 --- /dev/null +++ b/tests/test_parallel_failure_order.py @@ -0,0 +1,396 @@ +"""sdk-python#307: which member failure an all() group raises. + +A run records ``PARALLEL_FAILURE_ORDER_MARKER`` directly before the first +all-group that can fail, when a worker with this rule first schedules it. Such +a run raises the first member failure recorded in history, as soon as it is +recorded, whether or not the other members have settled. A run whose first +all-group was recorded without the marker keeps the rule of the workers that +started it for its whole life: on every later task, query and update, a +settled group raises its lowest-position failure. +""" + +from __future__ import annotations + +from typing import Any, cast + +import pytest + +from durable_workflow import serializer, workflow +from durable_workflow.errors import ActivityFailed, NonDeterministicReplayError +from durable_workflow.workflow import ( + PARALLEL_FAILURE_DECISION, + CompleteUpdate, + CompleteWorkflow, + FailWorkflow, + RecordSideEffect, + ScheduleActivity, + StartTimer, + WorkflowContext, + apply_update, + commands_to_server_commands, + query_state, + replay, +) +from tests.failure_order import FAILURE_ORDER_MARKER, failure_order_marker_event +from tests.test_cooperative_cancellation import request + +SIZE = 4 + + +def _entry(base: int, index: int) -> dict[str, Any]: + return { + "parallel_group_id": f"parallel-activities:{base}:{SIZE}", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": base, + "parallel_group_size": SIZE, + "parallel_group_index": index, + } + + +def _member(event_type: str, index: int, base: int) -> dict[str, Any]: + payload: dict[str, Any] = { + "sequence": base + index, "activity_execution_id": f"member-{index}", "activity_type": f"member-{index}", + **_entry(base, index), "parallel_group_path": [_entry(base, index)], + } + if event_type == "ActivityCompleted": + payload["result"] = serializer.envelope(index, codec=serializer.AVRO_CODEC) + if event_type == "ActivityFailed": + payload["message"] = f"member-{index} failed" + return {"event_type": event_type, "payload": payload} + + +def _activity(event_type: str, sequence: int, activity_type: str, result: Any = None) -> dict[str, Any]: + payload: dict[str, Any] = {"sequence": sequence, "activity_execution_id": f"{activity_type}-{sequence}", + "activity_type": activity_type} + if event_type == "ActivityCompleted": + payload["result"] = serializer.envelope(result, codec=serializer.AVRO_CODEC) + return {"event_type": event_type, "payload": payload} + + +def _run(*, marked: bool) -> tuple[list[dict[str, Any]], int]: + """A run with its group scheduled: with the marker (new) or without (legacy).""" + base = 2 if marked else 1 + history: list[dict[str, Any]] = [{"event_type": "WorkflowStarted", "payload": {}}] + if marked: + history.append(failure_order_marker_event(1)) + history.extend(_member("ActivityScheduled", index, base) for index in range(SIZE)) + return history, base + + +def _settled_out_of_order(base: int) -> list[dict[str, Any]]: + """Member 2 fails first; member 0, the lower position, fails later.""" + return [ + _member("ActivityFailed", 2, base), + _member("ActivityCompleted", 1, base), + _member("ActivityFailed", 0, base), + _member("ActivityCompleted", 3, base), + ] + + +@workflow.defn(name="tests.parallel-failure-order") +class FailureOrder: + def __init__(self) -> None: + self.caught: str | None = None + + @workflow.query("caught") + def caught_member(self) -> str | None: + return self.caught + + @workflow.update("report") + def report(self) -> dict[str, str | None]: + return {"compensating": self.caught} + + def run(self, ctx: WorkflowContext): # type: ignore[no-untyped-def] + try: + yield [ctx.schedule_activity(f"member-{index}", []) for index in range(SIZE)] + except ActivityFailed as failure: + self.caught = failure.activity_type + return (yield ctx.schedule_activity("compensate", [self.caught])) + return "no failure" + + +@workflow.defn(name="tests.parallel-failure-order-uncaught") +class FailureOrderUncaught: + def run(self, ctx: WorkflowContext): # type: ignore[no-untyped-def] + return (yield [ctx.schedule_activity(f"member-{index}", []) for index in range(SIZE)]) + + +def _commands(history: list[dict[str, Any]], definition: type = FailureOrder) -> list[Any]: + return replay(definition, history, [], payload_codec=serializer.AVRO_CODEC).commands + + +def _caught(history: list[dict[str, Any]]) -> Any: + return query_state(FailureOrder, history, [], "caught", payload_codec=serializer.AVRO_CODEC) + + +def test_a_new_run_records_the_marker_directly_before_its_first_group() -> None: + commands = _commands([{"event_type": "WorkflowStarted", "payload": {}}]) + assert commands[0] == FAILURE_ORDER_MARKER + wire = commands_to_server_commands(commands, "queue", payload_codec=serializer.AVRO_CODEC) + assert wire[0] == {"type": "record_version_marker", "change_id": "durable_workflow.parallel_failure_order", + "version": 1, "min_supported": 1, "max_supported": 1} + assert [command["parallel_group_base_sequence"] for command in wire[1:]] == [2] * SIZE + history, _ = _run(marked=True) + assert _commands(history) == [] + + +def test_the_marker_is_recorded_once_per_run() -> None: + @workflow.defn(name="tests.parallel-failure-order-two-groups") + class TwoGroups: + def run(self, ctx: WorkflowContext): # type: ignore[no-untyped-def] + yield [ctx.schedule_activity("first", []), ctx.schedule_activity("second", [])] + yield [ctx.schedule_activity("third", []), ctx.schedule_activity("fourth", [])] + return "done" + + def entry(base: int, index: int) -> dict[str, Any]: + return {"parallel_group_id": f"parallel-activities:{base}:2", "parallel_group_kind": "activity", + "parallel_group_base_sequence": base, "parallel_group_size": 2, "parallel_group_index": index} + + history: list[dict[str, Any]] = [{"event_type": "WorkflowStarted", "payload": {}}, failure_order_marker_event(1)] + for index, name in enumerate(["first", "second"]): + payload = {"sequence": 2 + index, "activity_type": name, **entry(2, index), + "parallel_group_path": [entry(2, index)]} + history.append({"event_type": "ActivityScheduled", "payload": dict(payload)}) + history.append({"event_type": "ActivityCompleted", "payload": { + **payload, "result": serializer.envelope(name, codec=serializer.AVRO_CODEC)}}) + commands = replay(TwoGroups, history, [], payload_codec=serializer.AVRO_CODEC).commands + assert [type(command) for command in commands] == [ScheduleActivity, ScheduleActivity] + wire = commands_to_server_commands(commands, "queue", payload_codec=serializer.AVRO_CODEC) + assert [command["parallel_group_base_sequence"] for command in wire] == [4, 4] + + +def test_a_timer_group_does_not_decide_the_rule() -> None: + @workflow.defn(name="tests.parallel-failure-order-timers-first") + class TimersFirst: + def run(self, ctx: WorkflowContext): # type: ignore[no-untyped-def] + yield [ctx.start_timer(1), ctx.start_timer(2)] + yield [ctx.schedule_activity("work", []), ctx.schedule_activity("more", [])] + return "done" + + commands = replay(TimersFirst, [], []).commands + assert [type(command) for command in commands] == [StartTimer, StartTimer] + fired = [{"event_type": "TimerFired", "payload": {"sequence": sequence}} for sequence in (1, 2)] + commands = replay(TimersFirst, fired, []).commands + assert commands[0] == FAILURE_ORDER_MARKER + assert [type(command) for command in commands[1:]] == [ScheduleActivity, ScheduleActivity] + + +def test_a_run_started_by_an_earlier_worker_records_the_marker_at_its_first_group() -> None: + """No all-group of the run has an outcome yet, so no task has chosen a + failure under the earlier rule.""" + @workflow.defn(name="tests.parallel-failure-order-late-group") + class LateGroup: + def run(self, ctx: WorkflowContext): # type: ignore[no-untyped-def] + yield ctx.schedule_activity("prepare", []) + return (yield [ctx.schedule_activity("a", []), ctx.schedule_activity("b", [])]) + + history = [{"event_type": "WorkflowStarted", "payload": {}}, _activity("ActivityScheduled", 1, "prepare"), + _activity("ActivityCompleted", 1, "prepare", "ready")] + commands = replay(LateGroup, history, [], payload_codec=serializer.AVRO_CODEC).commands + assert commands[0] == FAILURE_ORDER_MARKER + wire = commands_to_server_commands(commands[1:], "queue", payload_codec=serializer.AVRO_CODEC) + assert [command["parallel_group_base_sequence"] for command in wire] == [3, 3] + + +def test_a_cancellation_request_keeps_the_earlier_rule() -> None: + """A request may replace the group with its cancellation, so the run + records no marker and its group is recorded under the earlier rule.""" + outcome = replay(FailureOrder, [request()], [], run_id="run-1", payload_codec=serializer.AVRO_CODEC) + assert FAILURE_ORDER_MARKER not in outcome.commands + assert outcome.cancellation_delivery is not None + + +def test_a_marked_run_raises_the_first_recorded_failure_of_a_settled_group() -> None: + history, base = _run(marked=True) + history += _settled_out_of_order(base) + assert _commands(history) == [ScheduleActivity("compensate", ["member-2"])] + assert _caught(history) == "member-2" + uncaught = _commands(history, FailureOrderUncaught) + assert [type(command) for command in uncaught] == [FailWorkflow] + assert "member-2" in str(cast(FailWorkflow, uncaught[0]).message) + + +def test_a_marked_run_raises_at_once_without_recording_a_decision() -> None: + """History is append-only: the first failure this task sees is the first + failure every later replay sees, so nothing more needs recording.""" + history, base = _run(marked=True) + history += [_member("ActivityCompleted", 0, base), _member("ActivityFailed", 2, base)] + assert _commands(history) == [ScheduleActivity("compensate", ["member-2"])] + # Member 3 fails and member 1 completes after the compensation was + # scheduled; replay still raises member 2. + later = [*history, _activity("ActivityScheduled", base + SIZE, "compensate"), + _member("ActivityFailed", 3, base), _member("ActivityCompleted", 1, base), + _activity("ActivityCompleted", base + SIZE, "compensate", "undone")] + for end in range(len(history) + 1, len(later) + 1): + expected = [CompleteWorkflow("undone")] if end == len(later) else [] + assert _commands(later[:end]) == expected, end + assert _caught(later[:end]) == "member-2", end + + +def test_a_marked_run_keeps_its_choice_when_a_lower_position_fails_before_the_completion() -> None: + """The deciding task saw only member 2's failure. Member 0 failed and the + others completed before its completion was written.""" + history, base = _run(marked=True) + history += [ + _member("ActivityFailed", 2, base), _member("ActivityFailed", 0, base), + _member("ActivityCompleted", 1, base), _member("ActivityCompleted", 3, base), + _activity("ActivityScheduled", base + SIZE, "compensate"), + _activity("ActivityCompleted", base + SIZE, "compensate", "undone"), + ] + assert _commands(history) == [CompleteWorkflow("undone")] + assert _caught(history) == "member-2" + + +def test_an_unmarked_run_keeps_the_position_rule_at_every_task() -> None: + """The group was scheduled without the marker, by a released worker.""" + history, base = _run(marked=False) + history += _settled_out_of_order(base) + assert _commands(history) == [ScheduleActivity("compensate", ["member-0"])] + assert _caught(history) == "member-0" + moved_on = [*history, _activity("ActivityScheduled", base + SIZE, "compensate"), + _activity("ActivityCompleted", base + SIZE, "compensate", "undone")] + assert _commands(moved_on) == [CompleteWorkflow("undone")] + assert _caught(moved_on) == "member-0" + + +def test_an_unmarked_run_whose_task_recorded_nothing_after_the_group_keeps_the_position_rule() -> None: + """A released worker caught member 0 and then waited without recording a + command (for example on a select() handle it already held). A later task + must not take that for a new decision.""" + history, base = _run(marked=False) + history += _settled_out_of_order(base) + history.append({"event_type": "SignalReceived", "payload": {"signal_name": "nudge", "arguments": None}}) + assert _caught(history) == "member-0" + commands = _commands(history) + assert not any(isinstance(command, RecordSideEffect) for command in commands) + assert commands == [ScheduleActivity("compensate", ["member-0"])] + + +def test_an_update_in_an_unmarked_run_reports_the_position_rule() -> None: + history, base = _run(marked=False) + history += _settled_out_of_order(base) + history.append({"event_type": "UpdateAccepted", "payload": { + "update_id": "update-1", "update_name": "report", "arguments": None}}) + update = apply_update(FailureOrder, history, [], "update-1", payload_codec=serializer.AVRO_CODEC) + assert update == CompleteUpdate("update-1", {"compensating": "member-0"}) + marked, base = _run(marked=True) + marked += _settled_out_of_order(base) + marked.append(history[-1]) + update = apply_update(FailureOrder, marked, [], "update-1", payload_codec=serializer.AVRO_CODEC) + assert update == CompleteUpdate("update-1", {"compensating": "member-2"}) + + +def test_an_unmarked_run_still_honours_a_recorded_decision() -> None: + """2.19 records which member it raised while siblings were open (#338).""" + history, base = _run(marked=False) + decision = {PARALLEL_FAILURE_DECISION: { + "parallel_group_id": f"parallel-activities:{base}:{SIZE}", "parallel_group_base_sequence": base, + "parallel_group_size": SIZE, "member_sequence": base + 2, + }} + history += [ + _member("ActivityFailed", 2, base), _member("ActivityFailed", 0, base), + _member("ActivityCompleted", 1, base), _member("ActivityCompleted", 3, base), + {"event_type": "SideEffectRecorded", "payload": { + "sequence": base + SIZE, "result": serializer.envelope(decision, codec=serializer.AVRO_CODEC)}}, + ] + assert _commands(history) == [ScheduleActivity("compensate", ["member-2"])] + + +def test_an_unknown_marker_version_is_nondeterministic() -> None: + history, base = _run(marked=True) + history[1]["payload"].update({"version": 2, "max_supported": 2}) + history += _settled_out_of_order(base) + with pytest.raises(NonDeterministicReplayError): + _commands(history) + + +# Reproducers for the Server histories in +# tests/fixtures/replay_regressions/failure-order-*.json, recorded on a Server +# built from server main 52e3ec00 by published 2.19.1 and by this version +# (sdk-python#307 review). Two members: m1 fails first, then m0. +_ONCE = {"max_attempts": 1} + + +def _review_group(ctx: WorkflowContext) -> list[Any]: + return [ctx.schedule_activity(f"m{i}", [i], retry_policy=_ONCE) for i in range(2)] + + +@workflow.defn(name="rv348_handle") +class ReviewZeroCommandCatch: + """Catch the group failure, then await a select() handle started earlier: + the task that catches it records no command.""" + + def __init__(self) -> None: + self.caught: str | None = None + + @workflow.query("caught") + def q(self) -> str | None: + return self.caught + + @workflow.update("report") + def report(self, *_: Any) -> dict[str, str | None]: + return {"compensating": self.caught} + + def run(self, ctx: WorkflowContext, *_: Any): # type: ignore[no-untyped-def] + sel = yield ctx.select({"t": ctx.start_timer(1), "slow": ctx.schedule_activity("slow", [], retry_policy=_ONCE)}) + slow = sel.handles["slow"] + yield ctx.start_timer(1) + try: + yield _review_group(ctx) + except ActivityFailed as e: + self.caught = e.message if hasattr(e, "message") else str(e) + yield slow.await_result() + yield ctx.schedule_activity("compensate", [self.caught], retry_policy=_ONCE) + return self.caught + + +@workflow.defn(name="rv348_cancel") +class ReviewSelectCancel: + """Catch the group failure, then cancel a select() loser started earlier.""" + + def __init__(self) -> None: + self.caught: str | None = None + + @workflow.query("caught") + def q(self) -> str | None: + return self.caught + + @workflow.update("report") + def report(self, *_: Any) -> dict[str, str | None]: + return {"compensating": self.caught} + + def run(self, ctx: WorkflowContext, *_: Any): # type: ignore[no-untyped-def] + sel = yield ctx.select({"t": ctx.start_timer(1), "slow": ctx.schedule_activity("slow", [], retry_policy=_ONCE)}) + slow = sel.handles["slow"] + yield ctx.start_timer(1) + try: + yield _review_group(ctx) + except ActivityFailed as e: + self.caught = str(e) + yield slow.cancel() + yield ctx.schedule_activity("compensate", [self.caught], retry_policy=_ONCE) + return self.caught + + +@workflow.defn(name="rv348_plain") +class ReviewPlain: + """Catch the group failure, then schedule a compensation.""" + + def __init__(self) -> None: + self.caught: str | None = None + + @workflow.update("report") + def report(self, *_: Any) -> dict[str, str | None]: + return {"compensating": self.caught} + + @workflow.query("caught") + def q(self) -> str | None: + return self.caught + + def run(self, ctx: WorkflowContext, *_: Any): # type: ignore[no-untyped-def] + try: + yield _review_group(ctx) + except ActivityFailed as e: + self.caught = str(e) + yield ctx.schedule_activity("compensate", [self.caught], retry_policy=_ONCE) + return self.caught diff --git a/tests/test_parallel_group_policy.py b/tests/test_parallel_group_policy.py index cc297b3d..c8413d76 100644 --- a/tests/test_parallel_group_policy.py +++ b/tests/test_parallel_group_policy.py @@ -62,19 +62,15 @@ def test_absent_policy_raises_the_first_failure_while_a_sibling_is_open() -> Non assert query_state(ParallelFailurePolicyWorkflow, history, [], "caught") == "fast-second" -def test_absent_policy_selection_after_a_later_sibling_settles() -> None: - """Released workers raised the lower position once both members settled. - A closed history keeps that; a task reaching the settled group now raises - the first recorded failure and records it (sdk-python#307).""" +def test_absent_policy_preserves_positional_failure_after_a_later_sibling_settles() -> None: history = json.loads(FIXTURE.read_text())["history"] first = dict(history[1]["payload"]) first["message"] = "lower position failed later" history.append({"event_type": "ActivityFailed", "recorded_at": "2026-01-01T00:00:03Z", "payload": first}) - closed = replay(ParallelFailurePolicyWorkflow, [*history, {"event_type": "WorkflowCompleted", "payload": {}}], []) - assert closed.commands == [CompleteWorkflow("slow-first")] outcome = replay(ParallelFailurePolicyWorkflow, history, []) - assert [type(command) for command in outcome.commands] == [RecordSideEffect, CompleteWorkflow] - assert outcome.commands[1] == CompleteWorkflow("fast-second") + assert len(outcome.commands) == 1 + assert isinstance(outcome.commands[0], CompleteWorkflow) + assert outcome.commands[0].result == "slow-first" def explicit_history(policy: str = "propagate") -> list[dict]: @@ -115,8 +111,7 @@ def test_enabling_draft_support_does_not_change_absent_policy_history() -> None: def test_enabling_draft_support_keeps_the_legacy_failure_clock_gap() -> None: fixture = json.loads(FIXTURE.with_name("business-clock-parallel-failure-selection-gap.json").read_text()) - closed = [*fixture["history"], {"event_type": "WorkflowCompleted", "payload": {}}] - outcome = draft_replay(closed, ParallelFailureBusinessClockWorkflow) + outcome = draft_replay(fixture["history"], ParallelFailureBusinessClockWorkflow) assert outcome.commands[0].result == fixture["expected"]["command_sequence"][0]["result"] diff --git a/tests/test_parallel_member_failure_open_siblings.py b/tests/test_parallel_member_failure_open_siblings.py index c3cf2101..107a03e1 100644 --- a/tests/test_parallel_member_failure_open_siblings.py +++ b/tests/test_parallel_member_failure_open_siblings.py @@ -123,7 +123,9 @@ def test_an_open_group_is_never_scheduled_again() -> None: def test_a_new_group_is_still_scheduled_once() -> None: - commands = _wire([{"event_type": "WorkflowStarted", "payload": {}}]) + marker, *commands = _wire([{"event_type": "WorkflowStarted", "payload": {}}]) + # A new run records its failure-order marker before its first all-group (#307). + assert marker["type"] == "record_version_marker" assert [command["type"] for command in commands] == ["schedule_activity"] * SIZE assert [command["parallel_group_index"] for command in commands] == list(range(SIZE)) @@ -277,87 +279,22 @@ def test_a_late_sibling_never_takes_the_compensation_result(late_first: bool) -> assert outcome.commands == [CompleteWorkflow({"caught": "member-0", "compensated": "undone"})] -def _settled_out_of_order() -> list[dict[str, Any]]: - """Every member settled before a task reached the group. Member 2 failed - first; member 0, the lower position, failed later.""" - return [ +def test_a_group_that_settled_before_the_workflow_moved_on_keeps_positional_selection() -> None: + """A history recorded while the worker waited for every member: the + lower-position failure is the one raised, as before, and no decision is + recorded because the task saw every outcome.""" + history = [ *_scheduled(), _member("ActivityFailed", 2), _member("ActivityCompleted", 1), _member("ActivityFailed", 0), _member("ActivityCompleted", 3), ] - - -def test_a_settled_group_raises_its_first_recorded_failure_and_records_it() -> None: - """sdk-python#307: Workflow's embedded engine raises the earliest recorded - failure, not the lowest position. Released workers raised member 0 here - and wrote no decision, so this task records the member it raised.""" - history = _settled_out_of_order() - outcome = replay(AllFailureOpenSiblings, history, [], payload_codec=serializer.AVRO_CODEC) - assert outcome.commands == [ - RecordSideEffect({PARALLEL_FAILURE_DECISION: { - "parallel_group_id": f"parallel-activities:1:{SIZE}", "parallel_group_base_sequence": 1, - "parallel_group_size": SIZE, "member_sequence": 3, - }}), - ScheduleActivity("compensate", ["member-2"]), - ] - assert query_state(AllFailureOpenSiblings, history, [], "caught") == "member-2" - uncaught = replay(AllFailureUncaught, history, [], payload_codec=serializer.AVRO_CODEC) - assert [type(command) for command in uncaught.commands] == [RecordSideEffect, FailWorkflow] - assert "member-2" in str(cast(FailWorkflow, uncaught.commands[1]).message) - - -def test_a_settled_group_replays_the_recorded_first_failure_at_every_later_task() -> None: - history = [ - *_settled_out_of_order(), - _decision(2), - _compensation("ActivityScheduled"), - _compensation("ActivityCompleted", "undone"), + commands = _wire(history) + assert [(command["type"], command.get("activity_type")) for command in commands] == [ + ("schedule_activity", "compensate"), ] - for end in range(len(history) - 1, len(history) + 1): - expected = [] if end < len(history) else [ - CompleteWorkflow({"caught": "member-2", "compensated": "undone"}), - ] - outcome = replay(AllFailureOpenSiblings, history[:end], [], payload_codec=serializer.AVRO_CODEC) - assert outcome.commands == expected, end - assert query_state(AllFailureOpenSiblings, history[:end], [], "caught") == "member-2", end - - -@pytest.mark.parametrize("order", [(0, 2), (2,), (1, 3)]) -def test_a_settled_group_whose_first_failure_has_the_lowest_position_records_nothing( - order: tuple[int, ...], -) -> None: - """Both rules raise the same member, so the commands are exactly what - released workers sent.""" - history = [*_scheduled(), *(_member("ActivityFailed", index) for index in order), - *(_member("ActivityCompleted", index) for index in range(SIZE) if index not in order)] - first = order[0] - assert _wire(history) == _wire_scheduled_compensation(f"member-{first}") - assert query_state(AllFailureOpenSiblings, history, [], "caught") == f"member-{first}" - - -def _wire_scheduled_compensation(caught: str) -> list[dict[str, Any]]: - return commands_to_server_commands( - [ScheduleActivity("compensate", [caught])], "queue", payload_codec=serializer.AVRO_CODEC, - ) - - -def test_a_released_worker_history_of_a_settled_group_keeps_positional_selection() -> None: - """Recorded by 2.19.1 and earlier: the task saw every member settled, - raised member 0 by position and moved on with no decision. It replays - exactly as recorded.""" - history = [*_settled_out_of_order(), _compensation("ActivityScheduled", sequence=SIZE + 1)] - assert replay(AllFailureOpenSiblings, history, [], payload_codec=serializer.AVRO_CODEC).commands == [] assert query_state(AllFailureOpenSiblings, history, [], "caught") == "member-0" - history.append(_compensation("ActivityCompleted", "undone", sequence=SIZE + 1)) - assert replay(AllFailureOpenSiblings, history, [], payload_codec=serializer.AVRO_CODEC).commands == [ - CompleteWorkflow({"caught": "member-0", "compensated": "undone"}), - ] - closed = [*_settled_out_of_order(), {"event_type": "WorkflowFailed", "payload": {"message": "member-0 failed"}}] - outcome = replay(AllFailureUncaught, closed, [], payload_codec=serializer.AVRO_CODEC) - assert [type(command) for command in outcome.commands] == [FailWorkflow] - assert "member-0" in str(cast(FailWorkflow, outcome.commands[0]).message) def test_a_history_from_another_runtime_raises_its_first_recorded_failure() -> None: diff --git a/tests/test_parallel_saga_authoring.py b/tests/test_parallel_saga_authoring.py index 8e609a21..3a9df47b 100644 --- a/tests/test_parallel_saga_authoring.py +++ b/tests/test_parallel_saga_authoring.py @@ -14,6 +14,7 @@ commands_to_server_commands, replay, ) +from tests.failure_order import FAILURE_ORDER_MARKER def _entry(kind: str, base: int, size: int, index: int) -> dict[str, Any]: @@ -32,12 +33,12 @@ def _entry(kind: str, base: int, size: int, index: int) -> dict[str, Any]: } -def _paths() -> list[list[dict[str, Any]]]: - outer = [_entry("mixed", 1, 3, index) for index in range(3)] +def _paths(base: int = 1) -> list[list[dict[str, Any]]]: + outer = [_entry("mixed", base, 3, index) for index in range(3)] return [ [outer[0]], - [outer[1], _entry("mixed", 2, 2, 0)], - [outer[2], _entry("mixed", 2, 2, 1)], + [outer[1], _entry("mixed", base + 1, 2, 0)], + [outer[2], _entry("mixed", base + 1, 2, 1)], ] @@ -92,14 +93,17 @@ def run(self, ctx: WorkflowContext): # type: ignore[no-untyped-def] def test_nested_parallel_emits_flat_commands_with_stable_full_paths() -> None: outcome = replay(NestedParallelWorkflow, [], []) - assert [type(command) for command in outcome.commands] == [ + # A new run records its failure-order marker at sequence 1, before its + # first all-group (sdk-python#307). + assert outcome.commands[0] == FAILURE_ORDER_MARKER + assert [type(command) for command in outcome.commands[1:]] == [ ScheduleActivity, StartChildWorkflow, StartTimer, ] - commands = commands_to_server_commands(outcome.commands, "parallel-workers") - for command, path in zip(commands, _paths(), strict=True): + commands = commands_to_server_commands(outcome.commands[1:], "parallel-workers") + for command, path in zip(commands, _paths(2), strict=True): assert command["parallel_group_path"] == path assert command["parallel_group_id"] == path[-1]["parallel_group_id"] diff --git a/tests/test_replay.py b/tests/test_replay.py index 3c9048bf..0f5b5933 100644 --- a/tests/test_replay.py +++ b/tests/test_replay.py @@ -35,6 +35,7 @@ commands_to_server_commands, replay, ) +from tests.failure_order import FAILURE_ORDER_MARKER from tests.integration.polyglot_fixtures import ( PHP_HISTORY_EVENT_EXPECTED_KEYS, PHP_HISTORY_EVENT_FIXTURES, @@ -2260,15 +2261,19 @@ def run(self, ctx: WorkflowContext): # type: ignore[no-untyped-def] class TestFanOut: def test_no_history_emits_batch(self) -> None: outcome = replay(FanOutWorkflow, [], []) - assert len(outcome.commands) == 2 - assert isinstance(outcome.commands[0], ScheduleActivity) - assert outcome.commands[0].activity_type == "fetch-a" + # A new run records its failure-order marker before its first + # all-group (sdk-python#307). + assert len(outcome.commands) == 3 + assert outcome.commands[0] == FAILURE_ORDER_MARKER assert isinstance(outcome.commands[1], ScheduleActivity) - assert outcome.commands[1].activity_type == "fetch-b" + assert outcome.commands[1].activity_type == "fetch-a" + assert isinstance(outcome.commands[2], ScheduleActivity) + assert outcome.commands[2].activity_type == "fetch-b" def test_mixed_batch_emits_language_neutral_parallel_metadata(self) -> None: outcome = replay(ParallelMetadataProducerWorkflow, [], []) - commands = commands_to_server_commands(outcome.commands, "parallel-workers") + assert outcome.commands[0] == FAILURE_ORDER_MARKER + commands = commands_to_server_commands(outcome.commands[1:], "parallel-workers") assert [command["type"] for command in commands] == [ "schedule_activity", @@ -2277,9 +2282,9 @@ def test_mixed_batch_emits_language_neutral_parallel_metadata(self) -> None: ] for index, command in enumerate(commands): entry = { - "parallel_group_id": "parallel-calls:1:3", + "parallel_group_id": "parallel-calls:2:3", "parallel_group_kind": "mixed", - "parallel_group_base_sequence": 1, + "parallel_group_base_sequence": 2, "parallel_group_size": 3, "parallel_group_index": index, } @@ -2288,14 +2293,15 @@ def test_mixed_batch_emits_language_neutral_parallel_metadata(self) -> None: def test_repeated_deferred_command_keeps_occurrence_specific_metadata(self) -> None: outcome = replay(RepeatedCommandFanOutWorkflow, [], []) - commands = commands_to_server_commands(outcome.commands, "parallel-workers") + assert outcome.commands[0] == FAILURE_ORDER_MARKER + commands = commands_to_server_commands(outcome.commands[1:], "parallel-workers") - assert outcome.commands[0] is not outcome.commands[1] + assert outcome.commands[1] is not outcome.commands[2] for index, command in enumerate(commands): entry = { - "parallel_group_id": "parallel-activities:1:2", + "parallel_group_id": "parallel-activities:2:2", "parallel_group_kind": "activity", - "parallel_group_base_sequence": 1, + "parallel_group_base_sequence": 2, "parallel_group_size": 2, "parallel_group_index": index, } @@ -2514,8 +2520,9 @@ def run(self, ctx: WorkflowContext): # type: ignore[no-untyped-def] class TestNontrivialWorkflow: def test_first_replay_fans_out(self) -> None: outcome = replay(NontrivialWorkflow, [], []) - assert len(outcome.commands) == 2 - assert all(isinstance(c, ScheduleActivity) for c in outcome.commands) + assert outcome.commands[0] == FAILURE_ORDER_MARKER + assert len(outcome.commands) == 3 + assert all(isinstance(c, ScheduleActivity) for c in outcome.commands[1:]) def test_after_fan_out_starts_timer(self) -> None: history = [ diff --git a/tests/test_replay_regression_corpus.py b/tests/test_replay_regression_corpus.py index 1a8606e2..913c633f 100644 --- a/tests/test_replay_regression_corpus.py +++ b/tests/test_replay_regression_corpus.py @@ -49,6 +49,7 @@ GoldenTimeoutWaitWorkflow, GoldenVersionMarkerWorkflow, ) +from tests.test_parallel_failure_order import ReviewPlain, ReviewSelectCancel, ReviewZeroCommandCatch from tests.test_parallel_group_policy import ParallelFailurePolicyWorkflow from tests.test_parallel_member_failure_open_siblings import ( NativeCaughtMemberFailure, @@ -482,6 +483,9 @@ def run(self, ctx: WorkflowContext, message: str) -> None: NativeCaughtMemberFailure, NativeSettledGroupCaught, NativeUncaughtMemberFailure, + ReviewPlain, + ReviewSelectCancel, + ReviewZeroCommandCatch, UpdateSignalConditionTimerWorkflow, WorkflowStreamAuthorWorkflow, WorkflowMemoAuthorWorkflow, @@ -504,17 +508,26 @@ def run(self, ctx: WorkflowContext, message: str) -> None: WORKFLOW_TYPES["tests.replay.parallel-failure-policy"] = ParallelFailurePolicyWorkflow -# Immutable evidence whose recorded expectation no longer holds. The files stay -# unchanged. Each one's history replays under the named fixture instead, which -# cannot use ``supersedes`` because the protocol version is the same. -# - sdk-python#338: replay sent an already-scheduled group again, which Server -# refuses. -# - sdk-python#307: a task reaching a settled group raised the lowest-position -# failure instead of the first recorded one. +# Immutable evidence whose recorded expectation was the sdk-python#338 defect: +# replay sent an already-scheduled group again, which Server refuses. The files +# stay unchanged. Each one's history replays under the named fixture instead, +# which cannot use ``supersedes`` because the protocol version is the same. DEFECT_EVIDENCE = { "legacy-parallel-failure-remains-pending": "absent-policy-failure-raises-with-open-sibling", "run-closed-cancellation-not-a-group-outcome": "run-closed-all-failure-keeps-the-handled-failure", - "business-clock-parallel-failure-selection-gap": "settled-group-raises-first-recorded-failure", +} + +# Immutable evidence of the commands a released worker sent for a new run's +# first all-group. New runs now record the parallel failure-order version +# marker before that group (sdk-python#307), so the same input replays under +# the named fixture, at the same protocol version. No recorded history changes +# meaning: a history without the marker keeps its sequences and its failure +# selection. +SUPERSEDED_NEW_RUN_EMISSION = { + "python-completion-size-limit-fails-run-once": + "python-completion-size-limit-fails-run-once-after-failure-order-marker", + "python-completion-history-limit-fails-run-once": + "python-completion-history-limit-fails-run-once-after-failure-order-marker", } @@ -525,7 +538,7 @@ def _fixture_documents() -> dict[Path, dict[str, Any]]: def _fixture_paths() -> list[Path]: """The current fixtures. A superseded one stays as evidence but does not replay.""" documents = _fixture_documents() - retired = set(DEFECT_EVIDENCE) + retired = set(DEFECT_EVIDENCE) | set(SUPERSEDED_NEW_RUN_EMISSION) for document in documents.values(): retired.update(document.get("supersedes", [])) return [path for path, document in documents.items() if document.get("id") not in retired] @@ -533,9 +546,9 @@ def _fixture_paths() -> list[Path]: def test_retired_defect_evidence_replays_under_its_correction() -> None: by_id = {document["id"]: document for document in _fixture_documents().values()} - for retired, correction in DEFECT_EVIDENCE.items(): + for retired, correction in {**DEFECT_EVIDENCE, **SUPERSEDED_NEW_RUN_EMISSION}.items(): assert by_id[retired]["protocol_version"] == by_id[correction]["protocol_version"] - assert by_id[retired]["history"] == by_id[correction]["history"] + assert by_id[retired].get("history") == by_id[correction].get("history") assert by_id[retired]["workflow"] == by_id[correction]["workflow"] diff --git a/tests/test_run_closed_settlement_replay.py b/tests/test_run_closed_settlement_replay.py index 4001220b..5be31c3f 100644 --- a/tests/test_run_closed_settlement_replay.py +++ b/tests/test_run_closed_settlement_replay.py @@ -623,13 +623,9 @@ def test_cancellation_at_close_never_replaces_the_failure_the_workflow_handled() def test_ordinary_cancellation_is_still_an_activity_outcome() -> None: """Only ``cancellation_reason: run_closed`` is skipped.""" - history = _all_failure_history(cancelled_at_close=True) - del history[-2]["payload"]["cancellation_reason"] - history[-1]["payload"]["output"] = serializer.envelope("caught: activity failed")["blob"] + history = _all_failure_history(cancelled_at_close=True)[:-1] + del history[-1]["payload"]["cancellation_reason"] - # The run closed after both members settled, as a released worker that - # chose the lower position left it. A skipped cancellation would leave - # member 1 open and make replay raise "boom" instead. outcome = replay(AllFailureClosedWorkflow, history, []) assert len(outcome.commands) == 1 diff --git a/tests/test_worker.py b/tests/test_worker.py index 6c2959b1..672ce52c 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -1903,7 +1903,8 @@ def spy_envelope_many(values, *args, **kwargs): # type: ignore[no-untyped-def] await worker._run_workflow_task(task) - commands = mock_client.complete_workflow_task.call_args.kwargs["commands"] + marker, *commands = mock_client.complete_workflow_task.call_args.kwargs["commands"] + assert marker["type"] == "record_version_marker" assert [command["activity_type"] for command in commands] == ["first", "second"] assert calls == [[["a"], ["b"]]] From 906c93f84b901118f3f42281c53c58e4944342e3 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:38:43 +0000 Subject: [PATCH 3/3] Make writing the failure-order marker opt-in (#307) Published 2.19.1 cannot replay a run that recorded the parallel failure-order marker. The Server marks the workflow task replay-blocked with no retry, and only a repair resumes it, which can hand it to an old worker again. Rolling back would do the same to every marked run. Follow the two-phase pattern of workflow#1239: - Every worker reads and follows a recorded marker, on tasks, queries and updates. - Writing it is off by default. Worker(record_parallel_failure_order=True) or DURABLE_WORKFLOW_RECORD_PARALLEL_FAILURE_ORDER=true turns it on. Any other environment value is refused when the Worker is created. replay() and Replayer.replay() take the same keyword. - With the default, new runs send exactly the commands 2.19.1 sends. - The next minor release flips the default (#351). This removes the same-protocol supersession of new-run emission fixtures: parallel-group-metadata-emission and the two completion-limit fixtures are unchanged and pass again. New fixtures cover the first task with recording on and off. The marked native and PHP histories replay under the new rule without the setting. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 11 +- docs/sdk-reference.md | 47 +++-- src/durable_workflow/worker.py | 32 +++ src/durable_workflow/workflow.py | 33 ++- tests/failure_order.py | 19 +- ...s-run-once-after-failure-order-marker.json | 109 ---------- ...s-run-once-after-failure-order-marker.json | 107 ---------- ...lt-first-task-sends-released-commands.json | 184 +++++++++++++++++ ...rder-opt-in-first-task-records-marker.json | 189 ++++++++++++++++++ ...ailure-order-opt-in-new-run-emission.json} | 10 +- tests/test_activity_cancellation_policies.py | 9 +- tests/test_cancellation_scope_authoring.py | 7 +- tests/test_child_workflow_policies.py | 10 +- tests/test_completion_size_limit.py | 7 +- tests/test_parallel_failure_order.py | 128 +++++++++--- ...t_parallel_member_failure_open_siblings.py | 9 +- tests/test_parallel_saga_authoring.py | 18 +- tests/test_replay.py | 33 ++- tests/test_replay_regression_corpus.py | 20 +- tests/test_worker.py | 3 +- 20 files changed, 602 insertions(+), 383 deletions(-) delete mode 100644 tests/fixtures/replay_regressions/completion-history-limit-fails-run-once-after-failure-order-marker.json delete mode 100644 tests/fixtures/replay_regressions/completion-size-limit-fails-run-once-after-failure-order-marker.json create mode 100644 tests/fixtures/replay_regressions/failure-order-default-first-task-sends-released-commands.json create mode 100644 tests/fixtures/replay_regressions/failure-order-opt-in-first-task-records-marker.json rename tests/fixtures/replay_regressions/{parallel-group-metadata-emission-after-failure-order-marker.json => parallel-failure-order-opt-in-new-run-emission.json} (82%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 147a7c0a..c891e32b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,13 +12,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Workflow code can signal another workflow durably with `ctx.signal_external_workflow(workflow_id, signal_name, args=None, *, run_id=None)` (durable-workflow/.github#223, durable-workflow/server#1134). Yielding it sends the `signal_external_workflow` command and resumes with an `ExternalSignalResult` (`outcome`, `reason`, `target_run_id`, `target_command_id`, `target_signal_id`, `retained`, `delivered`). A missing, closed or refusing target is a returned outcome (`target_not_found`, `target_closed` or `rejected` with the Server's reason), not an exception, because the Server records it as a result and workflow code decides what to do. Arguments travel as an inline Avro payload envelope, like child workflow arguments. A signal name the Server would refuse (blank, a control character below U+0020 or U+007F, more than 191 characters or more than 255 UTF-8 bytes) raises the new `InvalidSignalName` (a `ValueError` with `reason`) in workflow code when the command is created, so nothing is sent and the task does not fail with a 422. The command is yielded on its own; a parallel group or selection member is refused. - **Replay safety:** replay matches `ExternalSignalRequested` at the command's sequence and returns the outcome from the following `ExternalSignalDelivered` or `ExternalSignalFailed`. A recorded signal is never sent again, including on a cold replacement worker and while its outcome is not yet recorded. A different target, run, signal name or arguments at that sequence raises `NonDeterministicReplayError`. Existing histories are unchanged. - **Server requirement:** the Worker submits the command only when the Server advertises `signal_external_workflow` in `worker_protocol.server_capabilities` (and lists it in `supported_workflow_task_commands` when that list is present). Otherwise it fails the workflow task with `signal_external_workflow_unavailable` before submitting, so the signal is never sent outside history. - -### Changed - -- **Behavior change:** in a new run, an `all()` group (`yield [...]`) with several failed members raises the failure recorded first in history, even if every member had settled by the time the workflow resumed. Workers up to 2.19.1 raised the lowest-position failure in that case, so the exception a workflow caught depended on whether its task ran before or after the other members finished. The first recorded failure is still raised as soon as it is recorded; the other members keep running, are not cancelled, and their outcomes are ignored. Ties follow durable event order, as the shared parallel-group contract specifies. Workflow's embedded engine compares millisecond timestamps and then member index, so failures recorded in the same millisecond can be chosen differently (durable-workflow/.github#415). (#307) - - **Replay safety:** the rule is recorded once per run. When a run first schedules a group that contains an activity or child workflow, the worker records the version marker `durable_workflow.parallel_failure_order` (version 1) directly before it, so that group and everything after it start one workflow sequence later. A run whose first such group was recorded without the marker keeps the earlier rule for its whole life, on every workflow task, query and update. This holds even when a task recorded no command after catching the failure, or another worker answered an update in between. That run's settled groups raise the lowest-position failure, and a failure raised while siblings are open still records the #338 decision. A run with a cancellation request when it reaches its first group records no marker and keeps the earlier rule. Marked runs need no per-group decision, because the first failure a task sees in append-only history is the first every later replay sees. - - **Upgrade:** published 2.19.1 and earlier fail a marked run's workflow task as nondeterministic at the marker. They do not choose a different failure, and the task is retried, so finish rolling out every worker on a task queue before relying on the new rule. Native and PHP Server built from server main accept the marker. - - **Regression evidence:** fixtures recorded by published 2.19.1 on a Server built from main replay under the earlier rule. They cover a settled group in flight, a catch followed by a task that recorded no command, an update answered by 2.19.1 before or between this version's tasks, and a select() cancellation. Fixtures recorded by this version replay under the new rule. `business-clock-parallel-failure-selection-gap` is unchanged and still passes, as released behaviour. Three fixtures recorded the commands for a new run's first group: `parallel-group-metadata-emission` (superseded at protocol 1.20) and the two completion-limit fixtures (whose inputs now replay under `*-after-failure-order-marker`). These now include the marker. +- Opt-in first-recorded-failure selection for `all()` groups (`yield [...]`), as a two-phase rollout (#307). With `Worker(..., record_parallel_failure_order=True)` or `DURABLE_WORKFLOW_RECORD_PARALLEL_FAILURE_ORDER=true`, a new run records the version marker `durable_workflow.parallel_failure_order` (version 1) directly before its first group that contains an activity or child workflow. A marked run raises the failure recorded first in history, even when every member had settled before the workflow resumed. Workers up to 2.19.1 raised the lowest-position failure in that case, so the exception a workflow caught depended on whether its task ran before or after the other members finished. The first recorded failure is still raised as soon as it is recorded, and the other members keep running, are not cancelled, and have their outcomes ignored. Ties follow durable event order, as the shared parallel-group contract specifies. Workflow's embedded engine compares millisecond timestamps and then member index, so failures recorded in the same millisecond can be chosen differently (durable-workflow/.github#415). Any other value of the environment variable is refused when the `Worker` is created. + - **Default off; enable it only after every worker on the task queue runs this release or later.** Published 2.19.1 and earlier cannot replay a marked run. The Server marks the workflow task replay-blocked with no retry, and the run waits until it is repaired, which can hand it to an old worker again. With recording off, new runs send exactly the commands 2.19.1 sends. Turning recording off again is safe. Rolling back below this release requires marked runs to be drained first. The next minor release turns recording on by default (#351). + - **Replay safety:** every worker of this release follows a marker a run already recorded, whether or not recording is on, for every workflow task, query and update. A marked run needs no per-group decision, because the first failure a task sees in append-only history is the first every later replay sees. A run whose first such group was recorded without the marker keeps the earlier rule for its whole life. Its settled groups raise the lowest-position failure, and a failure raised while siblings are open still records the #338 decision. This holds even when a task recorded no command after catching the failure, or another worker answered an update in between. A run with a cancellation request when it reaches its first group records no marker. + - **Regression evidence:** fixtures recorded by published 2.19.1 on a Server built from main replay under the earlier rule. They cover a settled group in flight, a catch followed by a task that recorded no command, an update answered by 2.19.1 before or between later tasks, and a select() cancellation. Marked runs recorded on native and PHP Server replay under the new rule without the setting. New fixtures cover the first task with recording on and off. Existing fixtures, including `parallel-group-metadata-emission`, the completion-limit fixtures and `business-clock-parallel-failure-selection-gap`, are unchanged. ## [2.19.1] - 2026-11-18 diff --git a/docs/sdk-reference.md b/docs/sdk-reference.md index 11101b51..7ae7316c 100644 --- a/docs/sdk-reference.md +++ b/docs/sdk-reference.md @@ -527,27 +527,36 @@ outcomes do not change the thrown failure or reach the workflow's next operations. A group whose members are already scheduled is never scheduled again on a later wake, such as a signal. -When several members have failed, a run started on this version throws the -failure recorded first in history, in durable event order, even if every -member had already settled when the workflow resumed. The run records this -choice once, as the version marker `durable_workflow.parallel_failure_order` -directly before its first group that contains an activity or child workflow, -so code after that point starts one workflow sequence later. Workflow's -embedded engine also throws the earliest failure, but compares millisecond -timestamps and then member index, so two failures recorded in the same -millisecond can be chosen differently +When several members have failed, a run that recorded the failure-order marker +throws the failure recorded first in history, in durable event order, even if +every member had already settled when the workflow resumed. The marker is the +version marker `durable_workflow.parallel_failure_order`, recorded once, directly +before the run's first group that contains an activity or child workflow; code +after it starts one workflow sequence later. Workflow's embedded engine also +throws the earliest failure, but compares millisecond timestamps and then member +index, so two failures recorded in the same millisecond can be chosen +differently ([durable-workflow/.github#415](https://github.com/durable-workflow/.github/issues/415)). -A run whose first such group was recorded without the marker, by an earlier -worker, keeps that worker's rule for its whole life, on every task, query and -update: if every member had settled when the workflow resumed, the failure -with the lowest input position is thrown. When such a run raises a failure -while other members are open, the worker records which member it raised as a -side effect directly after the group, so every later replay raises the same -one. A run that has a cancellation request when it reaches its first group -also keeps this rule. Workers up to 2.19.1 cannot replay a run that recorded -the marker; they fail its workflow task as nondeterministic, so finish -upgrading every worker on a task queue before relying on the new rule. +Recording the marker is opt-in in this release: pass +`Worker(..., record_parallel_failure_order=True)` or set +`DURABLE_WORKFLOW_RECORD_PARALLEL_FAILURE_ORDER=true`. Turn it on only after +every worker that polls the task queue runs this release or later. Workers up +to 2.19.1 cannot replay a marked run: the Server marks its workflow task +replay-blocked, and the run waits until it is repaired, which can hand it to an +old worker again. Every worker of this release follows a marker a run already +recorded, whether or not recording is on, so turning recording off again is +safe. Rolling back below this release needs marked runs drained first. A later +minor release turns recording on by default +([#351](https://github.com/durable-workflow/sdk-python/issues/351)). + +A run without the marker keeps the earlier rule for its whole life, on every +task, query and update: if every member had settled when the workflow resumed, +the failure with the lowest input position is thrown. When such a run raises a +failure while other members are open, the worker records which member it raised +as a side effect directly after the group, so every later replay raises the +same one. A run that has a cancellation request when it reaches its first group +records no marker. ## Saga compensation diff --git a/src/durable_workflow/worker.py b/src/durable_workflow/worker.py index 5c72627a..51fb0f7e 100644 --- a/src/durable_workflow/worker.py +++ b/src/durable_workflow/worker.py @@ -23,6 +23,7 @@ import json import logging import math +import os import pickle import random import sys @@ -1156,6 +1157,24 @@ def _contract_version_matches(value: Any, expected: int) -> bool: return False +RECORD_PARALLEL_FAILURE_ORDER_ENV = "DURABLE_WORKFLOW_RECORD_PARALLEL_FAILURE_ORDER" + + +def _record_parallel_failure_order(value: bool | None) -> bool: + """The worker's failure-order recording setting, refused when malformed.""" + if value is None: + raw = os.environ.get(RECORD_PARALLEL_FAILURE_ORDER_ENV, "") + setting = raw.strip().lower() + if setting in ("", "false", "0"): + return False + if setting in ("true", "1"): + return True + raise ValueError(f"{RECORD_PARALLEL_FAILURE_ORDER_ENV} must be true or false") + if type(value) is not bool: + raise ValueError("record_parallel_failure_order must be a bool or None") + return value + + class Worker: """Polls workflow and activity tasks and dispatches them to Python callables.""" @@ -1184,8 +1203,20 @@ def __init__( external_storage_threshold_bytes: int | None = None, external_storage_cache: ExternalPayloadCache | None = None, require_query_task_authority: bool = False, + record_parallel_failure_order: bool | None = None, ) -> None: + """Create a worker. + + ``record_parallel_failure_order`` lets new runs record the parallel + failure-order marker before their first all-group, so their groups + raise the first recorded member failure (sdk-python#307). ``None`` + reads ``DURABLE_WORKFLOW_RECORD_PARALLEL_FAILURE_ORDER`` (``true`` or + ``false``, default ``false``). Enable it only after every worker on the + task queue runs this release or later: older workers cannot replay a + marked run. Every worker follows a marker that a run already recorded. + """ self.client = client + self.record_parallel_failure_order = _record_parallel_failure_order(record_parallel_failure_order) self.require_query_task_authority = require_query_task_authority self._query_authority_supported = False self.task_queue = task_queue @@ -1960,6 +1991,7 @@ async def _replay_workflow_claim( local_activity_cancellation_policies=self._local_activity_cancellation_policies, allow_cancellation_scope_authoring=self._allow_cancellation_scope_authoring, allow_cancellation_scope_delivery=self._allow_cancellation_scope_delivery, + record_parallel_failure_order=self.record_parallel_failure_order, ) if outcome.cancellation_scope_delivery is not None: if outcome.commands: diff --git a/src/durable_workflow/workflow.py b/src/durable_workflow/workflow.py index 7ce03ae3..5223d188 100644 --- a/src/durable_workflow/workflow.py +++ b/src/durable_workflow/workflow.py @@ -3363,6 +3363,7 @@ def replay( external_storage: ExternalStorageDriver | None = None, external_storage_cache: ExternalPayloadCache | None = None, cancel_requested: bool = False, + record_parallel_failure_order: bool = False, ) -> ReplayOutcome: events = _history_events_from_export(history) selected_workflow_type = workflow_type or _workflow_type_from_history(events) @@ -3387,6 +3388,7 @@ def replay( external_storage=external_storage, external_storage_cache=external_storage_cache, cancel_requested=cancel_requested, + record_parallel_failure_order=record_parallel_failure_order, ) def _workflow_cls(self, workflow_type: str | None) -> type: @@ -3756,7 +3758,16 @@ def replay( local_activity_cancellation_policies: tuple[str, ...] = (), allow_cancellation_scope_authoring: bool = False, allow_cancellation_scope_delivery: bool = False, + record_parallel_failure_order: bool = False, ) -> ReplayOutcome: + """Replay ``history_events`` and return the next commands. + + ``record_parallel_failure_order`` lets a run that reaches its first + all-group now record ``PARALLEL_FAILURE_ORDER_MARKER`` and raise its first + recorded member failure (sdk-python#307). It is off by default, so new runs + send the same commands as 2.19.1. A history that already contains the + marker follows it whether or not this is set. + """ return _replay_state( workflow_cls, history_events, @@ -3775,6 +3786,7 @@ def replay( local_activity_cancellation_policies=local_activity_cancellation_policies, allow_cancellation_scope_authoring=allow_cancellation_scope_authoring, allow_cancellation_scope_delivery=allow_cancellation_scope_delivery, + record_parallel_failure_order=record_parallel_failure_order, ).outcome @@ -4266,9 +4278,10 @@ def _history_event_type(event: Mapping[str, Any]) -> str | None: # (sdk-python#338). Replay raises exactly that member. PARALLEL_FAILURE_DECISION = "durable_workflow.parallel_failure_decision" # The version marker a run records, directly before its first all-group, when -# that group is first scheduled by a worker that raises the first recorded -# member failure (sdk-python#307). A run without it keeps the failure -# selection of the workers that ran it before, for its whole life. +# that group is first scheduled by a worker with recording enabled +# (sdk-python#307). A marked run raises the first recorded member failure. A run +# without it keeps the failure selection of the workers that ran it before, for +# its whole life. Every worker reads it; writing it is opt-in in this release. PARALLEL_FAILURE_ORDER_MARKER = "durable_workflow.parallel_failure_order" @@ -5277,6 +5290,7 @@ def _replay_state( allow_cancellation_scope_authoring: bool = False, allow_cancellation_scope_delivery: bool = False, parallel_group_policies: tuple[str, ...] = (), + record_parallel_failure_order: bool = False, ) -> _ReplayState: _validate_workflow_entrypoint(workflow_cls) _workflow_handler_registries(workflow_cls) @@ -6829,8 +6843,10 @@ def _resolve_failure_order() -> bool: A run reaching its first all-group now records the marker directly before scheduling it. No all-group of the run has produced an outcome before that point, so no earlier task can have chosen a failure under - the other rule. A run with a cancellation request that may replace the - group records no marker and keeps the earlier rule. + the other rule. Recording is opt-in (``record_parallel_failure_order``) + because workers up to 2.19.1 cannot replay a marked run. A run with a + cancellation request that may replace the group records no marker and + keeps the earlier rule. """ nonlocal result_cursor, authored_sequence marker = RecordVersionMarker( @@ -6855,12 +6871,13 @@ def _resolve_failure_order() -> bool: interrupting = cancellation.request is not None or any( sequence not in consumed_scope_deliveries for sequence in committed_scopes.deliveries ) - if step is None and not closed and not interrupting: + if record_parallel_failure_order and step is None and not closed and not interrupting: pending.append(marker) authored_sequence += 1 return True - # Scheduled now without the marker, under a cancellation request that - # may replace it, or recorded without it: the earlier rule. + # Recorded without the marker, scheduled now with recording off (the + # default until every worker reads the marker), or scheduled under a + # cancellation request that may replace it: the earlier rule. return False def _parallel_leaf_kind(command: Any) -> str: diff --git a/tests/failure_order.py b/tests/failure_order.py index 8058f0fd..f9561a04 100644 --- a/tests/failure_order.py +++ b/tests/failure_order.py @@ -1,10 +1,4 @@ -"""Helpers for the run's parallel failure-order marker (sdk-python#307). - -A run records ``PARALLEL_FAILURE_ORDER_MARKER`` directly before the first -all-group that can fail, when that group is first scheduled. Tests about other -behavior that build a history from a fresh run's commands drop it, which -replays as a run an earlier worker started. -""" +"""Helpers for the run's parallel failure-order marker (sdk-python#307).""" from __future__ import annotations @@ -17,17 +11,6 @@ ) -def is_failure_order_marker(command: Any) -> bool: - if isinstance(command, RecordVersionMarker): - return command.change_id == PARALLEL_FAILURE_ORDER_MARKER - return (isinstance(command, dict) and command.get("type") == "record_version_marker" - and command.get("change_id") == PARALLEL_FAILURE_ORDER_MARKER) - - -def without_failure_order_marker(commands: list[Any]) -> list[Any]: - return [command for command in commands if not is_failure_order_marker(command)] - - def failure_order_marker_event(sequence: int = 1, **extra: Any) -> dict[str, Any]: return {"event_type": "VersionMarkerRecorded", **extra, "payload": { "sequence": sequence, "change_id": PARALLEL_FAILURE_ORDER_MARKER, "version": 1, diff --git a/tests/fixtures/replay_regressions/completion-history-limit-fails-run-once-after-failure-order-marker.json b/tests/fixtures/replay_regressions/completion-history-limit-fails-run-once-after-failure-order-marker.json deleted file mode 100644 index 45812194..00000000 --- a/tests/fixtures/replay_regressions/completion-history-limit-fails-run-once-after-failure-order-marker.json +++ /dev/null @@ -1,109 +0,0 @@ -{ - "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", - "fixture_schema": "durable-workflow.replay-regression/v1", - "id": "python-completion-history-limit-fails-run-once-after-failure-order-marker", - "protocol_version": "1.20", - "bindings": [ - "python" - ], - "parallel_failure_order": { - "issues": [ - "durable-workflow/sdk-python#307" - ], - "note": "The fan-out of python-completion-history-limit-fails-run-once. Its first completion now also carries the run's parallel failure-order version marker before the all-group. The refused completion is still never resent and the run still fails once." - }, - "workflow": { - "type": "sdk.completion-size-limit", - "input": [ - { - "activities": 4, - "argument_bytes": 2048, - "limit": 4096, - "refusal": "history_transaction_limit_exceeded", - "limit_kind": "history_transaction_bytes" - } - ] - }, - "command_sequence": [ - { - "type": "workflow_task_request", - "route": "complete", - "command_types": [ - "record_version_marker", - "schedule_activity", - "schedule_activity", - "schedule_activity", - "schedule_activity" - ], - "non_retryable": [ - false, - false, - false, - false, - false - ], - "structural_limit": [ - false, - false, - false, - false, - false - ] - }, - { - "type": "workflow_task_request", - "route": "complete", - "command_types": [ - "fail_workflow" - ], - "non_retryable": [ - true - ], - "structural_limit": [ - true - ] - } - ], - "expected": { - "command_sequence": [ - { - "type": "workflow_task_request", - "route": "complete", - "command_types": [ - "record_version_marker", - "schedule_activity", - "schedule_activity", - "schedule_activity", - "schedule_activity" - ], - "non_retryable": [ - false, - false, - false, - false, - false - ], - "structural_limit": [ - false, - false, - false, - false, - false - ] - }, - { - "type": "workflow_task_request", - "route": "complete", - "command_types": [ - "fail_workflow" - ], - "non_retryable": [ - true - ], - "structural_limit": [ - true - ] - } - ] - } -} diff --git a/tests/fixtures/replay_regressions/completion-size-limit-fails-run-once-after-failure-order-marker.json b/tests/fixtures/replay_regressions/completion-size-limit-fails-run-once-after-failure-order-marker.json deleted file mode 100644 index bd4e446c..00000000 --- a/tests/fixtures/replay_regressions/completion-size-limit-fails-run-once-after-failure-order-marker.json +++ /dev/null @@ -1,107 +0,0 @@ -{ - "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", - "fixture_schema": "durable-workflow.replay-regression/v1", - "id": "python-completion-size-limit-fails-run-once-after-failure-order-marker", - "protocol_version": "1.20", - "bindings": [ - "python" - ], - "parallel_failure_order": { - "issues": [ - "durable-workflow/sdk-python#307" - ], - "note": "The fan-out of python-completion-size-limit-fails-run-once. Its first completion now also carries the run's parallel failure-order version marker before the all-group. The refused completion is still never resent and the run still fails once." - }, - "workflow": { - "type": "sdk.completion-size-limit", - "input": [ - { - "activities": 4, - "argument_bytes": 2048, - "limit": 4096 - } - ] - }, - "command_sequence": [ - { - "type": "workflow_task_request", - "route": "complete", - "command_types": [ - "record_version_marker", - "schedule_activity", - "schedule_activity", - "schedule_activity", - "schedule_activity" - ], - "non_retryable": [ - false, - false, - false, - false, - false - ], - "structural_limit": [ - false, - false, - false, - false, - false - ] - }, - { - "type": "workflow_task_request", - "route": "complete", - "command_types": [ - "fail_workflow" - ], - "non_retryable": [ - true - ], - "structural_limit": [ - true - ] - } - ], - "expected": { - "command_sequence": [ - { - "type": "workflow_task_request", - "route": "complete", - "command_types": [ - "record_version_marker", - "schedule_activity", - "schedule_activity", - "schedule_activity", - "schedule_activity" - ], - "non_retryable": [ - false, - false, - false, - false, - false - ], - "structural_limit": [ - false, - false, - false, - false, - false - ] - }, - { - "type": "workflow_task_request", - "route": "complete", - "command_types": [ - "fail_workflow" - ], - "non_retryable": [ - true - ], - "structural_limit": [ - true - ] - } - ] - } -} diff --git a/tests/fixtures/replay_regressions/failure-order-default-first-task-sends-released-commands.json b/tests/fixtures/replay_regressions/failure-order-default-first-task-sends-released-commands.json new file mode 100644 index 00000000..f1d4aa11 --- /dev/null +++ b/tests/fixtures/replay_regressions/failure-order-default-first-task-sends-released-commands.json @@ -0,0 +1,184 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "failure-order-default-first-task-sends-released-commands", + "protocol_version": "1.20", + "bindings": [ + "python" + ], + "parallel_failure_order": { + "issues": [ + "durable-workflow/sdk-python#307" + ], + "note": "The same first workflow task as failure-order-opt-in-first-task-records-marker. With recording off, the default in this release, the worker sends exactly the commands published 2.19.1 sends, so older workers can replay the run." + }, + "workflow": { + "type": "rv348_plain", + "input": [], + "payload_codec": "avro", + "run_id": "01m7tfn1drg737gpdqqq521f99", + "workflow_id": "rv307-rv348_plain-021643" + }, + "history": [ + { + "id": "01m7tfn1dyeq9qmk4mqccjd4k3", + "sequence": 1, + "event_type": "StartAccepted", + "namespace": "default", + "payload": { + "memo": null, + "command": { + "id": "01m7tfn1dt9s08nvqqk4kh61am", + "type": "start", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "outcome": "started_new", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "sequence": 1, + "applied_at": "2026-11-19T18:03:41.369328Z", + "accepted_at": "2026-11-19T18:03:41.369324Z", + "auth_method": "token", + "auth_status": "authorized", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfn1drg737gpdqqq521f99", + "request_route_name": "generated::FqJuJMl97iiXlIDz", + "request_fingerprint": "sha256:6f781847735e7bee891c0cee63c564fe6ff8c5bdabac5982864acea8cb2c003c" + }, + "outcome": "started_new", + "business_key": null, + "workflow_type": "rv348_plain", + "workflow_class": "rv348_plain", + "workflow_run_id": "01m7tfn1drg737gpdqqq521f99", + "search_attributes": null, + "visibility_labels": null, + "workflow_command_id": "01m7tfn1dt9s08nvqqk4kh61am", + "workflow_instance_id": "rv307-rv348_plain-021643" + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfn1dt9s08nvqqk4kh61am", + "recorded_at": "2026-11-19T18:03:41.373188Z" + }, + { + "id": "01m7tfn1e1g7qswx5qqcjvx475", + "sequence": 2, + "event_type": "WorkflowStarted", + "namespace": "default", + "payload": { + "command": { + "id": "01m7tfn1dt9s08nvqqk4kh61am", + "type": "start", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "outcome": "started_new", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "sequence": 1, + "applied_at": "2026-11-19T18:03:41.369328Z", + "accepted_at": "2026-11-19T18:03:41.369324Z", + "auth_method": "token", + "auth_status": "authorized", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfn1drg737gpdqqq521f99", + "request_route_name": "generated::FqJuJMl97iiXlIDz", + "request_fingerprint": "sha256:6f781847735e7bee891c0cee63c564fe6ff8c5bdabac5982864acea8cb2c003c" + }, + "workflow_type": "rv348_plain", + "workflow_class": "rv348_plain", + "run_deadline_at": "2026-11-19T18:13:41+00:00", + "workflow_run_id": "01m7tfn1drg737gpdqqq521f99", + "declared_queries": [ + "caught" + ], + "declared_signals": [], + "declared_updates": [ + "report" + ], + "declared_entry_mode": "canonical", + "run_timeout_seconds": 600, + "workflow_command_id": "01m7tfn1dt9s08nvqqk4kh61am", + "workflow_instance_id": "rv307-rv348_plain-021643", + "declared_entry_method": "handle", + "execution_deadline_at": "2026-11-19T19:03:41+00:00", + "declared_query_contracts": [ + { + "name": "caught", + "parameters": [] + } + ], + "declared_signal_contracts": [], + "declared_update_contracts": [ + { + "name": "report", + "parameters": [ + { + "name": "_", + "type": null, + "default": null, + "position": 0, + "required": false, + "variadic": true, + "allows_null": true, + "default_available": false + } + ] + } + ], + "execution_timeout_seconds": 3600, + "declared_update_validators": [], + "declared_entry_declaring_class": "external:rv348_plain", + "workflow_definition_fingerprint": "sha256:4cca49f53721ff6d13fd7cd42526e63e2497bbd302876fde0ff10eb9748ac44b", + "workflow_definition_fingerprint_source": "worker" + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfn1dt9s08nvqqk4kh61am", + "recorded_at": "2026-11-19T18:03:41.376531Z" + } + ], + "expected": { + "command_sequence": [ + { + "type": "schedule_activity", + "activity_type": "m0", + "parallel_group_base_sequence": 1 + }, + { + "type": "schedule_activity", + "activity_type": "m1", + "parallel_group_base_sequence": 1 + } + ] + } +} diff --git a/tests/fixtures/replay_regressions/failure-order-opt-in-first-task-records-marker.json b/tests/fixtures/replay_regressions/failure-order-opt-in-first-task-records-marker.json new file mode 100644 index 00000000..38bfe1a8 --- /dev/null +++ b/tests/fixtures/replay_regressions/failure-order-opt-in-first-task-records-marker.json @@ -0,0 +1,189 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "failure-order-opt-in-first-task-records-marker", + "protocol_version": "1.20", + "bindings": [ + "python" + ], + "parallel_failure_order": { + "issues": [ + "durable-workflow/sdk-python#307" + ], + "note": "The first workflow task of failure-order-new-run-update, as a Server built from durable-workflow/server main 52e3ec00 delivered it. With failure-order recording enabled, the worker records the parallel failure-order marker before the run's first all-group." + }, + "workflow": { + "type": "rv348_plain", + "input": [], + "payload_codec": "avro", + "run_id": "01m7tfn1drg737gpdqqq521f99", + "workflow_id": "rv307-rv348_plain-021643", + "record_parallel_failure_order": true + }, + "history": [ + { + "id": "01m7tfn1dyeq9qmk4mqccjd4k3", + "sequence": 1, + "event_type": "StartAccepted", + "namespace": "default", + "payload": { + "memo": null, + "command": { + "id": "01m7tfn1dt9s08nvqqk4kh61am", + "type": "start", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "outcome": "started_new", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "sequence": 1, + "applied_at": "2026-11-19T18:03:41.369328Z", + "accepted_at": "2026-11-19T18:03:41.369324Z", + "auth_method": "token", + "auth_status": "authorized", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfn1drg737gpdqqq521f99", + "request_route_name": "generated::FqJuJMl97iiXlIDz", + "request_fingerprint": "sha256:6f781847735e7bee891c0cee63c564fe6ff8c5bdabac5982864acea8cb2c003c" + }, + "outcome": "started_new", + "business_key": null, + "workflow_type": "rv348_plain", + "workflow_class": "rv348_plain", + "workflow_run_id": "01m7tfn1drg737gpdqqq521f99", + "search_attributes": null, + "visibility_labels": null, + "workflow_command_id": "01m7tfn1dt9s08nvqqk4kh61am", + "workflow_instance_id": "rv307-rv348_plain-021643" + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfn1dt9s08nvqqk4kh61am", + "recorded_at": "2026-11-19T18:03:41.373188Z" + }, + { + "id": "01m7tfn1e1g7qswx5qqcjvx475", + "sequence": 2, + "event_type": "WorkflowStarted", + "namespace": "default", + "payload": { + "command": { + "id": "01m7tfn1dt9s08nvqqk4kh61am", + "type": "start", + "source": "control_plane", + "status": "accepted", + "context": { + "principal": { + "id": "legacy-token", + "type": "auth:token", + "label": "Admin" + } + }, + "outcome": "started_new", + "payload": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "sequence": 1, + "applied_at": "2026-11-19T18:03:41.369328Z", + "accepted_at": "2026-11-19T18:03:41.369324Z", + "auth_method": "token", + "auth_status": "authorized", + "caller_label": "Standalone Server", + "principal_id": "legacy-token", + "request_path": "/api/workflows", + "target_scope": "instance", + "payload_codec": "avro", + "principal_type": "auth:token", + "request_method": "POST", + "principal_label": "Admin", + "resolved_run_id": "01m7tfn1drg737gpdqqq521f99", + "request_route_name": "generated::FqJuJMl97iiXlIDz", + "request_fingerprint": "sha256:6f781847735e7bee891c0cee63c564fe6ff8c5bdabac5982864acea8cb2c003c" + }, + "workflow_type": "rv348_plain", + "workflow_class": "rv348_plain", + "run_deadline_at": "2026-11-19T18:13:41+00:00", + "workflow_run_id": "01m7tfn1drg737gpdqqq521f99", + "declared_queries": [ + "caught" + ], + "declared_signals": [], + "declared_updates": [ + "report" + ], + "declared_entry_mode": "canonical", + "run_timeout_seconds": 600, + "workflow_command_id": "01m7tfn1dt9s08nvqqk4kh61am", + "workflow_instance_id": "rv307-rv348_plain-021643", + "declared_entry_method": "handle", + "execution_deadline_at": "2026-11-19T19:03:41+00:00", + "declared_query_contracts": [ + { + "name": "caught", + "parameters": [] + } + ], + "declared_signal_contracts": [], + "declared_update_contracts": [ + { + "name": "report", + "parameters": [ + { + "name": "_", + "type": null, + "default": null, + "position": 0, + "required": false, + "variadic": true, + "allows_null": true, + "default_available": false + } + ] + } + ], + "execution_timeout_seconds": 3600, + "declared_update_validators": [], + "declared_entry_declaring_class": "external:rv348_plain", + "workflow_definition_fingerprint": "sha256:4cca49f53721ff6d13fd7cd42526e63e2497bbd302876fde0ff10eb9748ac44b", + "workflow_definition_fingerprint_source": "worker" + }, + "workflow_task_id": null, + "workflow_command_id": "01m7tfn1dt9s08nvqqk4kh61am", + "recorded_at": "2026-11-19T18:03:41.376531Z" + } + ], + "expected": { + "command_sequence": [ + { + "type": "record_version_marker", + "change_id": "durable_workflow.parallel_failure_order" + }, + { + "type": "schedule_activity", + "activity_type": "m0", + "parallel_group_base_sequence": 2 + }, + { + "type": "schedule_activity", + "activity_type": "m1", + "parallel_group_base_sequence": 2 + } + ] + } +} diff --git a/tests/fixtures/replay_regressions/parallel-group-metadata-emission-after-failure-order-marker.json b/tests/fixtures/replay_regressions/parallel-failure-order-opt-in-new-run-emission.json similarity index 82% rename from tests/fixtures/replay_regressions/parallel-group-metadata-emission-after-failure-order-marker.json rename to tests/fixtures/replay_regressions/parallel-failure-order-opt-in-new-run-emission.json index 3e124cc3..e939857a 100644 --- a/tests/fixtures/replay_regressions/parallel-group-metadata-emission-after-failure-order-marker.json +++ b/tests/fixtures/replay_regressions/parallel-failure-order-opt-in-new-run-emission.json @@ -1,24 +1,22 @@ { "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", "fixture_schema": "durable-workflow.replay-regression/v1", - "id": "parallel-group-metadata-emission-after-failure-order-marker", + "id": "parallel-failure-order-opt-in-new-run-emission", "protocol_version": "1.20", "bindings": [ "python" ], - "supersedes": [ - "parallel-group-metadata-emission" - ], "parallel_failure_order": { "issues": [ "durable-workflow/sdk-python#307" ], - "note": "The new run of parallel-group-metadata-emission. A run now records the parallel failure-order version marker directly before its first all-group that can fail, so the group's metadata starts at sequence 2. Histories recorded without the marker keep their sequences and their failure selection." + "note": "The new run of parallel-group-metadata-emission with failure-order recording enabled (record_parallel_failure_order). The run records the parallel failure-order version marker at sequence 1, directly before its first all-group, whose metadata then starts at sequence 2. With recording off (the default in this release), parallel-group-metadata-emission still holds unchanged." }, "workflow": { "type": "tests.replay.parallel-metadata-producer", "input": [], - "payload_codec": "avro" + "payload_codec": "avro", + "record_parallel_failure_order": true }, "history": [ { diff --git a/tests/test_activity_cancellation_policies.py b/tests/test_activity_cancellation_policies.py index c5c667c3..f08dfdb3 100644 --- a/tests/test_activity_cancellation_policies.py +++ b/tests/test_activity_cancellation_policies.py @@ -15,7 +15,6 @@ commands_to_server_commands, replay, ) -from tests.failure_order import failure_order_marker_event, without_failure_order_marker from tests.test_cooperative_cancellation import marker, request from tests.test_cooperative_cancellation_worker import ClaimServer, claimed_task @@ -79,13 +78,9 @@ def test_omitted_policy_preserves_wire_and_historical_try_default() -> None: def test_changed_policy_fails_group_matching_and_cancellation_delivery(mode: str, delivered: bool) -> None: options = {"cancellation_policy": CancellationPolicy.WAIT_CANCELLATION_COMPLETED} commands = commands_to_server_commands(replay(workflow(options, mode), [], []).commands, "queue") - # A new run records its failure-order marker before its first all-group. - first = 2 if mode == "parallel" else 1 - history = [failure_order_marker_event()] if mode == "parallel" else [] - history += [{"event_type": "ActivityScheduled", "payload": {"sequence": first, **command}} - for command in without_failure_order_marker(commands)] + history = [{"event_type": "ActivityScheduled", "payload": {"sequence": 1, **command}} for command in commands] if delivered: - history += [request(), marker(first, "activity" if mode == "sequential" else "parallel", sequence_span=1)] + history += [request(), marker(1, "activity" if mode == "sequential" else "parallel", sequence_span=1)] assert replay(workflow(options, mode), history, [], run_id="run-1").commands == [CompleteWorkflow("request-1")] with pytest.raises(NonDeterministicReplayError, match="activity_cancellation_policy_changed"): replay(workflow({"cancellation_policy": CancellationPolicy.TRY_CANCEL}, mode), history, [], run_id="run-1") diff --git a/tests/test_cancellation_scope_authoring.py b/tests/test_cancellation_scope_authoring.py index 4d15388e..02c30a77 100644 --- a/tests/test_cancellation_scope_authoring.py +++ b/tests/test_cancellation_scope_authoring.py @@ -12,7 +12,6 @@ from durable_workflow.errors import NonDeterministicReplayError from durable_workflow.worker import Worker from durable_workflow.workflow import LocalActivityExecutionAborted -from tests.failure_order import is_failure_order_marker from tests.test_cooperative_cancellation import request @@ -148,12 +147,10 @@ def body(): # type: ignore[no-untyped-def] return (yield from ctx.cancellation_scope(body)) outcome = run(Probe, [started(), opening()]) - marker, *wire = workflow.commands_to_server_commands(outcome.commands, "queue", size_warning=None) - # The run's failure-order marker takes sequence 2, before its first all-group. - assert is_failure_order_marker(marker) + wire = workflow.commands_to_server_commands(outcome.commands, "queue", size_warning=None) assert len(wire) == 2 assert [command["cancellation_scope_id"] for command in wire] == ["scope-one", "scope-one"] - assert [command["parallel_group_base_sequence"] for command in wire] == [3, 3] + assert [command["parallel_group_base_sequence"] for command in wire] == [2, 2] assert [command["parallel_group_index"] for command in wire] == [0, 1] diff --git a/tests/test_child_workflow_policies.py b/tests/test_child_workflow_policies.py index a23eae2d..014bca24 100644 --- a/tests/test_child_workflow_policies.py +++ b/tests/test_child_workflow_policies.py @@ -15,7 +15,6 @@ commands_to_server_commands, replay, ) -from tests.failure_order import failure_order_marker_event, is_failure_order_marker from tests.test_cooperative_cancellation import marker, request from tests.test_cooperative_cancellation_worker import ClaimServer, claimed_task @@ -47,10 +46,7 @@ def run(self, ctx: WorkflowContext): # type: ignore[no-untyped-def] def scheduled(commands: list[Any]) -> list[dict[str, Any]]: - """History for these commands. A new run's failure-order marker, recorded - before its first all-group, keeps its own sequence (sdk-python#307).""" return [ - failure_order_marker_event(index + 1) if is_failure_order_marker(command) else {"event_type": "ChildWorkflowScheduled", "payload": { "sequence": index + 1, "child_workflow_type": command["workflow_type"], "child_workflow_run_id": f"child-run-{index + 1}", **command, @@ -118,10 +114,8 @@ def test_existing_positional_constructor_and_omitted_wire_fields_are_preserved() def test_changed_policies_fail_cold_replay_and_cancellation_delivery(mode: str, field: str, delivered: bool) -> None: policies = options() history = scheduled(replay(workflow(mode, policies), [], []).commands) - first = 2 if mode == "parallel" else 1 if delivered: - history += [request(), marker(first, "child" if mode == "sequential" else "parallel", - sequence_span=len(history) - first + 1)] + history += [request(), marker(1, "child" if mode == "sequential" else "parallel", sequence_span=len(history))] assert replay(workflow(mode, policies), history, [], run_id="run-1").commands == [CompleteWorkflow("request-1")] else: cold = replay(workflow(mode, policies), history, []) @@ -129,7 +123,7 @@ def test_changed_policies_fail_cold_replay_and_cancellation_delivery(mode: str, policies[field] = "abandon" with pytest.raises(NonDeterministicReplayError, match="child_workflow_policy_changed") as captured: replay(workflow(mode, policies), history, [], run_id="run-1") - assert captured.value.workflow_sequence == first + assert captured.value.workflow_sequence == 1 @pytest.mark.parametrize("mode", ["sequential", "parallel", "selection"]) diff --git a/tests/test_completion_size_limit.py b/tests/test_completion_size_limit.py index 4b61be84..cb2829d3 100644 --- a/tests/test_completion_size_limit.py +++ b/tests/test_completion_size_limit.py @@ -119,10 +119,7 @@ async def test_typed_413_fails_the_run_once_without_resending_the_completion() - "/api/worker/workflow-tasks/task-413/complete", ] oversized, failure = (body(request) for request in requests) - # A new run records its failure-order marker before its first all-group (#307). - assert [command["type"] for command in oversized["commands"]] == [ - "record_version_marker", *["schedule_activity"] * 4, - ] + assert [command["type"] for command in oversized["commands"]] == ["schedule_activity"] * 4 assert (failure["lease_owner"], failure["workflow_task_attempt"]) == ("worker-413", 3) assert len(failure["commands"]) == 1 command = failure["commands"][0] @@ -280,7 +277,7 @@ async def test_typed_409_fails_the_run_once_without_resending_or_failing_the_tas "/api/worker/workflow-tasks/task-413/complete", ] refused, failure = (body(request) for request in requests) - assert [command["type"] for command in refused["commands"]] == ["record_version_marker", *["schedule_activity"] * 4] + assert [command["type"] for command in refused["commands"]] == ["schedule_activity"] * 4 assert (failure["lease_owner"], failure["workflow_task_attempt"]) == ("worker-413", 3) command = failure["commands"][0] assert (command["type"], command["non_retryable"]) == ("fail_workflow", True) diff --git a/tests/test_parallel_failure_order.py b/tests/test_parallel_failure_order.py index e59dcc21..5483cdfe 100644 --- a/tests/test_parallel_failure_order.py +++ b/tests/test_parallel_failure_order.py @@ -1,22 +1,26 @@ """sdk-python#307: which member failure an all() group raises. -A run records ``PARALLEL_FAILURE_ORDER_MARKER`` directly before the first -all-group that can fail, when a worker with this rule first schedules it. Such -a run raises the first member failure recorded in history, as soon as it is -recorded, whether or not the other members have settled. A run whose first -all-group was recorded without the marker keeps the rule of the workers that -started it for its whole life: on every later task, query and update, a -settled group raises its lowest-position failure. +With recording enabled (``record_parallel_failure_order``, off by default in +this release), a run records ``PARALLEL_FAILURE_ORDER_MARKER`` directly before +the first all-group that can fail, when that group is first scheduled. A marked +run raises the first member failure recorded in history, as soon as it is +recorded, whether or not the other members have settled. Every worker follows a +recorded marker, with recording on or off. A run whose first all-group was +recorded without the marker keeps the rule of the workers that started it for +its whole life: on every later task, query and update, a settled group raises +its lowest-position failure. """ from __future__ import annotations from typing import Any, cast +from unittest.mock import AsyncMock import pytest -from durable_workflow import serializer, workflow +from durable_workflow import Client, serializer, workflow from durable_workflow.errors import ActivityFailed, NonDeterministicReplayError +from durable_workflow.worker import RECORD_PARALLEL_FAILURE_ORDER_ENV, Worker from durable_workflow.workflow import ( PARALLEL_FAILURE_DECISION, CompleteUpdate, @@ -115,8 +119,22 @@ def run(self, ctx: WorkflowContext): # type: ignore[no-untyped-def] return (yield [ctx.schedule_activity(f"member-{index}", []) for index in range(SIZE)]) -def _commands(history: list[dict[str, Any]], definition: type = FailureOrder) -> list[Any]: - return replay(definition, history, [], payload_codec=serializer.AVRO_CODEC).commands +def _commands(history: list[dict[str, Any]], definition: type = FailureOrder, *, record: bool = True) -> list[Any]: + return replay(definition, history, [], payload_codec=serializer.AVRO_CODEC, + record_parallel_failure_order=record).commands + + +def test_recording_is_off_by_default_and_new_runs_send_the_released_commands() -> None: + started = [{"event_type": "WorkflowStarted", "payload": {}}] + commands = replay(FailureOrder, started, [], payload_codec=serializer.AVRO_CODEC).commands + assert FAILURE_ORDER_MARKER not in commands + wire = commands_to_server_commands(commands, "queue", payload_codec=serializer.AVRO_CODEC) + assert [command["parallel_group_base_sequence"] for command in wire] == [1] * SIZE + # The group was recorded without the marker, so the run keeps the + # released rule even for a worker with recording enabled. + history, base = _run(marked=False) + history += _settled_out_of_order(base) + assert _commands(history, record=True) == [ScheduleActivity("compensate", ["member-0"])] def _caught(history: list[dict[str, Any]]) -> Any: @@ -153,7 +171,8 @@ def entry(base: int, index: int) -> dict[str, Any]: history.append({"event_type": "ActivityScheduled", "payload": dict(payload)}) history.append({"event_type": "ActivityCompleted", "payload": { **payload, "result": serializer.envelope(name, codec=serializer.AVRO_CODEC)}}) - commands = replay(TwoGroups, history, [], payload_codec=serializer.AVRO_CODEC).commands + commands = replay(TwoGroups, history, [], payload_codec=serializer.AVRO_CODEC, + record_parallel_failure_order=True).commands assert [type(command) for command in commands] == [ScheduleActivity, ScheduleActivity] wire = commands_to_server_commands(commands, "queue", payload_codec=serializer.AVRO_CODEC) assert [command["parallel_group_base_sequence"] for command in wire] == [4, 4] @@ -167,10 +186,10 @@ def run(self, ctx: WorkflowContext): # type: ignore[no-untyped-def] yield [ctx.schedule_activity("work", []), ctx.schedule_activity("more", [])] return "done" - commands = replay(TimersFirst, [], []).commands + commands = replay(TimersFirst, [], [], record_parallel_failure_order=True).commands assert [type(command) for command in commands] == [StartTimer, StartTimer] fired = [{"event_type": "TimerFired", "payload": {"sequence": sequence}} for sequence in (1, 2)] - commands = replay(TimersFirst, fired, []).commands + commands = replay(TimersFirst, fired, [], record_parallel_failure_order=True).commands assert commands[0] == FAILURE_ORDER_MARKER assert [type(command) for command in commands[1:]] == [ScheduleActivity, ScheduleActivity] @@ -186,7 +205,8 @@ def run(self, ctx: WorkflowContext): # type: ignore[no-untyped-def] history = [{"event_type": "WorkflowStarted", "payload": {}}, _activity("ActivityScheduled", 1, "prepare"), _activity("ActivityCompleted", 1, "prepare", "ready")] - commands = replay(LateGroup, history, [], payload_codec=serializer.AVRO_CODEC).commands + commands = replay(LateGroup, history, [], payload_codec=serializer.AVRO_CODEC, + record_parallel_failure_order=True).commands assert commands[0] == FAILURE_ORDER_MARKER wire = commands_to_server_commands(commands[1:], "queue", payload_codec=serializer.AVRO_CODEC) assert [command["parallel_group_base_sequence"] for command in wire] == [3, 3] @@ -195,27 +215,30 @@ def run(self, ctx: WorkflowContext): # type: ignore[no-untyped-def] def test_a_cancellation_request_keeps_the_earlier_rule() -> None: """A request may replace the group with its cancellation, so the run records no marker and its group is recorded under the earlier rule.""" - outcome = replay(FailureOrder, [request()], [], run_id="run-1", payload_codec=serializer.AVRO_CODEC) + outcome = replay(FailureOrder, [request()], [], run_id="run-1", payload_codec=serializer.AVRO_CODEC, + record_parallel_failure_order=True) assert FAILURE_ORDER_MARKER not in outcome.commands assert outcome.cancellation_delivery is not None -def test_a_marked_run_raises_the_first_recorded_failure_of_a_settled_group() -> None: +@pytest.mark.parametrize("record", [False, True]) +def test_a_marked_run_raises_the_first_recorded_failure_of_a_settled_group(record: bool) -> None: history, base = _run(marked=True) history += _settled_out_of_order(base) - assert _commands(history) == [ScheduleActivity("compensate", ["member-2"])] + assert _commands(history, record=record) == [ScheduleActivity("compensate", ["member-2"])] assert _caught(history) == "member-2" - uncaught = _commands(history, FailureOrderUncaught) + uncaught = _commands(history, FailureOrderUncaught, record=record) assert [type(command) for command in uncaught] == [FailWorkflow] assert "member-2" in str(cast(FailWorkflow, uncaught[0]).message) -def test_a_marked_run_raises_at_once_without_recording_a_decision() -> None: +@pytest.mark.parametrize("record", [False, True]) +def test_a_marked_run_raises_at_once_without_recording_a_decision(record: bool) -> None: """History is append-only: the first failure this task sees is the first failure every later replay sees, so nothing more needs recording.""" history, base = _run(marked=True) history += [_member("ActivityCompleted", 0, base), _member("ActivityFailed", 2, base)] - assert _commands(history) == [ScheduleActivity("compensate", ["member-2"])] + assert _commands(history, record=record) == [ScheduleActivity("compensate", ["member-2"])] # Member 3 fails and member 1 completes after the compensation was # scheduled; replay still raises member 2. later = [*history, _activity("ActivityScheduled", base + SIZE, "compensate"), @@ -223,11 +246,12 @@ def test_a_marked_run_raises_at_once_without_recording_a_decision() -> None: _activity("ActivityCompleted", base + SIZE, "compensate", "undone")] for end in range(len(history) + 1, len(later) + 1): expected = [CompleteWorkflow("undone")] if end == len(later) else [] - assert _commands(later[:end]) == expected, end + assert _commands(later[:end], record=record) == expected, end assert _caught(later[:end]) == "member-2", end -def test_a_marked_run_keeps_its_choice_when_a_lower_position_fails_before_the_completion() -> None: +@pytest.mark.parametrize("record", [False, True]) +def test_a_marked_run_keeps_its_choice_when_a_lower_position_fails_before_the_completion(record: bool) -> None: """The deciding task saw only member 2's failure. Member 0 failed and the others completed before its completion was written.""" history, base = _run(marked=True) @@ -237,23 +261,25 @@ def test_a_marked_run_keeps_its_choice_when_a_lower_position_fails_before_the_co _activity("ActivityScheduled", base + SIZE, "compensate"), _activity("ActivityCompleted", base + SIZE, "compensate", "undone"), ] - assert _commands(history) == [CompleteWorkflow("undone")] + assert _commands(history, record=record) == [CompleteWorkflow("undone")] assert _caught(history) == "member-2" -def test_an_unmarked_run_keeps_the_position_rule_at_every_task() -> None: +@pytest.mark.parametrize("record", [False, True]) +def test_an_unmarked_run_keeps_the_position_rule_at_every_task(record: bool) -> None: """The group was scheduled without the marker, by a released worker.""" history, base = _run(marked=False) history += _settled_out_of_order(base) - assert _commands(history) == [ScheduleActivity("compensate", ["member-0"])] + assert _commands(history, record=record) == [ScheduleActivity("compensate", ["member-0"])] assert _caught(history) == "member-0" moved_on = [*history, _activity("ActivityScheduled", base + SIZE, "compensate"), _activity("ActivityCompleted", base + SIZE, "compensate", "undone")] - assert _commands(moved_on) == [CompleteWorkflow("undone")] + assert _commands(moved_on, record=record) == [CompleteWorkflow("undone")] assert _caught(moved_on) == "member-0" -def test_an_unmarked_run_whose_task_recorded_nothing_after_the_group_keeps_the_position_rule() -> None: +@pytest.mark.parametrize("record", [False, True]) +def test_an_unmarked_run_whose_task_recorded_nothing_after_the_group_keeps_the_position_rule(record: bool) -> None: """A released worker caught member 0 and then waited without recording a command (for example on a select() handle it already held). A later task must not take that for a new decision.""" @@ -261,7 +287,7 @@ def test_an_unmarked_run_whose_task_recorded_nothing_after_the_group_keeps_the_p history += _settled_out_of_order(base) history.append({"event_type": "SignalReceived", "payload": {"signal_name": "nudge", "arguments": None}}) assert _caught(history) == "member-0" - commands = _commands(history) + commands = _commands(history, record=record) assert not any(isinstance(command, RecordSideEffect) for command in commands) assert commands == [ScheduleActivity("compensate", ["member-0"])] @@ -280,7 +306,8 @@ def test_an_update_in_an_unmarked_run_reports_the_position_rule() -> None: assert update == CompleteUpdate("update-1", {"compensating": "member-2"}) -def test_an_unmarked_run_still_honours_a_recorded_decision() -> None: +@pytest.mark.parametrize("record", [False, True]) +def test_an_unmarked_run_still_honours_a_recorded_decision(record: bool) -> None: """2.19 records which member it raised while siblings were open (#338).""" history, base = _run(marked=False) decision = {PARALLEL_FAILURE_DECISION: { @@ -293,7 +320,7 @@ def test_an_unmarked_run_still_honours_a_recorded_decision() -> None: {"event_type": "SideEffectRecorded", "payload": { "sequence": base + SIZE, "result": serializer.envelope(decision, codec=serializer.AVRO_CODEC)}}, ] - assert _commands(history) == [ScheduleActivity("compensate", ["member-2"])] + assert _commands(history, record=record) == [ScheduleActivity("compensate", ["member-2"])] def test_an_unknown_marker_version_is_nondeterministic() -> None: @@ -394,3 +421,44 @@ def run(self, ctx: WorkflowContext, *_: Any): # type: ignore[no-untyped-def] self.caught = str(e) yield ctx.schedule_activity("compensate", [self.caught], retry_policy=_ONCE) return self.caught + + +@pytest.mark.parametrize("raw,expected", [(None, False), ("", False), ("false", False), ("0", False), + ("true", True), ("1", True), (" TRUE ", True)]) +def test_the_worker_setting_reads_its_environment_variable( + monkeypatch: pytest.MonkeyPatch, raw: str | None, expected: bool, +) -> None: + if raw is None: + monkeypatch.delenv(RECORD_PARALLEL_FAILURE_ORDER_ENV, raising=False) + else: + monkeypatch.setenv(RECORD_PARALLEL_FAILURE_ORDER_ENV, raw) + worker = Worker(AsyncMock(spec=Client), task_queue="q", workflows=[FailureOrder]) + assert worker.record_parallel_failure_order is expected + assert Worker(AsyncMock(spec=Client), task_queue="q", workflows=[FailureOrder], + record_parallel_failure_order=not expected).record_parallel_failure_order is (not expected) + + +@pytest.mark.parametrize("raw", ["yes", "on", "2", "v2"]) +def test_a_malformed_worker_setting_is_refused(monkeypatch: pytest.MonkeyPatch, raw: str) -> None: + monkeypatch.setenv(RECORD_PARALLEL_FAILURE_ORDER_ENV, raw) + with pytest.raises(ValueError, match=RECORD_PARALLEL_FAILURE_ORDER_ENV): + Worker(AsyncMock(spec=Client), task_queue="q", workflows=[FailureOrder]) + monkeypatch.delenv(RECORD_PARALLEL_FAILURE_ORDER_ENV) + with pytest.raises(ValueError, match="record_parallel_failure_order"): + Worker(AsyncMock(spec=Client), task_queue="q", workflows=[FailureOrder], + record_parallel_failure_order="true") # type: ignore[arg-type] + + +@pytest.mark.parametrize("record", [False, True]) +@pytest.mark.asyncio +async def test_the_worker_writes_the_marker_only_when_enabled(record: bool) -> None: + client = AsyncMock(spec=Client) + worker = Worker(client, task_queue="q", workflows=[FailureOrder], record_parallel_failure_order=record) + await worker._run_workflow_task({ + "task_id": "t-307", "workflow_type": "tests.parallel-failure-order", "workflow_task_attempt": 1, + "history_events": [{"event_type": "WorkflowStarted", "payload": {}}], + "arguments": serializer.encode([], codec="avro"), "payload_codec": "avro", + }) + commands = client.complete_workflow_task.call_args.kwargs["commands"] + types = [command["type"] for command in commands] + assert types == (["record_version_marker"] if record else []) + ["schedule_activity"] * SIZE diff --git a/tests/test_parallel_member_failure_open_siblings.py b/tests/test_parallel_member_failure_open_siblings.py index 107a03e1..fb3493fd 100644 --- a/tests/test_parallel_member_failure_open_siblings.py +++ b/tests/test_parallel_member_failure_open_siblings.py @@ -123,9 +123,7 @@ def test_an_open_group_is_never_scheduled_again() -> None: def test_a_new_group_is_still_scheduled_once() -> None: - marker, *commands = _wire([{"event_type": "WorkflowStarted", "payload": {}}]) - # A new run records its failure-order marker before its first all-group (#307). - assert marker["type"] == "record_version_marker" + commands = _wire([{"event_type": "WorkflowStarted", "payload": {}}]) assert [command["type"] for command in commands] == ["schedule_activity"] * SIZE assert [command["parallel_group_index"] for command in commands] == list(range(SIZE)) @@ -352,8 +350,9 @@ def run(self, ctx: WorkflowContext): # type: ignore[no-untyped-def] # tests/fixtures/replay_regressions/settled-group-*.json (sdk-python#307). # Member 1 fails at once and member 0 fails later; the others complete. The # workflow worker was stopped while they ran, so the woken task saw the whole -# group settled. Published 2.19.1 raised member 0 by position; this version -# raises member 1, whose failure was recorded first. +# group settled. Published 2.19.1 raised member 0 by position; a run that +# recorded the failure-order marker raises member 1, whose failure was recorded +# first. PY307_ACTIVITIES = "py307-activities" diff --git a/tests/test_parallel_saga_authoring.py b/tests/test_parallel_saga_authoring.py index 3a9df47b..8e609a21 100644 --- a/tests/test_parallel_saga_authoring.py +++ b/tests/test_parallel_saga_authoring.py @@ -14,7 +14,6 @@ commands_to_server_commands, replay, ) -from tests.failure_order import FAILURE_ORDER_MARKER def _entry(kind: str, base: int, size: int, index: int) -> dict[str, Any]: @@ -33,12 +32,12 @@ def _entry(kind: str, base: int, size: int, index: int) -> dict[str, Any]: } -def _paths(base: int = 1) -> list[list[dict[str, Any]]]: - outer = [_entry("mixed", base, 3, index) for index in range(3)] +def _paths() -> list[list[dict[str, Any]]]: + outer = [_entry("mixed", 1, 3, index) for index in range(3)] return [ [outer[0]], - [outer[1], _entry("mixed", base + 1, 2, 0)], - [outer[2], _entry("mixed", base + 1, 2, 1)], + [outer[1], _entry("mixed", 2, 2, 0)], + [outer[2], _entry("mixed", 2, 2, 1)], ] @@ -93,17 +92,14 @@ def run(self, ctx: WorkflowContext): # type: ignore[no-untyped-def] def test_nested_parallel_emits_flat_commands_with_stable_full_paths() -> None: outcome = replay(NestedParallelWorkflow, [], []) - # A new run records its failure-order marker at sequence 1, before its - # first all-group (sdk-python#307). - assert outcome.commands[0] == FAILURE_ORDER_MARKER - assert [type(command) for command in outcome.commands[1:]] == [ + assert [type(command) for command in outcome.commands] == [ ScheduleActivity, StartChildWorkflow, StartTimer, ] - commands = commands_to_server_commands(outcome.commands[1:], "parallel-workers") - for command, path in zip(commands, _paths(2), strict=True): + commands = commands_to_server_commands(outcome.commands, "parallel-workers") + for command, path in zip(commands, _paths(), strict=True): assert command["parallel_group_path"] == path assert command["parallel_group_id"] == path[-1]["parallel_group_id"] diff --git a/tests/test_replay.py b/tests/test_replay.py index 0f5b5933..3c9048bf 100644 --- a/tests/test_replay.py +++ b/tests/test_replay.py @@ -35,7 +35,6 @@ commands_to_server_commands, replay, ) -from tests.failure_order import FAILURE_ORDER_MARKER from tests.integration.polyglot_fixtures import ( PHP_HISTORY_EVENT_EXPECTED_KEYS, PHP_HISTORY_EVENT_FIXTURES, @@ -2261,19 +2260,15 @@ def run(self, ctx: WorkflowContext): # type: ignore[no-untyped-def] class TestFanOut: def test_no_history_emits_batch(self) -> None: outcome = replay(FanOutWorkflow, [], []) - # A new run records its failure-order marker before its first - # all-group (sdk-python#307). - assert len(outcome.commands) == 3 - assert outcome.commands[0] == FAILURE_ORDER_MARKER + assert len(outcome.commands) == 2 + assert isinstance(outcome.commands[0], ScheduleActivity) + assert outcome.commands[0].activity_type == "fetch-a" assert isinstance(outcome.commands[1], ScheduleActivity) - assert outcome.commands[1].activity_type == "fetch-a" - assert isinstance(outcome.commands[2], ScheduleActivity) - assert outcome.commands[2].activity_type == "fetch-b" + assert outcome.commands[1].activity_type == "fetch-b" def test_mixed_batch_emits_language_neutral_parallel_metadata(self) -> None: outcome = replay(ParallelMetadataProducerWorkflow, [], []) - assert outcome.commands[0] == FAILURE_ORDER_MARKER - commands = commands_to_server_commands(outcome.commands[1:], "parallel-workers") + commands = commands_to_server_commands(outcome.commands, "parallel-workers") assert [command["type"] for command in commands] == [ "schedule_activity", @@ -2282,9 +2277,9 @@ def test_mixed_batch_emits_language_neutral_parallel_metadata(self) -> None: ] for index, command in enumerate(commands): entry = { - "parallel_group_id": "parallel-calls:2:3", + "parallel_group_id": "parallel-calls:1:3", "parallel_group_kind": "mixed", - "parallel_group_base_sequence": 2, + "parallel_group_base_sequence": 1, "parallel_group_size": 3, "parallel_group_index": index, } @@ -2293,15 +2288,14 @@ def test_mixed_batch_emits_language_neutral_parallel_metadata(self) -> None: def test_repeated_deferred_command_keeps_occurrence_specific_metadata(self) -> None: outcome = replay(RepeatedCommandFanOutWorkflow, [], []) - assert outcome.commands[0] == FAILURE_ORDER_MARKER - commands = commands_to_server_commands(outcome.commands[1:], "parallel-workers") + commands = commands_to_server_commands(outcome.commands, "parallel-workers") - assert outcome.commands[1] is not outcome.commands[2] + assert outcome.commands[0] is not outcome.commands[1] for index, command in enumerate(commands): entry = { - "parallel_group_id": "parallel-activities:2:2", + "parallel_group_id": "parallel-activities:1:2", "parallel_group_kind": "activity", - "parallel_group_base_sequence": 2, + "parallel_group_base_sequence": 1, "parallel_group_size": 2, "parallel_group_index": index, } @@ -2520,9 +2514,8 @@ def run(self, ctx: WorkflowContext): # type: ignore[no-untyped-def] class TestNontrivialWorkflow: def test_first_replay_fans_out(self) -> None: outcome = replay(NontrivialWorkflow, [], []) - assert outcome.commands[0] == FAILURE_ORDER_MARKER - assert len(outcome.commands) == 3 - assert all(isinstance(c, ScheduleActivity) for c in outcome.commands[1:]) + assert len(outcome.commands) == 2 + assert all(isinstance(c, ScheduleActivity) for c in outcome.commands) def test_after_fan_out_starts_timer(self) -> None: history = [ diff --git a/tests/test_replay_regression_corpus.py b/tests/test_replay_regression_corpus.py index 913c633f..c0f13a4c 100644 --- a/tests/test_replay_regression_corpus.py +++ b/tests/test_replay_regression_corpus.py @@ -517,19 +517,6 @@ def run(self, ctx: WorkflowContext, message: str) -> None: "run-closed-cancellation-not-a-group-outcome": "run-closed-all-failure-keeps-the-handled-failure", } -# Immutable evidence of the commands a released worker sent for a new run's -# first all-group. New runs now record the parallel failure-order version -# marker before that group (sdk-python#307), so the same input replays under -# the named fixture, at the same protocol version. No recorded history changes -# meaning: a history without the marker keeps its sequences and its failure -# selection. -SUPERSEDED_NEW_RUN_EMISSION = { - "python-completion-size-limit-fails-run-once": - "python-completion-size-limit-fails-run-once-after-failure-order-marker", - "python-completion-history-limit-fails-run-once": - "python-completion-history-limit-fails-run-once-after-failure-order-marker", -} - def _fixture_documents() -> dict[Path, dict[str, Any]]: return {path: json.loads(path.read_text(encoding="utf-8")) for path in sorted(FIXTURE_DIR.glob("*.json"))} @@ -538,7 +525,7 @@ def _fixture_documents() -> dict[Path, dict[str, Any]]: def _fixture_paths() -> list[Path]: """The current fixtures. A superseded one stays as evidence but does not replay.""" documents = _fixture_documents() - retired = set(DEFECT_EVIDENCE) | set(SUPERSEDED_NEW_RUN_EMISSION) + retired = set(DEFECT_EVIDENCE) for document in documents.values(): retired.update(document.get("supersedes", [])) return [path for path, document in documents.items() if document.get("id") not in retired] @@ -546,9 +533,9 @@ def _fixture_paths() -> list[Path]: def test_retired_defect_evidence_replays_under_its_correction() -> None: by_id = {document["id"]: document for document in _fixture_documents().values()} - for retired, correction in {**DEFECT_EVIDENCE, **SUPERSEDED_NEW_RUN_EMISSION}.items(): + for retired, correction in DEFECT_EVIDENCE.items(): assert by_id[retired]["protocol_version"] == by_id[correction]["protocol_version"] - assert by_id[retired].get("history") == by_id[correction].get("history") + assert by_id[retired]["history"] == by_id[correction]["history"] assert by_id[retired]["workflow"] == by_id[correction]["workflow"] @@ -881,6 +868,7 @@ def _execute_fixture(fixture: dict[str, Any]) -> list[dict[str, Any]]: workflow_id=workflow.get("workflow_id"), run_id=workflow.get("run_id", ""), payload_codec=("json" if payload_codec is None else payload_codec), + record_parallel_failure_order=workflow.get("record_parallel_failure_order") is True, ) commands = _command_documents(outcome.commands) message_stream_cursors = outcome.message_stream_cursors diff --git a/tests/test_worker.py b/tests/test_worker.py index 672ce52c..6c2959b1 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -1903,8 +1903,7 @@ def spy_envelope_many(values, *args, **kwargs): # type: ignore[no-untyped-def] await worker._run_workflow_task(task) - marker, *commands = mock_client.complete_workflow_task.call_args.kwargs["commands"] - assert marker["type"] == "record_version_marker" + commands = mock_client.complete_workflow_task.call_args.kwargs["commands"] assert [command["activity_type"] for command in commands] == ["first", "second"] assert calls == [[["a"], ["b"]]]