From 65d02dcd5d55c56837d7ffc869ccc9b2176a49ba Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 19 Nov 2026 10:14:01 +0000 Subject: [PATCH 01/12] Keep fan-out workflow tasks linear in the member count (#1231, #1203) A 300-member all() of 10 KB activities took 134 s to schedule and 58-85 s to finish on 2.32.0, so the default 60-second queue job timeout killed the final workflow task on every retry. Profiling the real jobs on SQLite and MySQL found per-member steps that rescanned the whole run: - Publishing each member's task after commit rebuilt the run's whole summary, in its own autocommit transaction (90% of the scheduling task). RunWorkflowTask now publishes the tasks it committed in bounded batch transactions and refreshes each affected run summary once. - Replay checked every member with WorkflowStepHistory scans over the whole history, and the executor, ParallelChildGroup, ChildRunHistory and RunLineageView looked up each member's events, links and policy outcomes the same way. Each lookup family now indexes the loaded history once (HistoryEventIndex), keyed by the collection and invalidated when it grows. A child closure's parent projection was cubic in the width before this. - Activity starts refresh only their own timeline points, like outcomes, and the scoped timeline and failure snapshots skip decoding payloads whose stored text cannot name the activity or a failure. Recorded history and the final projection rows are identical to 2.32.0 for activity and child fan-outs. New replay-regression evidence records both fan-outs; its consumer proves lookups for one member no longer read other members' events (it fails on main) and that a live run records the evidence history unchanged. V2FanOutWidthScalingTest bounds 300 members to 2.5 times linear growth over 30, and the fan-out scaling job runs the literal 1,000-activity and 1,000-child cases. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/fan-out-scaling.yml | 106 +++ CHANGELOG.md | 19 + src/V2/Jobs/RunWorkflowTask.php | 7 + src/V2/Support/ChildRunHistory.php | 128 ++- .../Support/DefaultHistoryProjectionRole.php | 12 +- src/V2/Support/FailureSnapshots.php | 6 +- src/V2/Support/HistoryEventIndex.php | 71 ++ src/V2/Support/HistoryTimeline.php | 16 +- src/V2/Support/ParallelChildGroup.php | 133 ++- src/V2/Support/RunLineageView.php | 55 +- src/V2/Support/RunSummaryProjector.php | 1 + src/V2/Support/StoredHistoryPayload.php | 37 + src/V2/Support/TaskDispatcher.php | 211 ++++- src/V2/Support/WorkflowExecutor.php | 87 +- src/V2/Support/WorkflowStepHistory.php | 75 +- .../V2EmbeddedReplayRegressionCorpusTest.php | 213 +++++ tests/Feature/V2/V2FanOutWidthScalingTest.php | 117 +++ .../fan-out-activity-all-history-index.json | 799 ++++++++++++++++++ .../fan-out-child-all-history-index.json | 394 +++++++++ .../Fixtures/V2/TestFanOutPayloadActivity.php | 17 + .../V2/TestFanOutPayloadActivityWorkflow.php | 34 + .../V2/TestFanOutPayloadChildWorkflow.php | 34 + .../V2/TestFanOutPayloadLeafWorkflow.php | 17 + tests/Performance/FanOutWidthLiteralTest.php | 105 +++ tests/Support/FanOutWidthProbe.php | 208 +++++ tests/Support/MemberPhaseProjectionRole.php | 69 ++ 26 files changed, 2802 insertions(+), 169 deletions(-) create mode 100644 .github/workflows/fan-out-scaling.yml create mode 100644 src/V2/Support/HistoryEventIndex.php create mode 100644 src/V2/Support/StoredHistoryPayload.php create mode 100644 tests/Feature/V2/V2FanOutWidthScalingTest.php create mode 100644 tests/Fixtures/V2/ReplayRegression/fan-out-activity-all-history-index.json create mode 100644 tests/Fixtures/V2/ReplayRegression/fan-out-child-all-history-index.json create mode 100644 tests/Fixtures/V2/TestFanOutPayloadActivity.php create mode 100644 tests/Fixtures/V2/TestFanOutPayloadActivityWorkflow.php create mode 100644 tests/Fixtures/V2/TestFanOutPayloadChildWorkflow.php create mode 100644 tests/Fixtures/V2/TestFanOutPayloadLeafWorkflow.php create mode 100644 tests/Performance/FanOutWidthLiteralTest.php create mode 100644 tests/Support/FanOutWidthProbe.php create mode 100644 tests/Support/MemberPhaseProjectionRole.php diff --git a/.github/workflows/fan-out-scaling.yml b/.github/workflows/fan-out-scaling.yml new file mode 100644 index 000000000..f0ace9201 --- /dev/null +++ b/.github/workflows/fan-out-scaling.yml @@ -0,0 +1,106 @@ +name: Fan-out scaling + +# The literal documented fan-out limit (#1203, #1231): one all() of 1,000 +# activities and one of 1,000 child workflows with 10 KB arguments. Each +# workflow task must finish well inside the default 60-second queue job +# timeout and cost roughly ten times the 100-member run's. The 30-versus-300 +# member guard runs in the ordinary feature shards. + +on: + pull_request: + branches: [main] + paths: + - 'src/V2/**' + - 'tests/Performance/FanOutWidthLiteralTest.php' + - 'tests/Support/FanOutWidthProbe.php' + - 'tests/Support/MemberPhaseProjectionRole.php' + - 'tests/Fixtures/V2/TestFanOutPayload*.php' + - '.github/workflows/fan-out-scaling.yml' + schedule: + - cron: '17 4 * * *' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: fan-out-scaling-${{ github.event_name }}-${{ github.event_name == 'pull_request' && github.ref || github.sha }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + literal-fan-out: + name: Literal 1,000-member fan-out (${{ matrix.database }}) + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + database: [sqlite, mysql] + + services: + mysql: + image: mysql + env: + MYSQL_ROOT_PASSWORD: password + options: >- + --tmpfs /var/lib/mysql:rw,noexec,nosuid,size=2048m + ports: + - 3306 + + redis: + image: redis + ports: + - 6379 + + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + + - name: Install dependencies + run: composer install --prefer-dist --no-progress + + - name: Resolve service endpoints and export test defaults + env: + DATABASE: ${{ matrix.database }} + MYSQL_PUBLISHED_PORT: ${{ job.services.mysql.ports['3306'] }} + REDIS_PUBLISHED_PORT: ${{ job.services.redis.ports['6379'] }} + run: | + { + echo "APP_KEY=base64:i3g6f+dV8FfsIkcxqd7gbiPn2oXk5r00sTmdD6V5utI=" + echo "REDIS_HOST=127.0.0.1" + echo "REDIS_PORT=${REDIS_PUBLISHED_PORT}" + echo "REDIS_PASSWORD=" + echo "QUEUE_CONNECTION=redis" + echo "QUEUE_FAILED_DRIVER=null" + echo "CACHE_DRIVER=array" + echo "CACHE_STORE=array" + if [ "$DATABASE" = mysql ]; then + echo "DB_CONNECTION=mysql" + echo "DB_HOST=127.0.0.1" + echo "DB_PORT=${MYSQL_PUBLISHED_PORT}" + echo "DB_DATABASE=testbench" + echo "DB_USERNAME=root" + echo "DB_PASSWORD=password" + else + echo "DB_CONNECTION=sqlite" + echo "DB_DATABASE=${RUNNER_TEMP}/fan-out-scaling.sqlite" + fi + } >> "$GITHUB_ENV" + + - name: Create the database + run: | + if [ "$DB_CONNECTION" = mysql ]; then + mysql -e 'CREATE DATABASE testbench' -h"$DB_HOST" -uroot -ppassword -P "$DB_PORT" + else + touch "$DB_DATABASE" + fi + + - name: Run the literal 1,000-activity and 1,000-child fan-outs + run: >- + vendor/bin/phpunit --no-coverage --fail-on-skipped + --log-junit build/test-results/fan-out-scaling.xml + tests/Performance/FanOutWidthLiteralTest.php + env: + FANOUT_LITERAL: '1' + XDEBUG_MODE: off diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c1deee00..5a3c48525 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,25 @@ ## Unreleased +- A wide `all()` no longer makes its workflow tasks grow with the square of the + fan-out width (#1231, #1203). A 300-member fan-out of 10 KB activities took + 134 s to schedule and 58–85 s to finish on 2.32.0, so Laravel's default + 60-second queue job timeout killed it on every retry. Per-member cost is now + roughly constant: + - A workflow task publishes the tasks it committed in bounded batches and + refreshes each affected run summary once, instead of rebuilding the run's + whole summary, in its own transaction, for every member it published. + - Replay, child-call, parallel-group, lineage and activity lookups index the + loaded history by sequence once, instead of rescanning the whole history + for every member. A child closure no longer rescans the parent's history + for every sibling. + - Activity starts refresh only their own timeline points, like outcomes, and + projections skip decoding payloads that cannot name the activity or a + failure. + + Recorded history and every projection row are unchanged. Each member's own + activity or child task still rebuilds its run summary once. + ## 2.32.0 - 2026-11-19 Compensation scopes remain **experimental** in 2.32.0, without a diff --git a/src/V2/Jobs/RunWorkflowTask.php b/src/V2/Jobs/RunWorkflowTask.php index 19ef65c46..456fbef0b 100644 --- a/src/V2/Jobs/RunWorkflowTask.php +++ b/src/V2/Jobs/RunWorkflowTask.php @@ -55,6 +55,13 @@ public function __construct( } public function handle(WorkflowExecutor $executor): void + { + // A fan-out publishes one task per member after commit. Refresh each + // affected run summary once for the whole task (#1231). + TaskDispatcher::coalescingRunSummaries(fn () => $this->handleTask($executor)); + } + + private function handleTask(WorkflowExecutor $executor): void { // Apps sharing a database and queue names can receive each other's // jobs. Refuse a task outside this app's namespaces before any effect. diff --git a/src/V2/Support/ChildRunHistory.php b/src/V2/Support/ChildRunHistory.php index 04bf1bddf..c7ffe7147 100644 --- a/src/V2/Support/ChildRunHistory.php +++ b/src/V2/Support/ChildRunHistory.php @@ -5,6 +5,7 @@ namespace Workflow\V2\Support; use Carbon\CarbonInterface; +use Illuminate\Support\Collection; use RuntimeException; use Throwable; use Workflow\Serializers\Serializer; @@ -292,53 +293,43 @@ public static function resolutionEventTypes(): array public static function scheduledEventForSequence(WorkflowRun $run, int $sequence): ?WorkflowHistoryEvent { - /** @var WorkflowHistoryEvent|null $event */ - $event = $run->historyEvents->first( - static fn (WorkflowHistoryEvent $event): bool => $event->event_type === HistoryEventType::ChildWorkflowScheduled - && ($event->payload['sequence'] ?? null) === $sequence - ); - - return $event; + return self::childEventsBySequence($run)['scheduled'][$sequence] ?? null; } public static function startedEventForSequence(WorkflowRun $run, int $sequence): ?WorkflowHistoryEvent { - /** @var WorkflowHistoryEvent|null $event */ - $event = $run->historyEvents - ->filter( - static fn (WorkflowHistoryEvent $event): bool => $event->event_type === HistoryEventType::ChildRunStarted - && ($event->payload['sequence'] ?? null) === $sequence - ) - ->sortByDesc('sequence') - ->first(); - - return $event; + return self::childEventsBySequence($run)['started'][$sequence] ?? null; } public static function resolutionEventForSequence(WorkflowRun $run, int $sequence): ?WorkflowHistoryEvent { - /** @var WorkflowHistoryEvent|null $event */ - $event = $run->historyEvents->first( - static fn (WorkflowHistoryEvent $event): bool => in_array( - $event->event_type, - self::resolutionEventTypes(), - true - ) - && ($event->payload['sequence'] ?? null) === $sequence - ); - - return $event; + return self::childEventsBySequence($run)['resolution'][$sequence] ?? null; } public static function latestLinkForSequence(WorkflowRun $run, int $sequence): ?WorkflowLink { - /** @var WorkflowLink|null $link */ - $link = $run->childLinks - ->filter( - static fn (WorkflowLink $link): bool => $link->link_type === 'child_workflow' - && $link->sequence === $sequence - ) - ->sort(static function (WorkflowLink $left, WorkflowLink $right): int { + $linksBySequence = HistoryEventIndex::remember( + $run->childLinks, + 'child-workflow-links-by-sequence', + static function (Collection $links): array { + $index = []; + + foreach ($links as $link) { + if ($link instanceof WorkflowLink && $link->link_type === 'child_workflow' && is_int( + $link->sequence + )) { + $index[$link->sequence][] = $link; + } + } + + return $index; + }, + ); + + $candidates = $linksBySequence[$sequence] ?? []; + + if (count($candidates) > 1) { + usort($candidates, static function (WorkflowLink $left, WorkflowLink $right): int { $leftRunNumber = $left->childRun?->run_number ?? 0; $rightRunNumber = $right->childRun?->run_number ?? 0; @@ -354,10 +345,10 @@ public static function latestLinkForSequence(WorkflowRun $run, int $sequence): ? } return $right->id <=> $left->id; - }) - ->first(); + }); + } - return $link; + return $candidates[0] ?? null; } /** @@ -1132,6 +1123,67 @@ private static function followContinuedRun(?WorkflowRun $childRun): ?WorkflowRun return $childRun; } + /** + * The parent's child-call history events by durable sequence. A child + * projection looks up every fan-out member, so the loaded history is + * indexed once instead of scanned per member (#1203, #1231). + * + * Scheduled and resolution events keep the first match in history order; + * a started event keeps the one with the highest history sequence. + * + * @return array{ + * scheduled: array, + * started: array, + * resolution: array + * } + */ + private static function childEventsBySequence(WorkflowRun $run): array + { + return HistoryEventIndex::remember( + $run->historyEvents, + 'child-run-history-by-sequence', + static function (Collection $events): array { + $index = [ + 'scheduled' => [], + 'started' => [], + 'resolution' => [], + ]; + $resolutionTypes = self::resolutionEventTypes(); + + foreach ($events as $event) { + if (! $event instanceof WorkflowHistoryEvent) { + continue; + } + + $kind = match (true) { + $event->event_type === HistoryEventType::ChildWorkflowScheduled => 'scheduled', + $event->event_type === HistoryEventType::ChildRunStarted => 'started', + in_array($event->event_type, $resolutionTypes, true) => 'resolution', + default => null, + }; + + if ($kind === null) { + continue; + } + + $sequence = $event->payload['sequence'] ?? null; + + if (! is_int($sequence)) { + continue; + } + + $existing = $index[$kind][$sequence] ?? null; + + if ($existing === null || ($kind === 'started' && $event->sequence > $existing->sequence)) { + $index[$kind][$sequence] = $event; + } + } + + return $index; + }, + ); + } + private static function stringValue(mixed $value): ?string { return is_string($value) && $value !== '' diff --git a/src/V2/Support/DefaultHistoryProjectionRole.php b/src/V2/Support/DefaultHistoryProjectionRole.php index 5e4e9ab30..a55a1f8f5 100644 --- a/src/V2/Support/DefaultHistoryProjectionRole.php +++ b/src/V2/Support/DefaultHistoryProjectionRole.php @@ -55,7 +55,7 @@ public function recordActivityStarted( $parallelMetadataPath = ParallelChildGroup::metadataPathForSequence($run, (int) $execution->sequence); $parallelMetadata = ParallelChildGroup::payloadForPath($parallelMetadataPath); - WorkflowHistoryEvent::record($run, HistoryEventType::ActivityStarted, array_merge([ + $started = WorkflowHistoryEvent::record($run, HistoryEventType::ActivityStarted, array_merge([ 'activity_execution_id' => $execution->id, 'activity_attempt_id' => $attempt->id, 'activity_class' => $execution->activity_class, @@ -74,7 +74,15 @@ public function recordActivityStarted( (int) $attempt->attempt_number, ); - return $this->projectRun($run->fresh(['instance', 'tasks', 'activityExecutions', 'failures']) ?? $run); + // Like an outcome, a start changes only its own activity's timeline + // points, so a current summary refreshes just those (#1231). + return ActivityOutcomeProjectionContext::run( + $run, + $started, + fn (): WorkflowRunSummary => $this->projectRun( + $run->fresh(['instance', 'tasks', 'activityExecutions', 'failures']) ?? $run + ), + ); } public function pruneStaleProjectionRowsForRun( diff --git a/src/V2/Support/FailureSnapshots.php b/src/V2/Support/FailureSnapshots.php index 03271dc8b..4b1224244 100644 --- a/src/V2/Support/FailureSnapshots.php +++ b/src/V2/Support/FailureSnapshots.php @@ -37,7 +37,11 @@ public static function forRun(WorkflowRun $run): array } foreach ($run->historyEvents->sortBy('sequence') as $event) { - if (! $event instanceof WorkflowHistoryEvent) { + // Most events carry no failure; skip decoding them (#1231). + if (! $event instanceof WorkflowHistoryEvent || ! StoredHistoryPayload::mayContain( + $event, + '"failure_id"' + )) { continue; } diff --git a/src/V2/Support/HistoryEventIndex.php b/src/V2/Support/HistoryEventIndex.php new file mode 100644 index 000000000..33d9e2de0 --- /dev/null +++ b/src/V2/Support/HistoryEventIndex.php @@ -0,0 +1,71 @@ +, array>|null + */ + private static ?WeakMap $indexes = null; + + /** + * @template T + * @param Collection $events + * @param Closure(Collection): T $build + * @return T + */ + public static function remember(Collection $events, string $name, Closure $build): mixed + { + $fingerprint = self::fingerprint($events); + self::$indexes ??= new WeakMap(); + $indexes = self::$indexes[$events] ?? []; + $cached = $indexes[$name] ?? null; + + if ($cached !== null && $cached['fingerprint'] === $fingerprint) { + return $cached['index']; + } + + $index = $build($events); + $indexes[$name] = [ + 'fingerprint' => $fingerprint, + 'index' => $index, + ]; + self::$indexes[$events] = $indexes; + + return $index; + } + + /** + * @param Collection $events + * @return array{int, int|null, int|null} + */ + private static function fingerprint(Collection $events): array + { + $first = $events->first(); + $last = $events->last(); + + return [ + $events->count(), + is_object($first) ? spl_object_id($first) : null, + is_object($last) ? spl_object_id($last) : null, + ]; + } +} diff --git a/src/V2/Support/HistoryTimeline.php b/src/V2/Support/HistoryTimeline.php index 897e64ce5..b5d54ad83 100644 --- a/src/V2/Support/HistoryTimeline.php +++ b/src/V2/Support/HistoryTimeline.php @@ -65,14 +65,24 @@ public static function iterateFromHistory(WorkflowRun $run, ?string $activityExe foreach ($run->historyEvents->sortBy('sequence') as $event) { if ($activityExecutionId !== null) { - $payload = is_array($event->payload) ? $event->payload : []; - $activitySnapshot = is_array($payload['activity'] ?? null) ? $payload['activity'] : []; $task = $tasks->get((string) $event->workflow_task_id); - $eventActivityId = $payload['activity_execution_id'] ?? $activitySnapshot['id'] ?? null; $taskActivityId = $task instanceof WorkflowTask ? ($task->payload['activity_execution_id'] ?? null) : null; + // A sibling's stored payload that never names this activity + // cannot match it, so skip decoding it (#1231). + if ( + $taskActivityId !== $activityExecutionId + && ! StoredHistoryPayload::mayContain($event, $activityExecutionId) + ) { + continue; + } + + $payload = is_array($event->payload) ? $event->payload : []; + $activitySnapshot = is_array($payload['activity'] ?? null) ? $payload['activity'] : []; + $eventActivityId = $payload['activity_execution_id'] ?? $activitySnapshot['id'] ?? null; + // Older points can use current execution or task metadata as // a fallback. Refresh those points along with the new outcome. if ($eventActivityId !== $activityExecutionId && $taskActivityId !== $activityExecutionId) { diff --git a/src/V2/Support/ParallelChildGroup.php b/src/V2/Support/ParallelChildGroup.php index e062d5350..68038ca4d 100644 --- a/src/V2/Support/ParallelChildGroup.php +++ b/src/V2/Support/ParallelChildGroup.php @@ -19,6 +19,33 @@ final class ParallelChildGroup { + /** + * History events that carry a parallel member's group metadata. + */ + private const PARALLEL_MEMBER_EVENT_TYPES = [ + HistoryEventType::ActivityScheduled, + HistoryEventType::ActivityStarted, + HistoryEventType::ActivityRetryScheduled, + HistoryEventType::ActivityCompleted, + HistoryEventType::ActivityFailed, + HistoryEventType::ActivityCancelled, + HistoryEventType::ActivityTimedOut, + HistoryEventType::TimerScheduled, + HistoryEventType::TimerFired, + HistoryEventType::TimerCancelled, + HistoryEventType::ConditionWaitOpened, + HistoryEventType::ConditionWaitSatisfied, + HistoryEventType::ConditionWaitTimedOut, + HistoryEventType::SignalWaitOpened, + HistoryEventType::SignalApplied, + HistoryEventType::ChildWorkflowScheduled, + HistoryEventType::ChildRunStarted, + HistoryEventType::ChildRunCompleted, + HistoryEventType::ChildRunFailed, + HistoryEventType::ChildRunCancelled, + HistoryEventType::ChildRunTerminated, + ]; + /** * @return array{ * parallel_group_id: string, @@ -107,41 +134,44 @@ public static function metadataForSequence(WorkflowRun $run, int $sequence): ?ar */ public static function metadataPathForSequence(WorkflowRun $run, int $sequence): array { - /** @var WorkflowHistoryEvent|null $event */ - $event = $run->historyEvents->first( - static fn (WorkflowHistoryEvent $event): bool => in_array( - $event->event_type, - [ - HistoryEventType::ActivityScheduled, - HistoryEventType::ActivityStarted, - HistoryEventType::ActivityRetryScheduled, - HistoryEventType::ActivityCompleted, - HistoryEventType::ActivityFailed, - HistoryEventType::ActivityCancelled, - HistoryEventType::ActivityTimedOut, - HistoryEventType::TimerScheduled, - HistoryEventType::TimerFired, - HistoryEventType::TimerCancelled, - HistoryEventType::ConditionWaitOpened, - HistoryEventType::ConditionWaitSatisfied, - HistoryEventType::ConditionWaitTimedOut, - HistoryEventType::SignalWaitOpened, - HistoryEventType::SignalApplied, - HistoryEventType::ChildWorkflowScheduled, - HistoryEventType::ChildRunStarted, - HistoryEventType::ChildRunCompleted, - HistoryEventType::ChildRunFailed, - HistoryEventType::ChildRunCancelled, - HistoryEventType::ChildRunTerminated, - ], - true, - ) && ($event->payload['sequence'] ?? null) === $sequence - && ( - is_string($event->payload['parallel_group_id'] ?? null) - || is_array($event->payload['parallel_group_path'] ?? null) - ) + // Each fan-out member looks up its own path. Index the loaded history + // once instead of scanning it per member (#1231). + $eventsBySequence = HistoryEventIndex::remember( + $run->historyEvents, + 'parallel-metadata-event-by-sequence', + static function (Collection $events): array { + $index = []; + + foreach ($events as $event) { + if (! $event instanceof WorkflowHistoryEvent || ! in_array( + $event->event_type, + self::PARALLEL_MEMBER_EVENT_TYPES, + true, + )) { + continue; + } + + $eventSequence = $event->payload['sequence'] ?? null; + + if (! is_int($eventSequence) || array_key_exists($eventSequence, $index)) { + continue; + } + + if ( + is_string($event->payload['parallel_group_id'] ?? null) + || is_array($event->payload['parallel_group_path'] ?? null) + ) { + $index[$eventSequence] = $event; + } + } + + return $index; + }, ); + /** @var WorkflowHistoryEvent|null $event */ + $event = $eventsBySequence[$sequence] ?? null; + if (! $event instanceof WorkflowHistoryEvent || ! is_array($event->payload)) { return []; } @@ -1195,21 +1225,32 @@ private static function operationIdentityForMember( private static function waitKindForSequence(WorkflowRun $run, int $sequence): ?string { - foreach ($run->historyEvents as $event) { - if (($event->payload['sequence'] ?? null) !== $sequence) { - continue; - } - - if ($event->event_type === HistoryEventType::SignalWaitOpened) { - return 'signal'; - } + // Every member of a completed group is checked; index the first wait + // opening per sequence once per loaded history (#1203, #1231). + $kinds = HistoryEventIndex::remember( + $run->historyEvents, + 'parallel-wait-kind-by-sequence', + static function (Collection $events): array { + $index = []; + + foreach ($events as $event) { + $kind = match ($event->event_type) { + HistoryEventType::SignalWaitOpened => 'signal', + HistoryEventType::ConditionWaitOpened => 'condition', + default => null, + }; + $eventSequence = $kind === null ? null : ($event->payload['sequence'] ?? null); + + if (is_int($eventSequence) && ! array_key_exists($eventSequence, $index)) { + $index[$eventSequence] = $kind; + } + } - if ($event->event_type === HistoryEventType::ConditionWaitOpened) { - return 'condition'; - } - } + return $index; + }, + ); - return null; + return $kinds[$sequence] ?? null; } private static function waitCompletedSuccessfully(WorkflowRun $run, int $sequence, string $kind): bool diff --git a/src/V2/Support/RunLineageView.php b/src/V2/Support/RunLineageView.php index c3c56896b..c6499135a 100644 --- a/src/V2/Support/RunLineageView.php +++ b/src/V2/Support/RunLineageView.php @@ -5,6 +5,7 @@ namespace Workflow\V2\Support; use Carbon\CarbonInterface; +use Illuminate\Support\Collection; use Workflow\V2\Enums\HistoryEventType; use Workflow\V2\Enums\ParentClosePolicy; use Workflow\V2\Enums\RunStatus; @@ -677,19 +678,47 @@ private static function latestParentClosePolicyEventForChild( ): ?WorkflowHistoryEvent { $parentRun->loadMissing('historyEvents'); - /** @var WorkflowHistoryEvent|null $event */ - $event = $parentRun->historyEvents - ->filter(static fn (WorkflowHistoryEvent $event): bool => in_array( - $event->event_type, - [HistoryEventType::ParentClosePolicyApplied, HistoryEventType::ParentClosePolicyFailed], - true, - ) && self::stringValue( - is_array($event->payload) ? ($event->payload['child_run_id'] ?? null) : null - ) === $childRunId) - ->sortBy(static fn (WorkflowHistoryEvent $event): int => $event->sequence ?? PHP_INT_MAX) - ->last(); - - return $event instanceof WorkflowHistoryEvent ? $event : null; + // Lineage lists every child of a fan-out. Index the latest policy + // outcome per child once per loaded history (#1203, #1231): the + // highest history sequence wins, and a tie keeps the later event. + $latestByChild = HistoryEventIndex::remember( + $parentRun->historyEvents, + 'parent-close-policy-event-by-child-run', + static function (Collection $events): array { + $index = []; + + foreach ($events as $event) { + if (! $event instanceof WorkflowHistoryEvent || ! in_array( + $event->event_type, + [HistoryEventType::ParentClosePolicyApplied, HistoryEventType::ParentClosePolicyFailed], + true, + )) { + continue; + } + + $eventChildRunId = self::stringValue( + is_array($event->payload) ? ($event->payload['child_run_id'] ?? null) : null + ); + + if ($eventChildRunId === null) { + continue; + } + + $order = $event->sequence ?? PHP_INT_MAX; + + if (! isset($index[$eventChildRunId]) || $order >= $index[$eventChildRunId]['order']) { + $index[$eventChildRunId] = [ + 'order' => $order, + 'event' => $event, + ]; + } + } + + return $index; + }, + ); + + return $latestByChild[$childRunId]['event'] ?? null; } private static function normalizePolicy(mixed $value): ?string diff --git a/src/V2/Support/RunSummaryProjector.php b/src/V2/Support/RunSummaryProjector.php index 4be6fcd7e..7bf76b19f 100644 --- a/src/V2/Support/RunSummaryProjector.php +++ b/src/V2/Support/RunSummaryProjector.php @@ -158,6 +158,7 @@ public static function project(WorkflowRun $run): WorkflowRunSummary $scopeActivityTimeline = ! $run->status->isTerminal() && $activityResolution?->workflow_run_id === $run->id && in_array($activityResolution?->event_type, [ + HistoryEventType::ActivityStarted, HistoryEventType::ActivityCompleted, HistoryEventType::ActivityFailed, ], true) diff --git a/src/V2/Support/StoredHistoryPayload.php b/src/V2/Support/StoredHistoryPayload.php new file mode 100644 index 000000000..c3414e13b --- /dev/null +++ b/src/V2/Support/StoredHistoryPayload.php @@ -0,0 +1,37 @@ +getCasts()['payload'] ?? null) !== 'array' + || (class_exists(Json::class) && ! HistoryPayloadJson::usesDefaultDecoder()) + || preg_match('/^[A-Za-z0-9_"-]+$/', $text) !== 1) { + return true; + } + + $stored = $event->getAttributes()['payload'] ?? null; + + return ! is_string($stored) || str_contains($stored, $text); + } +} diff --git a/src/V2/Support/TaskDispatcher.php b/src/V2/Support/TaskDispatcher.php index f08a67cde..d052e773d 100644 --- a/src/V2/Support/TaskDispatcher.php +++ b/src/V2/Support/TaskDispatcher.php @@ -5,6 +5,7 @@ namespace Workflow\V2\Support; use Illuminate\Contracts\Bus\Dispatcher as BusDispatcher; +use Illuminate\Database\Connection; use Throwable; use Workflow\Queue\DatabaseTaskDelivery; use Workflow\Support\AfterCommit; @@ -22,6 +23,70 @@ final class TaskDispatcher { + /** + * Rows a coalescing scope writes per transaction. The scope commits its + * deferred publishes and summary refreshes in bounded batches instead of + * one autocommit transaction per row (#1231). + */ + private const COALESCED_BATCH_SIZE = 50; + + /** + * Publishes committed inside the active coalescing scope, in commit order. + * + * @var list|null + */ + private static ?array $deferredPublishes = null; + + /** + * Run-summary refreshes owed by tasks published inside the active + * coalescing scope, keyed by run id (#1231). + * + * @var array>|null + */ + private static ?array $deferredRunSummaries = null; + + private static bool $flushingPublishes = false; + + /** + * Publish the tasks committed inside $operation after it returns, and + * refresh each affected run's summary once instead of once per task. + * + * A workflow task that schedules a fan-out commits one task per member. + * Each publish used to rebuild the run's whole summary in its own + * transaction, so the step cost grew with the square of the fan-out width. + * The deferred refresh reads the same committed state the last per-task + * refresh read. Publishing still happens only after the owning commit. + * + * @template T + * @param callable(): T $operation + * @return T + */ + public static function coalescingRunSummaries(callable $operation): mixed + { + if (self::$deferredRunSummaries !== null) { + return $operation(); + } + + self::$deferredPublishes = []; + self::$deferredRunSummaries = []; + + try { + $result = $operation(); + } catch (Throwable $throwable) { + try { + self::flushCoalesced(); + } catch (Throwable) { + // The operation's own failure is the one to report. + } + + throw $throwable; + } + + self::flushCoalesced(); + + return $result; + } + public static function dispatch( WorkflowTask $task, bool $queueTimerWakeup = false, @@ -29,9 +94,10 @@ public static function dispatch( ): void { $connection = $task->getConnection(); if ($connection->transactionLevel() > 0) { + $taskId = (string) $task->id; AfterCommit::on( $connection, - static fn () => self::publish($task->id, $queueTimerWakeup, $boundedWorkflowTaskProjection) + static fn () => self::publishOrDefer($taskId, $queueTimerWakeup, $boundedWorkflowTaskProjection) ); return; @@ -62,6 +128,105 @@ public static function dispatchRepairedPollTask( self::publish($task->id, false, $boundedWorkflowTaskProjection); } + private static function publishOrDefer( + string $taskId, + bool $queueTimerWakeup, + bool $boundedWorkflowTaskProjection, + ): void { + if (self::$deferredPublishes !== null && ! WorkflowStub::faked()) { + self::$deferredPublishes[] = [ + 'task_id' => $taskId, + 'queue_timer_wakeup' => $queueTimerWakeup, + 'bounded' => $boundedWorkflowTaskProjection, + ]; + + return; + } + + self::publish($taskId, $queueTimerWakeup, $boundedWorkflowTaskProjection); + } + + private static function flushCoalesced(): void + { + $publishes = self::$deferredPublishes ?? []; + self::$deferredPublishes = null; + $failure = null; + self::$flushingPublishes = true; + + try { + foreach (array_chunk($publishes, self::COALESCED_BATCH_SIZE) as $batch) { + $failure = self::inBatchTransaction( + $batch[0]['task_id'], + array_map( + static fn (array $publish): \Closure => static fn () => self::publish( + $publish['task_id'], + $publish['queue_timer_wakeup'], + $publish['bounded'], + ), + $batch, + ), + ) ?? $failure; + } + } finally { + self::$flushingPublishes = false; + $summaries = self::$deferredRunSummaries ?? []; + self::$deferredRunSummaries = null; + } + + foreach (array_chunk($summaries, self::COALESCED_BATCH_SIZE, true) as $batch) { + $entries = array_values($batch); + $failure = self::inBatchTransaction( + $entries[0][0]['task_id'], + array_map( + static fn (string $runId, array $runEntries): \Closure => static fn () => self::refreshDeferredRunSummary( + $runId, + $runEntries, + ), + array_map('strval', array_keys($batch)), + $entries, + ), + ) ?? $failure; + } + + if ($failure !== null) { + throw $failure; + } + } + + /** + * Run each step in one batch transaction, isolating every step in its own + * savepoint so one failure keeps the others, as separate transactions did. + * Returns the first failure for the caller to report after the batch. + * + * @param list<\Closure(): void> $steps + */ + private static function inBatchTransaction(string $admissionTaskId, array $steps): ?Throwable + { + $failure = null; + $runSteps = static function (Connection $connection) use ($steps, &$failure): void { + foreach ($steps as $step) { + try { + $connection->transaction(static fn () => $step()); + } catch (Throwable $throwable) { + $failure ??= $throwable; + } + } + }; + + $connection = ConfiguredV2Models::query('task_model', WorkflowTask::class)->getModel() + ->getConnection(); + + if ($connection->getDriverName() === 'sqlite') { + // Take SQLite's writer once for the batch. The steps publish jobs, + // so the batch is never repeated after it starts. + SqliteTransaction::forTask($admissionTaskId, static fn (Connection $database) => $runSteps($database)); + } else { + $connection->transaction(static fn (Connection $database) => $runSteps($database), 1); + } + + return $failure; + } + private static function publish( string $taskId, bool $queueTimerWakeup = false, @@ -239,6 +404,50 @@ private static function markDispatchFailure( } private static function refreshRunSummary(WorkflowTask $task, bool $boundedWorkflowTaskProjection): void + { + // A publish inside a caller's transaction (atomic poll repair) keeps + // its refresh in that transaction. Scope batches defer theirs. + if ( + self::$deferredRunSummaries !== null + && ! WorkflowStub::faked() + && (self::$flushingPublishes || $task->getConnection()->transactionLevel() === 0) + ) { + self::$deferredRunSummaries[(string) $task->workflow_run_id][] = [ + 'task_id' => (string) $task->id, + 'bounded' => $boundedWorkflowTaskProjection, + ]; + + return; + } + + self::refreshRunSummaryNow($task, $boundedWorkflowTaskProjection); + } + + /** + * @param list $entries + */ + private static function refreshDeferredRunSummary(string $runId, array $entries): void + { + if (count($entries) === 1) { + /** @var WorkflowTask|null $task */ + $task = ConfiguredV2Models::query('task_model', WorkflowTask::class)->find($entries[0]['task_id']); + + if ($task instanceof WorkflowTask) { + self::refreshRunSummaryNow($task, $entries[0]['bounded']); + + return; + } + } + + /** @var WorkflowRun|null $run */ + $run = ConfiguredV2Models::query('run_model', WorkflowRun::class)->find($runId); + + if ($run instanceof WorkflowRun) { + self::historyProjectionRole()->projectRun($run); + } + } + + private static function refreshRunSummaryNow(WorkflowTask $task, bool $boundedWorkflowTaskProjection): void { /** @var WorkflowRun|null $run */ $run = ConfiguredV2Models::query('run_model', WorkflowRun::class)->find($task->workflow_run_id); diff --git a/src/V2/Support/WorkflowExecutor.php b/src/V2/Support/WorkflowExecutor.php index b532827bb..c2fd0dc55 100644 --- a/src/V2/Support/WorkflowExecutor.php +++ b/src/V2/Support/WorkflowExecutor.php @@ -5,6 +5,7 @@ namespace Workflow\V2\Support; use Carbon\CarbonInterface; +use Illuminate\Support\Collection; use Illuminate\Support\Str; use InvalidArgumentException; use LogicException; @@ -5825,21 +5826,7 @@ private function unserializePayloadWithRun(string $serialized, ?WorkflowRun $run private function activityCompletionEvent(WorkflowRun $run, int $sequence): ?WorkflowHistoryEvent { - /** @var WorkflowHistoryEvent|null $event */ - $event = $run->historyEvents->first( - static fn (WorkflowHistoryEvent $event): bool => in_array( - $event->event_type, - [ - HistoryEventType::ActivityCompleted, - HistoryEventType::ActivityFailed, - HistoryEventType::ActivityCancelled, - HistoryEventType::ActivityTimedOut, - ], - true, - ) && ($event->payload['sequence'] ?? null) === $sequence - // The run's own close recorded this; the workflow never saw it (#1117). - && ! ClosingRunOpenWork::settledAtClose($event) - ); + $event = $this->activityEventsBySequence($run)['completion'][$sequence] ?? null; ExecutionTracing::linkOutcome($run, $event); @@ -5848,21 +5835,63 @@ private function activityCompletionEvent(WorkflowRun $run, int $sequence): ?Work private function activityOpenEvent(WorkflowRun $run, int $sequence): ?WorkflowHistoryEvent { - /** @var WorkflowHistoryEvent|null $event */ - $event = $run->historyEvents->first( - static fn (WorkflowHistoryEvent $event): bool => in_array( - $event->event_type, - [ - HistoryEventType::ActivityScheduled, - HistoryEventType::ActivityStarted, - HistoryEventType::ActivityHeartbeatRecorded, - HistoryEventType::ActivityRetryScheduled, - ], - true, - ) && ($event->payload['sequence'] ?? null) === $sequence - ); + return $this->activityEventsBySequence($run)['open'][$sequence] ?? null; + } - return $event; + /** + * The first activity resolution and open event per durable sequence, in + * history order. Replaying a fan-out reads every member's events, so the + * loaded history is indexed once instead of scanned per member (#1231). + * + * @return array{ + * completion: array, + * open: array + * } + */ + private function activityEventsBySequence(WorkflowRun $run): array + { + return HistoryEventIndex::remember( + $run->historyEvents, + 'activity-events-by-sequence', + static function (Collection $events): array { + $index = [ + 'completion' => [], + 'open' => [], + ]; + + foreach ($events as $event) { + if (! $event instanceof WorkflowHistoryEvent) { + continue; + } + + $kind = match ($event->event_type) { + HistoryEventType::ActivityCompleted, + HistoryEventType::ActivityFailed, + HistoryEventType::ActivityCancelled, + HistoryEventType::ActivityTimedOut => 'completion', + HistoryEventType::ActivityScheduled, + HistoryEventType::ActivityStarted, + HistoryEventType::ActivityHeartbeatRecorded, + HistoryEventType::ActivityRetryScheduled => 'open', + default => null, + }; + $sequence = $kind === null ? null : ($event->payload['sequence'] ?? null); + + if (! is_int($sequence) || isset($index[$kind][$sequence])) { + continue; + } + + // The run's own close recorded this; the workflow never saw it (#1117). + if ($kind === 'completion' && ClosingRunOpenWork::settledAtClose($event)) { + continue; + } + + $index[$kind][$sequence] = $event; + } + + return $index; + }, + ); } private function activityHistoryEvent(WorkflowRun $run, int $sequence): ?WorkflowHistoryEvent diff --git a/src/V2/Support/WorkflowStepHistory.php b/src/V2/Support/WorkflowStepHistory.php index de8b10944..11f15486e 100644 --- a/src/V2/Support/WorkflowStepHistory.php +++ b/src/V2/Support/WorkflowStepHistory.php @@ -239,19 +239,7 @@ public static function conflictingEventTypesForSequence( ): array { $eventTypes = []; - foreach (self::historyEventsForRun($run)->sortBy('sequence') as $event) { - if (! $event instanceof WorkflowHistoryEvent) { - continue; - } - - if (! self::isWorkflowStepEvent($event)) { - continue; - } - - if (self::intValue($event->payload['sequence'] ?? null) !== $sequence) { - continue; - } - + foreach (self::stepEventsForSequence($run, $sequence) as $event) { if (self::eventMatchesShape($event, $expectedShape)) { continue; } @@ -317,16 +305,8 @@ private static function detailMismatchForSequence( return null; } - foreach (self::historyEventsForRun($run)->sortBy('sequence') as $event) { - if (! $event instanceof WorkflowHistoryEvent) { - continue; - } - - if (! self::isWorkflowStepEvent($event) || ! self::eventMatchesShape($event, $expectedShape)) { - continue; - } - - if (self::intValue($event->payload['sequence'] ?? null) !== $sequence) { + foreach (self::stepEventsForSequence($run, $sequence) as $event) { + if (! self::eventMatchesShape($event, $expectedShape)) { continue; } @@ -637,25 +617,48 @@ private static function workflowStepEventTypesForSequence(WorkflowRun $run, int { $eventTypes = []; - foreach (self::historyEventsForRun($run)->sortBy('sequence') as $event) { - if (! $event instanceof WorkflowHistoryEvent) { - continue; - } - - if (! self::isWorkflowStepEvent($event)) { - continue; - } - - if (self::intValue($event->payload['sequence'] ?? null) !== $sequence) { - continue; - } - + foreach (self::stepEventsForSequence($run, $sequence) as $event) { $eventTypes[] = $event->event_type->value; } return array_values(array_unique($eventTypes)); } + /** + * The run's workflow-step events recorded for one durable sequence, in + * history order. Replay validates every fan-out member, so the events are + * indexed by sequence once per loaded history instead of rescanned per + * member (#1231). + * + * @return list + */ + private static function stepEventsForSequence(WorkflowRun $run, int $sequence): array + { + $index = HistoryEventIndex::remember( + self::historyEventsForRun($run), + 'workflow-step-events-by-sequence', + static function (Collection $events): array { + $index = []; + + foreach ($events->sortBy('sequence') as $event) { + if (! $event instanceof WorkflowHistoryEvent || ! self::isWorkflowStepEvent($event)) { + continue; + } + + $eventSequence = self::intValue($event->payload['sequence'] ?? null); + + if ($eventSequence !== null) { + $index[$eventSequence][] = $event; + } + } + + return $index; + }, + ); + + return $index[$sequence] ?? []; + } + /** * @return Collection */ diff --git a/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php b/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php index 81fff917e..8e6e7c858 100644 --- a/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php +++ b/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php @@ -5,6 +5,7 @@ namespace Tests\Feature\V2; use DateTimeImmutable; +use Illuminate\Database\Eloquent\Collection as EloquentCollection; use Illuminate\Queue\WorkerOptions; use Illuminate\Support\Carbon; use Illuminate\Support\Facades\Artisan; @@ -15,6 +16,7 @@ use Illuminate\Support\Str; use Illuminate\Validation\ValidationException; use Mockery; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\Attributes\PreserveGlobalState; use PHPUnit\Framework\Attributes\RunInSeparateProcess; use Symfony\Component\Uid\Ulid; @@ -93,6 +95,7 @@ use Workflow\V2\Support\EmbeddedV2HistoryImport; use Workflow\V2\Support\HistoryExport; use Workflow\V2\Support\MemoPayload; +use Workflow\V2\Support\ParallelChildGroup; use Workflow\V2\Support\QueryStateReplayer; use Workflow\V2\Support\ReplayDiff; use Workflow\V2\Support\RunActivityView; @@ -3114,6 +3117,103 @@ public function testSelectionLoserCorpusIsSettledWhenTheRunCloses(): void } } + /** + * Replaying a fan-out looks up every member by sequence. Each lookup + * family indexes the loaded history once, so the recorded events are read + * a bounded number of times however many members the group has; a scan + * per member read the whole history again for each one (#1203, #1231). + * A live run on the current code records the evidence history unchanged. + * + * @return iterable + */ + public static function fanOutHistoryIndexCorpus(): iterable + { + yield 'activities' => [ + 'fan-out-activity-all-history-index', + WorkflowStepHistory::ACTIVITY, + 'activity_type', + ]; + yield 'children' => [ + 'fan-out-child-all-history-index', + WorkflowStepHistory::CHILD_WORKFLOW, + 'child_workflow_type', + ]; + } + + #[DataProvider('fanOutHistoryIndexCorpus')] + public function testFanOutCorpusReadsRecordedHistoryOncePerLookupFamily( + string $fixtureId, + string $shape, + string $detailField, + ): void { + $path = self::FIXTURE_DIR . "/{$fixtureId}.json"; + if (! is_file($path)) { + $this->markTestSkipped('The fan-out history index evidence is absent.'); + } + /** @var array $fixture */ + $fixture = json_decode((string) file_get_contents($path), true, flags: JSON_THROW_ON_ERROR); + $history = $fixture['history']; + $members = []; + foreach ($history as $event) { + if (in_array($event['event_type'], ['ActivityScheduled', 'ChildWorkflowScheduled'], true)) { + $members[$event['payload']['sequence']] = $event['payload'][$detailField]; + } + } + $this->assertCount(3, $members); + + $run = new WorkflowRun([ + 'id' => 'fan-out-corpus-' . $fixtureId, + ]); + $run->setRelation('historyEvents', new EloquentCollection(array_map( + static fn (array $event): FanOutPayloadReadCountingEvent => (new FanOutPayloadReadCountingEvent()) + ->forceFill([ + ...$event, + 'workflow_run_id' => $run->id, + ]), + $history, + ))); + FanOutPayloadReadCountingEvent::$foreignReads = 0; + $indexed = false; + + foreach ($members as $sequence => $type) { + // The first member's lookups may read the whole history to index it. + FanOutPayloadReadCountingEvent::$member = $indexed ? $sequence : null; + WorkflowStepHistory::assertCompatible($run, $sequence, $shape, [ + $detailField => $type, + ]); + $this->assertSame( + $sequence, + ParallelChildGroup::metadataPathForSequence($run, $sequence)[0]['parallel_group_base_sequence'] + + ParallelChildGroup::metadataPathForSequence($run, $sequence)[0]['parallel_group_index'], + ); + if ($shape === WorkflowStepHistory::CHILD_WORKFLOW) { + $this->assertSame( + 'ChildWorkflowScheduled', + ChildRunHistory::scheduledEventForSequence($run, $sequence)?->event_type->value, + ); + $this->assertSame( + 'ChildRunStarted', + ChildRunHistory::startedEventForSequence($run, $sequence)?->event_type->value, + ); + $this->assertSame( + 'ChildRunCompleted', + ChildRunHistory::resolutionEventForSequence($run, $sequence)?->event_type->value, + ); + } + $indexed = true; + } + FanOutPayloadReadCountingEvent::$member = null; + + // Once indexed, a member's lookups read only that member's events. + $this->assertSame( + 0, + FanOutPayloadReadCountingEvent::$foreignReads, + 'Fan-out replay lookups rescanned other members\' recorded events for each member.', + ); + + $this->assertLiveFanOutRecordsEvidenceHistory($fixture); + } + /** * @param array $fixture */ @@ -3162,6 +3262,94 @@ class_exists(VersionUsageQuery::class), $this->assertSame($history, array_slice($run->historyEvents()->get()->toArray(), 0, count($history))); } + /** + * @param array $fixture + */ + private function assertLiveFanOutRecordsEvidenceHistory(array $fixture): void + { + self::stopWorkers(); + // The evidence was recorded on the redis queue connection. + config([ + 'queue.default' => 'redis', + ]); + Queue::fake(); + $started = array_values(array_filter( + $fixture['history'], + static fn (array $event): bool => $event['event_type'] === 'WorkflowStarted', + ))[0]; + $workflow = WorkflowStub::make($fixture['workflow']['type'], $started['payload']['workflow_instance_id']); + $workflow->start(...$fixture['workflow']['arguments']); + $runId = $workflow->runId(); + + for ($step = 0; $step < 100; $step++) { + /** @var WorkflowTask|null $task */ + $task = WorkflowTask::query() + ->where('status', TaskStatus::Ready->value) + ->orderBy('created_at') + ->orderBy('id') + ->first(); + if ($task === null) { + break; + } + $job = match ($task->task_type) { + TaskType::Workflow => new RunWorkflowTask($task->id), + TaskType::Activity => new RunActivityTask($task->id), + TaskType::Timer => new RunTimerTask($task->id), + }; + $this->app->call([$job, 'handle']); + } + + $this->assertTrue($workflow->refresh()->completed()); + $this->assertSame($fixture['expected']['result'], $workflow->output()); + $recorded = WorkflowHistoryEvent::query() + ->where('workflow_run_id', $runId) + ->orderBy('sequence') + ->get() + ->map(static fn (WorkflowHistoryEvent $event): array => [ + 'sequence' => $event->sequence, + 'event_type' => $event->event_type->value, + // Evidence omits the claimed task and command snapshots. + 'payload' => array_diff_key($event->payload, [ + 'task' => true, + 'command' => true, + ]), + ]) + ->all(); + $evidence = array_map(static fn (array $event): array => [ + 'sequence' => $event['sequence'], + 'event_type' => $event['event_type'], + 'payload' => $event['payload'], + ], $fixture['history']); + + $this->assertSame(self::withoutRunIdentity($evidence), self::withoutRunIdentity($recorded)); + } + + /** + * Mask generated identifiers and timestamps; every other recorded value, + * key and event position must match. + */ + private static function withoutRunIdentity(mixed $value): mixed + { + if (is_array($value)) { + return array_map(self::withoutRunIdentity(...), $value); + } + if (! is_string($value)) { + return $value; + } + if (strlen($value) > 40 && preg_match('/^[A-Za-z0-9+\/]+={0,2}$/', $value) === 1) { + $decoded = base64_decode($value, true); + if ($decoded !== false && preg_match('/01[0-9A-Za-z]{24}/', $decoded) === 1) { + return 'payload-with-identity'; + } + } + + return preg_replace( + ['/01[0-9A-Za-z]{24}/', '/\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:?\d{2})?/'], + ['', '