diff --git a/.github/feature-test-timings.json b/.github/feature-test-timings.json index e876481ea..a1c2be173 100644 --- a/.github/feature-test-timings.json +++ b/.github/feature-test-timings.json @@ -84,6 +84,7 @@ "tests/Feature/V2/V2ChildTerminalOutcomeTest.php": 10.083904, "tests/Feature/V2/V2ChildWorkflowExternalOutputReplayTest.php": 0.475813, "tests/Feature/V2/V2ChildWorkflowNamespaceProjectionTest.php": 0.230284, + "tests/Feature/V2/V2CoalescedTaskPublishTest.php": 12.0, "tests/Feature/V2/V2CodFundingMachineTest.php": 671.423589, "tests/Feature/V2/V2CodMerchantSettlementTest.php": 171.978789, "tests/Feature/V2/V2CodProcessLossTest.php": 420.587435, @@ -121,6 +122,7 @@ "tests/Feature/V2/V2FailedRunRedrivePlanTest.php": 1.295295, "tests/Feature/V2/V2FailedRunRedriveTest.php": 310.561609, "tests/Feature/V2/V2FailureMessageStorageMigrationTest.php": 1.444116, + "tests/Feature/V2/V2FanOutWidthScalingTest.php": 420.0, "tests/Feature/V2/V2FinallyCleanupTest.php": 48.866892, "tests/Feature/V2/V2FinancialInspectionTest.php": 31.853246, "tests/Feature/V2/V2FinancialRetainedScopeTest.php": 38.727937, @@ -337,6 +339,7 @@ "tests/Feature/V2/V2ChildTerminalOutcomeTest.php": 6.158146, "tests/Feature/V2/V2ChildWorkflowExternalOutputReplayTest.php": 0.25019, "tests/Feature/V2/V2ChildWorkflowNamespaceProjectionTest.php": 0.330892, + "tests/Feature/V2/V2CoalescedTaskPublishTest.php": 10.0, "tests/Feature/V2/V2CodFundingMachineTest.php": 310.971588, "tests/Feature/V2/V2CodMerchantSettlementTest.php": 81.328249, "tests/Feature/V2/V2CodProcessLossTest.php": 217.0842, @@ -374,6 +377,7 @@ "tests/Feature/V2/V2FailedRunRedrivePlanTest.php": 0.9241, "tests/Feature/V2/V2FailedRunRedriveTest.php": 184.550012, "tests/Feature/V2/V2FailureMessageStorageMigrationTest.php": 1.397007, + "tests/Feature/V2/V2FanOutWidthScalingTest.php": 360.0, "tests/Feature/V2/V2FinallyCleanupTest.php": 34.060109, "tests/Feature/V2/V2FinancialInspectionTest.php": 17.846207, "tests/Feature/V2/V2FinancialRetainedScopeTest.php": 25.234874, diff --git a/.github/workflows/fan-out-scaling.yml b/.github/workflows/fan-out-scaling.yml new file mode 100644 index 000000000..2845abbd0 --- /dev/null +++ b/.github/workflows/fan-out-scaling.yml @@ -0,0 +1,126 @@ +name: Fan-out scaling + +# The literal documented fan-out limit (#1203, #1231): one all() of 1,000 +# activities and one of 1,000 child workflows with 10 KB arguments. Each +# workflow task must finish inside the default 60-second queue job +# timeout and cost roughly ten times the 100-member run's. The 30-versus-300 +# member guard runs in the ordinary feature shards. + +on: + pull_request: + branches: [main] + paths: + - 'src/V2/**' + - 'tests/Performance/FanOutWidthLiteralTest.php' + - 'tests/Support/FanOutWidthProbe.php' + - 'tests/Support/MemberPhaseProjectionRole.php' + - 'tests/Fixtures/V2/TestFanOutPayload*.php' + - '.github/workflows/fan-out-scaling.yml' + schedule: + - cron: '17 4 * * *' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: fan-out-scaling-${{ github.event_name }}-${{ github.event_name == 'pull_request' && github.ref || github.sha }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + literal-fan-out: + name: Literal 1,000-member fan-out (${{ matrix.kind }}, ${{ matrix.database }}) + runs-on: ubuntu-latest + timeout-minutes: 60 + # Starting 1,000 children takes 33-65 s on hosted MySQL runners, around + # the bound (#1238); report it without failing the build until then. + continue-on-error: ${{ matrix.kind == 'children' }} + strategy: + fail-fast: false + matrix: + database: [sqlite, mysql] + kind: [activities, children] + + services: + mysql: + image: mysql + env: + MYSQL_ROOT_PASSWORD: password + options: >- + --tmpfs /var/lib/mysql:rw,noexec,nosuid,size=2048m + ports: + - 3306 + + redis: + image: redis + ports: + - 6379 + + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + + - name: Cache Composer packages + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} + uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5 + with: + path: vendor + key: ${{ github.event_name }}-${{ runner.os }}-php-${{ hashFiles('**/composer.lock') }} + restore-keys: | + ${{ github.event_name }}-${{ runner.os }}-php- + + - name: Install dependencies + run: | + for attempt in 1 2 3; do + composer install --prefer-dist --no-progress && exit 0 + sleep $((attempt * 20)) + done + exit 1 + + - name: Resolve service endpoints and export test defaults + env: + DATABASE: ${{ matrix.database }} + MYSQL_PUBLISHED_PORT: ${{ job.services.mysql.ports['3306'] }} + REDIS_PUBLISHED_PORT: ${{ job.services.redis.ports['6379'] }} + run: | + { + echo "APP_KEY=base64:i3g6f+dV8FfsIkcxqd7gbiPn2oXk5r00sTmdD6V5utI=" + echo "REDIS_HOST=127.0.0.1" + echo "REDIS_PORT=${REDIS_PUBLISHED_PORT}" + echo "REDIS_PASSWORD=" + echo "QUEUE_CONNECTION=redis" + echo "QUEUE_FAILED_DRIVER=null" + echo "CACHE_DRIVER=array" + echo "CACHE_STORE=array" + if [ "$DATABASE" = mysql ]; then + echo "DB_CONNECTION=mysql" + echo "DB_HOST=127.0.0.1" + echo "DB_PORT=${MYSQL_PUBLISHED_PORT}" + echo "DB_DATABASE=testbench" + echo "DB_USERNAME=root" + echo "DB_PASSWORD=password" + else + echo "DB_CONNECTION=sqlite" + echo "DB_DATABASE=${RUNNER_TEMP}/fan-out-scaling.sqlite" + fi + } >> "$GITHUB_ENV" + + - name: Create the database + run: | + if [ "$DB_CONNECTION" = mysql ]; then + mysql -e 'CREATE DATABASE testbench' -h"$DB_HOST" -uroot -ppassword -P "$DB_PORT" + else + touch "$DB_DATABASE" + fi + + - name: Run the literal 1,000-member fan-out + run: >- + vendor/bin/phpunit --no-coverage --fail-on-skipped + --filter "testLiteralThousandMemberFanOut@${KIND}" + --log-junit build/test-results/fan-out-scaling.xml + tests/Performance/FanOutWidthLiteralTest.php + env: + FANOUT_LITERAL: '1' + KIND: ${{ matrix.kind }} + XDEBUG_MODE: off diff --git a/CHANGELOG.md b/CHANGELOG.md index 85e9be144..9181dc8c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -166,6 +166,25 @@ in a minor release: `DurableScope::reserveChild()`, original root admission The activity calls and their order are unchanged, so recorded commands are the same. Examples only. +- A wide `all()` no longer makes its workflow tasks grow with the square of the + fan-out width (#1231, #1203). A 300-member fan-out of 10 KB activities took + 134 s to schedule and 58–85 s to finish on 2.32.0, so Laravel's default + 60-second queue job timeout killed it on every retry. Per-member cost is now + roughly constant: + - A workflow task publishes the tasks it committed in bounded batches and + refreshes each affected run summary once, instead of rebuilding the run's + whole summary, in its own transaction, for every member it published. + - Replay, child-call, parallel-group, lineage and activity lookups index the + loaded history by sequence once, instead of rescanning the whole history + for every member. A child closure no longer rescans the parent's history + for every sibling. + - Activity starts refresh only their own timeline points, like outcomes, and + projections skip decoding payloads that cannot name the activity or a + failure. + + Recorded history and every projection row are unchanged. Each member's own + activity or child task still rebuilds its run summary once. + ## 2.32.0 - 2026-11-19 Compensation scopes remain **experimental** in 2.32.0, without a diff --git a/src/V2/Jobs/RunWorkflowTask.php b/src/V2/Jobs/RunWorkflowTask.php index 19ef65c46..bc8386356 100644 --- a/src/V2/Jobs/RunWorkflowTask.php +++ b/src/V2/Jobs/RunWorkflowTask.php @@ -28,6 +28,7 @@ use Workflow\V2\Support\TaskCompatibility; use Workflow\V2\Support\TaskDispatcher; use Workflow\V2\Support\TaskOutcomeCarrier; +use Workflow\V2\Support\TaskScopeMemo; use Workflow\V2\Support\TaskTraceTimings; use Workflow\V2\Support\TraceCarrier; use Workflow\V2\Support\WorkerCompatibilityFleet; @@ -55,6 +56,16 @@ public function __construct( } public function handle(WorkflowExecutor $executor): void + { + // A fan-out publishes one task per member after commit. Refresh each + // affected run summary once for the whole task, and repeat a member's + // immutable lookups at most once per task (#1203, #1231). + TaskScopeMemo::scope( + fn () => TaskDispatcher::coalescingRunSummaries(fn () => $this->handleTask($executor)), + ); + } + + private function handleTask(WorkflowExecutor $executor): void { // Apps sharing a database and queue names can receive each other's // jobs. Refuse a task outside this app's namespaces before any effect. diff --git a/src/V2/Support/ActivityReplayIdentity.php b/src/V2/Support/ActivityReplayIdentity.php index 107d213a0..3f80fc9a5 100644 --- a/src/V2/Support/ActivityReplayIdentity.php +++ b/src/V2/Support/ActivityReplayIdentity.php @@ -6,6 +6,7 @@ use Workflow\V2\Enums\HistoryEventType; use Workflow\V2\Exceptions\ActivityRetryPolicyMismatchException; +use Workflow\V2\Models\WorkflowHistoryEvent; use Workflow\V2\Models\WorkflowRun; final class ActivityReplayIdentity @@ -41,34 +42,21 @@ public static function assertCall( if (! $run->relationLoaded('historyEvents')) { $run->load('historyEvents'); } - foreach ($run->historyEvents as $event) { - if ($event->event_type !== HistoryEventType::ActivityScheduled) { - continue; - } - - $recordedSequence = $event->payload['sequence'] ?? null; - if (is_string($recordedSequence) && is_numeric($recordedSequence)) { - $recordedSequence = (int) $recordedSequence; - } - if ($recordedSequence !== $sequence - || ($event->payload['execution_mode'] ?? null) === LocalActivityRuntime::EXECUTION_MODE - || ($event->payload['activity']['execution_mode'] ?? null) === LocalActivityRuntime::EXECUTION_MODE) { - continue; - } - - $snapshot = $event->payload['activity']['retry_policy'] ?? $event->payload['retry_policy'] ?? null; - if ($snapshot === null) { - // Legacy absence is not an invitation to invent today's class defaults. - return; - } - if (! is_array($snapshot)) { - throw new ActivityRetryPolicyMismatchException($sequence, 'retry_policy'); - } - - self::assertPolicy($snapshot, $call->options, $sequence); + $event = self::scheduledEventForSequence($run, $sequence); + if (! $event instanceof WorkflowHistoryEvent) { + return; + } + $snapshot = $event->payload['activity']['retry_policy'] ?? $event->payload['retry_policy'] ?? null; + if ($snapshot === null) { + // Legacy absence is not an invitation to invent today's class defaults. return; } + if (! is_array($snapshot)) { + throw new ActivityRetryPolicyMismatchException($sequence, 'retry_policy'); + } + + self::assertPolicy($snapshot, $call->options, $sequence); } /** @@ -129,4 +117,41 @@ public static function assertPolicy(array $snapshot, ?ActivityOptions $options, } } } + + /** + * The first non-local ActivityScheduled event for a recorded sequence, in + * history order. Every member of a replayed fan-out is checked, so the + * loaded history is indexed as far as each lookup needs instead of + * scanned per member (#1231). + */ + private static function scheduledEventForSequence(WorkflowRun $run, int $sequence): ?WorkflowHistoryEvent + { + /** @var WorkflowHistoryEvent|null $event */ + $event = HistoryEventIndex::find( + $run->historyEvents, + 'activity-scheduled-event-by-sequence', + $sequence, + static function (mixed $event): array { + if ($event->event_type !== HistoryEventType::ActivityScheduled) { + return []; + } + + $recordedSequence = $event->payload['sequence'] ?? null; + if (is_string($recordedSequence) && is_numeric($recordedSequence)) { + $recordedSequence = (int) $recordedSequence; + } + if (! is_int($recordedSequence) + || ($event->payload['execution_mode'] ?? null) === LocalActivityRuntime::EXECUTION_MODE + || ($event->payload['activity']['execution_mode'] ?? null) === LocalActivityRuntime::EXECUTION_MODE) { + return []; + } + + return [ + $recordedSequence => $event, + ]; + }, + ); + + return $event; + } } diff --git a/src/V2/Support/ChildRunHistory.php b/src/V2/Support/ChildRunHistory.php index 04bf1bddf..3b9f9d0f5 100644 --- a/src/V2/Support/ChildRunHistory.php +++ b/src/V2/Support/ChildRunHistory.php @@ -5,10 +5,10 @@ namespace Workflow\V2\Support; use Carbon\CarbonInterface; +use Illuminate\Support\Collection; use RuntimeException; use Throwable; use Workflow\Serializers\Serializer; -use Workflow\Support\AfterCommit; use Workflow\V2\Enums\ChildCallStatus; use Workflow\V2\Enums\HistoryEventType; use Workflow\V2\Enums\ParentClosePolicy; @@ -293,52 +293,48 @@ public static function resolutionEventTypes(): array public static function scheduledEventForSequence(WorkflowRun $run, int $sequence): ?WorkflowHistoryEvent { /** @var WorkflowHistoryEvent|null $event */ - $event = $run->historyEvents->first( - static fn (WorkflowHistoryEvent $event): bool => $event->event_type === HistoryEventType::ChildWorkflowScheduled - && ($event->payload['sequence'] ?? null) === $sequence - ); + $event = self::firstChildEventForSequence($run, 'scheduled', $sequence); return $event; } public static function startedEventForSequence(WorkflowRun $run, int $sequence): ?WorkflowHistoryEvent { - /** @var WorkflowHistoryEvent|null $event */ - $event = $run->historyEvents - ->filter( - static fn (WorkflowHistoryEvent $event): bool => $event->event_type === HistoryEventType::ChildRunStarted - && ($event->payload['sequence'] ?? null) === $sequence - ) - ->sortByDesc('sequence') - ->first(); - - return $event; + return self::startedEventsBySequence($run)[$sequence] ?? null; } public static function resolutionEventForSequence(WorkflowRun $run, int $sequence): ?WorkflowHistoryEvent { /** @var WorkflowHistoryEvent|null $event */ - $event = $run->historyEvents->first( - static fn (WorkflowHistoryEvent $event): bool => in_array( - $event->event_type, - self::resolutionEventTypes(), - true - ) - && ($event->payload['sequence'] ?? null) === $sequence - ); + $event = self::firstChildEventForSequence($run, 'resolution', $sequence); return $event; } public static function latestLinkForSequence(WorkflowRun $run, int $sequence): ?WorkflowLink { - /** @var WorkflowLink|null $link */ - $link = $run->childLinks - ->filter( - static fn (WorkflowLink $link): bool => $link->link_type === 'child_workflow' - && $link->sequence === $sequence - ) - ->sort(static function (WorkflowLink $left, WorkflowLink $right): int { + $linksBySequence = HistoryEventIndex::remember( + $run->childLinks, + 'child-workflow-links-by-sequence', + static function (Collection $links): array { + $index = []; + + foreach ($links as $link) { + if ($link instanceof WorkflowLink && $link->link_type === 'child_workflow' && is_int( + $link->sequence + )) { + $index[$link->sequence][] = $link; + } + } + + return $index; + }, + ); + + $candidates = $linksBySequence[$sequence] ?? []; + + if (count($candidates) > 1) { + usort($candidates, static function (WorkflowLink $left, WorkflowLink $right): int { $leftRunNumber = $left->childRun?->run_number ?? 0; $rightRunNumber = $right->childRun?->run_number ?? 0; @@ -354,10 +350,10 @@ public static function latestLinkForSequence(WorkflowRun $run, int $sequence): ? } return $right->id <=> $left->id; - }) - ->first(); + }); + } - return $link; + return $candidates[0] ?? null; } /** @@ -793,24 +789,20 @@ public static function terminalEventForRun(?WorkflowRun $childRun): ?WorkflowHis private static function afterAuthoritativeCommit(callable $projection): void { - $callback = static function () use ($projection): void { - try { - $projection(); - } catch (Throwable $throwable) { - report($throwable); - } - }; - if (ConfiguredV2Models::connection()->transactionLevel() > 0) { // The projection writes one child-call row after commit, outside // the completion's meter: charge it now, before any write (#1116). HistoryTransactionMeter::current()?->chargeCompletionRows(1); - AfterCommit::on(ConfiguredV2Models::connection(), $callback); + TaskDispatcher::afterCommitWrite(ConfiguredV2Models::connection(), static fn () => $projection()); return; } - $callback(); + try { + $projection(); + } catch (Throwable $throwable) { + report($throwable); + } } /** @@ -1132,6 +1124,76 @@ private static function followContinuedRun(?WorkflowRun $childRun): ?WorkflowRun return $childRun; } + /** + * The first scheduled or resolution event for one child-call sequence in + * history order. A child projection looks up every fan-out member, so the + * loaded history is indexed as far as each lookup needs instead of + * scanned per member (#1203, #1231). + * + * @param 'scheduled'|'resolution' $kind + */ + private static function firstChildEventForSequence(WorkflowRun $run, string $kind, int $sequence): mixed + { + $types = $kind === 'scheduled' + ? [HistoryEventType::ChildWorkflowScheduled] + : self::resolutionEventTypes(); + + return HistoryEventIndex::find( + $run->historyEvents, + "child-run-{$kind}-event-by-sequence", + $sequence, + static function (mixed $event) use ($types): array { + if (! $event instanceof WorkflowHistoryEvent || ! in_array($event->event_type, $types, true)) { + return []; + } + + $eventSequence = $event->payload['sequence'] ?? null; + + return is_int($eventSequence) ? [ + $eventSequence => $event, + ] : []; + }, + ); + } + + /** + * The started event with the highest history sequence per child-call + * sequence, indexed once per loaded history (#1203, #1231). + * + * @return array + */ + private static function startedEventsBySequence(WorkflowRun $run): array + { + return HistoryEventIndex::remember( + $run->historyEvents, + 'child-run-started-event-by-sequence', + static function (Collection $events): array { + $index = []; + + foreach ($events as $event) { + if (! $event instanceof WorkflowHistoryEvent + || $event->event_type !== HistoryEventType::ChildRunStarted) { + continue; + } + + $sequence = $event->payload['sequence'] ?? null; + + if (! is_int($sequence)) { + continue; + } + + $existing = $index[$sequence] ?? null; + + if ($existing === null || $event->sequence > $existing->sequence) { + $index[$sequence] = $event; + } + } + + return $index; + }, + ); + } + private static function stringValue(mixed $value): ?string { return is_string($value) && $value !== '' diff --git a/src/V2/Support/ChildWorkflowNamespaceProjection.php b/src/V2/Support/ChildWorkflowNamespaceProjection.php index 8999f9cf3..ff4fd8df1 100644 --- a/src/V2/Support/ChildWorkflowNamespaceProjection.php +++ b/src/V2/Support/ChildWorkflowNamespaceProjection.php @@ -118,11 +118,22 @@ private function namespaceForParent(?string $parentInstanceId, ?string $parentRu $runModel = ConfiguredV2Models::resolve('run_model', WorkflowRun::class); if ($parentRunId !== null) { - $namespace = $runModel::query() - ->whereKey($parentRunId) - ->value('namespace'); - - if (is_string($namespace) && $namespace !== '') { + // Every child of a fan-out binds to the same parent. On PostgreSQL + // each read of the parent row, which the open task updates once + // per child, checks every version that task wrote (#1203). + $namespace = TaskScopeMemo::remember( + 'child-namespace-parent-run:' . $parentRunId, + static function () use ($runModel, $parentRunId): ?string { + $namespace = $runModel::query() + ->whereKey($parentRunId) + ->value('namespace'); + + // An empty string remembers "no run namespace" too. + return is_string($namespace) ? $namespace : ''; + }, + ); + + if ($namespace !== '') { return $namespace; } } diff --git a/src/V2/Support/DefaultHistoryProjectionRole.php b/src/V2/Support/DefaultHistoryProjectionRole.php index 5e4e9ab30..a55a1f8f5 100644 --- a/src/V2/Support/DefaultHistoryProjectionRole.php +++ b/src/V2/Support/DefaultHistoryProjectionRole.php @@ -55,7 +55,7 @@ public function recordActivityStarted( $parallelMetadataPath = ParallelChildGroup::metadataPathForSequence($run, (int) $execution->sequence); $parallelMetadata = ParallelChildGroup::payloadForPath($parallelMetadataPath); - WorkflowHistoryEvent::record($run, HistoryEventType::ActivityStarted, array_merge([ + $started = WorkflowHistoryEvent::record($run, HistoryEventType::ActivityStarted, array_merge([ 'activity_execution_id' => $execution->id, 'activity_attempt_id' => $attempt->id, 'activity_class' => $execution->activity_class, @@ -74,7 +74,15 @@ public function recordActivityStarted( (int) $attempt->attempt_number, ); - return $this->projectRun($run->fresh(['instance', 'tasks', 'activityExecutions', 'failures']) ?? $run); + // Like an outcome, a start changes only its own activity's timeline + // points, so a current summary refreshes just those (#1231). + return ActivityOutcomeProjectionContext::run( + $run, + $started, + fn (): WorkflowRunSummary => $this->projectRun( + $run->fresh(['instance', 'tasks', 'activityExecutions', 'failures']) ?? $run + ), + ); } public function pruneStaleProjectionRowsForRun( diff --git a/src/V2/Support/FailureSnapshots.php b/src/V2/Support/FailureSnapshots.php index 03271dc8b..4b1224244 100644 --- a/src/V2/Support/FailureSnapshots.php +++ b/src/V2/Support/FailureSnapshots.php @@ -37,7 +37,11 @@ public static function forRun(WorkflowRun $run): array } foreach ($run->historyEvents->sortBy('sequence') as $event) { - if (! $event instanceof WorkflowHistoryEvent) { + // Most events carry no failure; skip decoding them (#1231). + if (! $event instanceof WorkflowHistoryEvent || ! StoredHistoryPayload::mayContain( + $event, + '"failure_id"' + )) { continue; } diff --git a/src/V2/Support/HistoryEventIndex.php b/src/V2/Support/HistoryEventIndex.php new file mode 100644 index 000000000..48cadda5e --- /dev/null +++ b/src/V2/Support/HistoryEventIndex.php @@ -0,0 +1,115 @@ +, array>|null + */ + private static ?WeakMap $indexes = null; + + /** + * @template T + * @param Collection $events + * @param Closure(Collection): T $build + * @return T + */ + public static function remember(Collection $events, string $name, Closure $build): mixed + { + $fingerprint = self::fingerprint($events); + self::$indexes ??= new WeakMap(); + $indexes = self::$indexes[$events] ?? []; + $cached = $indexes[$name] ?? null; + + if ($cached !== null && $cached['fingerprint'] === $fingerprint) { + return $cached['index']; + } + + $index = $build($events); + $indexes[$name] = [ + 'fingerprint' => $fingerprint, + 'index' => $index, + ]; + self::$indexes[$events] = $indexes; + + return $index; + } + + /** + * Find the first entry for $key in history order, indexing the events + * only as far as needed. A single lookup stops at its match like a scan; + * every lookup together reads each event at most once. + * + * @param Collection $events + * @param Closure(mixed): array $entriesFor the keyed entries one event contributes + */ + public static function find(Collection $events, string $name, int|string $key, Closure $entriesFor): mixed + { + $fingerprint = self::fingerprint($events); + self::$indexes ??= new WeakMap(); + $indexes = self::$indexes[$events] ?? []; + $state = $indexes[$name] ?? null; + + if ($state === null || $state['fingerprint'] !== $fingerprint) { + $state = [ + 'fingerprint' => $fingerprint, + 'index' => [ + 'entries' => [], + 'items' => array_values($events->all()), + 'position' => 0, + ], + ]; + } + + $index = $state['index']; + + while (! array_key_exists($key, $index['entries']) && $index['position'] < count($index['items'])) { + foreach ($entriesFor($index['items'][$index['position']]) as $entryKey => $value) { + if (! array_key_exists($entryKey, $index['entries'])) { + $index['entries'][$entryKey] = $value; + } + } + ++$index['position']; + } + + $state['index'] = $index; + $indexes[$name] = $state; + self::$indexes[$events] = $indexes; + + return $index['entries'][$key] ?? null; + } + + /** + * @param Collection $events + * @return array{int, int|null, int|null} + */ + private static function fingerprint(Collection $events): array + { + $first = $events->first(); + $last = $events->last(); + + return [ + $events->count(), + is_object($first) ? spl_object_id($first) : null, + is_object($last) ? spl_object_id($last) : null, + ]; + } +} diff --git a/src/V2/Support/HistoryTimeline.php b/src/V2/Support/HistoryTimeline.php index 897e64ce5..b5d54ad83 100644 --- a/src/V2/Support/HistoryTimeline.php +++ b/src/V2/Support/HistoryTimeline.php @@ -65,14 +65,24 @@ public static function iterateFromHistory(WorkflowRun $run, ?string $activityExe foreach ($run->historyEvents->sortBy('sequence') as $event) { if ($activityExecutionId !== null) { - $payload = is_array($event->payload) ? $event->payload : []; - $activitySnapshot = is_array($payload['activity'] ?? null) ? $payload['activity'] : []; $task = $tasks->get((string) $event->workflow_task_id); - $eventActivityId = $payload['activity_execution_id'] ?? $activitySnapshot['id'] ?? null; $taskActivityId = $task instanceof WorkflowTask ? ($task->payload['activity_execution_id'] ?? null) : null; + // A sibling's stored payload that never names this activity + // cannot match it, so skip decoding it (#1231). + if ( + $taskActivityId !== $activityExecutionId + && ! StoredHistoryPayload::mayContain($event, $activityExecutionId) + ) { + continue; + } + + $payload = is_array($event->payload) ? $event->payload : []; + $activitySnapshot = is_array($payload['activity'] ?? null) ? $payload['activity'] : []; + $eventActivityId = $payload['activity_execution_id'] ?? $activitySnapshot['id'] ?? null; + // Older points can use current execution or task metadata as // a fallback. Refresh those points along with the new outcome. if ($eventActivityId !== $activityExecutionId && $taskActivityId !== $activityExecutionId) { diff --git a/src/V2/Support/ParallelChildGroup.php b/src/V2/Support/ParallelChildGroup.php index e062d5350..5f49d1dc8 100644 --- a/src/V2/Support/ParallelChildGroup.php +++ b/src/V2/Support/ParallelChildGroup.php @@ -19,6 +19,33 @@ final class ParallelChildGroup { + /** + * History events that carry a parallel member's group metadata. + */ + private const PARALLEL_MEMBER_EVENT_TYPES = [ + HistoryEventType::ActivityScheduled, + HistoryEventType::ActivityStarted, + HistoryEventType::ActivityRetryScheduled, + HistoryEventType::ActivityCompleted, + HistoryEventType::ActivityFailed, + HistoryEventType::ActivityCancelled, + HistoryEventType::ActivityTimedOut, + HistoryEventType::TimerScheduled, + HistoryEventType::TimerFired, + HistoryEventType::TimerCancelled, + HistoryEventType::ConditionWaitOpened, + HistoryEventType::ConditionWaitSatisfied, + HistoryEventType::ConditionWaitTimedOut, + HistoryEventType::SignalWaitOpened, + HistoryEventType::SignalApplied, + HistoryEventType::ChildWorkflowScheduled, + HistoryEventType::ChildRunStarted, + HistoryEventType::ChildRunCompleted, + HistoryEventType::ChildRunFailed, + HistoryEventType::ChildRunCancelled, + HistoryEventType::ChildRunTerminated, + ]; + /** * @return array{ * parallel_group_id: string, @@ -107,39 +134,31 @@ public static function metadataForSequence(WorkflowRun $run, int $sequence): ?ar */ public static function metadataPathForSequence(WorkflowRun $run, int $sequence): array { + // Each fan-out member looks up its own path. Index the loaded history + // as far as each lookup needs instead of scanning it per member (#1231). /** @var WorkflowHistoryEvent|null $event */ - $event = $run->historyEvents->first( - static fn (WorkflowHistoryEvent $event): bool => in_array( - $event->event_type, - [ - HistoryEventType::ActivityScheduled, - HistoryEventType::ActivityStarted, - HistoryEventType::ActivityRetryScheduled, - HistoryEventType::ActivityCompleted, - HistoryEventType::ActivityFailed, - HistoryEventType::ActivityCancelled, - HistoryEventType::ActivityTimedOut, - HistoryEventType::TimerScheduled, - HistoryEventType::TimerFired, - HistoryEventType::TimerCancelled, - HistoryEventType::ConditionWaitOpened, - HistoryEventType::ConditionWaitSatisfied, - HistoryEventType::ConditionWaitTimedOut, - HistoryEventType::SignalWaitOpened, - HistoryEventType::SignalApplied, - HistoryEventType::ChildWorkflowScheduled, - HistoryEventType::ChildRunStarted, - HistoryEventType::ChildRunCompleted, - HistoryEventType::ChildRunFailed, - HistoryEventType::ChildRunCancelled, - HistoryEventType::ChildRunTerminated, - ], - true, - ) && ($event->payload['sequence'] ?? null) === $sequence - && ( + $event = HistoryEventIndex::find( + $run->historyEvents, + 'parallel-metadata-event-by-sequence', + $sequence, + static function (mixed $event): array { + if (! $event instanceof WorkflowHistoryEvent || ! in_array( + $event->event_type, + self::PARALLEL_MEMBER_EVENT_TYPES, + true, + )) { + return []; + } + + $eventSequence = $event->payload['sequence'] ?? null; + + return is_int($eventSequence) && ( is_string($event->payload['parallel_group_id'] ?? null) || is_array($event->payload['parallel_group_path'] ?? null) - ) + ) ? [ + $eventSequence => $event, + ] : []; + }, ); if (! $event instanceof WorkflowHistoryEvent || ! is_array($event->payload)) { @@ -1195,21 +1214,25 @@ private static function operationIdentityForMember( private static function waitKindForSequence(WorkflowRun $run, int $sequence): ?string { - foreach ($run->historyEvents as $event) { - if (($event->payload['sequence'] ?? null) !== $sequence) { - continue; - } - - if ($event->event_type === HistoryEventType::SignalWaitOpened) { - return 'signal'; - } - - if ($event->event_type === HistoryEventType::ConditionWaitOpened) { - return 'condition'; - } - } - - return null; + // Every member of a completed group is checked; index the first wait + // opening per sequence as far as each lookup needs (#1203, #1231). + return HistoryEventIndex::find( + $run->historyEvents, + 'parallel-wait-kind-by-sequence', + $sequence, + static function (mixed $event): array { + $kind = match ($event->event_type) { + HistoryEventType::SignalWaitOpened => 'signal', + HistoryEventType::ConditionWaitOpened => 'condition', + default => null, + }; + $eventSequence = $kind === null ? null : ($event->payload['sequence'] ?? null); + + return is_int($eventSequence) ? [ + $eventSequence => $kind, + ] : []; + }, + ); } private static function waitCompletedSuccessfully(WorkflowRun $run, int $sequence, string $kind): bool diff --git a/src/V2/Support/RunLineageView.php b/src/V2/Support/RunLineageView.php index c3c56896b..c6499135a 100644 --- a/src/V2/Support/RunLineageView.php +++ b/src/V2/Support/RunLineageView.php @@ -5,6 +5,7 @@ namespace Workflow\V2\Support; use Carbon\CarbonInterface; +use Illuminate\Support\Collection; use Workflow\V2\Enums\HistoryEventType; use Workflow\V2\Enums\ParentClosePolicy; use Workflow\V2\Enums\RunStatus; @@ -677,19 +678,47 @@ private static function latestParentClosePolicyEventForChild( ): ?WorkflowHistoryEvent { $parentRun->loadMissing('historyEvents'); - /** @var WorkflowHistoryEvent|null $event */ - $event = $parentRun->historyEvents - ->filter(static fn (WorkflowHistoryEvent $event): bool => in_array( - $event->event_type, - [HistoryEventType::ParentClosePolicyApplied, HistoryEventType::ParentClosePolicyFailed], - true, - ) && self::stringValue( - is_array($event->payload) ? ($event->payload['child_run_id'] ?? null) : null - ) === $childRunId) - ->sortBy(static fn (WorkflowHistoryEvent $event): int => $event->sequence ?? PHP_INT_MAX) - ->last(); - - return $event instanceof WorkflowHistoryEvent ? $event : null; + // Lineage lists every child of a fan-out. Index the latest policy + // outcome per child once per loaded history (#1203, #1231): the + // highest history sequence wins, and a tie keeps the later event. + $latestByChild = HistoryEventIndex::remember( + $parentRun->historyEvents, + 'parent-close-policy-event-by-child-run', + static function (Collection $events): array { + $index = []; + + foreach ($events as $event) { + if (! $event instanceof WorkflowHistoryEvent || ! in_array( + $event->event_type, + [HistoryEventType::ParentClosePolicyApplied, HistoryEventType::ParentClosePolicyFailed], + true, + )) { + continue; + } + + $eventChildRunId = self::stringValue( + is_array($event->payload) ? ($event->payload['child_run_id'] ?? null) : null + ); + + if ($eventChildRunId === null) { + continue; + } + + $order = $event->sequence ?? PHP_INT_MAX; + + if (! isset($index[$eventChildRunId]) || $order >= $index[$eventChildRunId]['order']) { + $index[$eventChildRunId] = [ + 'order' => $order, + 'event' => $event, + ]; + } + } + + return $index; + }, + ); + + return $latestByChild[$childRunId]['event'] ?? null; } private static function normalizePolicy(mixed $value): ?string diff --git a/src/V2/Support/RunSummaryProjector.php b/src/V2/Support/RunSummaryProjector.php index 8e71ae833..9bff88a11 100644 --- a/src/V2/Support/RunSummaryProjector.php +++ b/src/V2/Support/RunSummaryProjector.php @@ -158,6 +158,7 @@ public static function project(WorkflowRun $run): WorkflowRunSummary $scopeActivityTimeline = ! $run->status->isTerminal() && $activityResolution?->workflow_run_id === $run->id && in_array($activityResolution?->event_type, [ + HistoryEventType::ActivityStarted, HistoryEventType::ActivityCompleted, HistoryEventType::ActivityFailed, ], true) diff --git a/src/V2/Support/StoredHistoryPayload.php b/src/V2/Support/StoredHistoryPayload.php new file mode 100644 index 000000000..c3414e13b --- /dev/null +++ b/src/V2/Support/StoredHistoryPayload.php @@ -0,0 +1,37 @@ +getCasts()['payload'] ?? null) !== 'array' + || (class_exists(Json::class) && ! HistoryPayloadJson::usesDefaultDecoder()) + || preg_match('/^[A-Za-z0-9_"-]+$/', $text) !== 1) { + return true; + } + + $stored = $event->getAttributes()['payload'] ?? null; + + return ! is_string($stored) || str_contains($stored, $text); + } +} diff --git a/src/V2/Support/StructuralLimits.php b/src/V2/Support/StructuralLimits.php index f4ce36fe3..34dd08926 100644 --- a/src/V2/Support/StructuralLimits.php +++ b/src/V2/Support/StructuralLimits.php @@ -324,7 +324,14 @@ public static function guardPendingActivities(WorkflowRun $run): void return; } - self::guardPendingActivityCount(self::pendingActivityCount($run)); + $count = self::pendingCount( + 'activities', + $run, + true, + self::pendingActivityLimit(), + static fn (): int => self::pendingActivityCount($run), + ); + self::guardPendingActivityCount($count); } /** @@ -353,7 +360,14 @@ public static function guardPendingChildren(WorkflowRun $run): void return; } - self::guardPendingChildCount(self::pendingChildCount($run)); + $count = self::pendingCount( + 'children', + $run, + true, + self::pendingChildLimit(), + static fn (): int => self::pendingChildCount($run), + ); + self::guardPendingChildCount($count); } /** @@ -559,10 +573,13 @@ public static function checkApproaching(StructuralLimitKind $kind, int $current) */ public static function warnApproachingPendingActivities(WorkflowRun $run): ?array { - $count = ActivityExecution::query() - ->where('workflow_run_id', $run->id) - ->whereIn('status', [ActivityStatus::Pending->value, ActivityStatus::Running->value]) - ->count(); + $count = self::pendingCount( + 'activities', + $run, + false, + self::warningThreshold(StructuralLimitKind::PendingActivityCount), + static fn (): int => self::pendingActivityCount($run), + ); return self::checkApproaching(StructuralLimitKind::PendingActivityCount, $count); } @@ -572,18 +589,13 @@ public static function warnApproachingPendingActivities(WorkflowRun $run): ?arra */ public static function warnApproachingPendingChildren(WorkflowRun $run): ?array { - $count = WorkflowLink::query() - ->where('parent_workflow_run_id', $run->id) - ->where('link_type', 'child_workflow') - ->whereHas('childRun', static function ($query) { - $query->whereNotIn('status', [ - RunStatus::Completed->value, - RunStatus::Failed->value, - RunStatus::Cancelled->value, - RunStatus::Terminated->value, - ]); - }) - ->count(); + $count = self::pendingCount( + 'children', + $run, + false, + self::warningThreshold(StructuralLimitKind::PendingChildCount), + static fn (): int => self::pendingChildCount($run), + ); return self::checkApproaching(StructuralLimitKind::PendingChildCount, $count); } @@ -748,6 +760,63 @@ public static function snapshot(): array ]; } + /** + * A pending count that decides exactly like the stored count (#1203, #1231). + * + * A workflow task schedules a fan-out's members one at a time while it + * holds the run lock, checking the pending count before each. Counting + * the stored rows every time made the task's cost grow with the square of + * the width. Within one task, the first check counts the rows, and each + * later guard check adds the one member scheduled since; members that + * close meanwhile only lower the stored count, so the running value is an + * upper bound. Below $decisionAt the bound decides like the stored count; + * at or above it the rows are counted again and that count is returned. + * + * @param 'activities'|'children' $kind + * @param \Closure(): int $countRows + */ + private static function pendingCount( + string $kind, + WorkflowRun $run, + bool $advance, + int $decisionAt, + \Closure $countRows, + ): int { + $key = "structural-limits-pending-{$kind}:{$run->id}"; + $remembered = TaskScopeMemo::get($key); + + if (is_int($remembered)) { + $bound = $remembered + ($advance ? 1 : 0); + + if ($bound < $decisionAt) { + TaskScopeMemo::put($key, $bound); + + return $bound; + } + } + + $count = $countRows(); + TaskScopeMemo::put($key, $count); + + return $count; + } + + /** + * The count at which checkApproaching() starts warning, or PHP_INT_MAX + * when it never warns for $kind. + */ + private static function warningThreshold(StructuralLimitKind $kind): int + { + $thresholdPercent = self::warningThresholdPercent(); + $limit = self::limitForKind($kind); + + if ($thresholdPercent <= 0 || $limit <= 0) { + return PHP_INT_MAX; + } + + return (int) floor($limit * $thresholdPercent / 100); + } + // ── Internal ────��─────────────────────────────────────────────── private static function cappedLimit(int $configured, int $fixedCap): int diff --git a/src/V2/Support/TaskDispatcher.php b/src/V2/Support/TaskDispatcher.php index f08a67cde..2ade4846f 100644 --- a/src/V2/Support/TaskDispatcher.php +++ b/src/V2/Support/TaskDispatcher.php @@ -5,6 +5,7 @@ namespace Workflow\V2\Support; use Illuminate\Contracts\Bus\Dispatcher as BusDispatcher; +use Illuminate\Database\Connection; use Throwable; use Workflow\Queue\DatabaseTaskDelivery; use Workflow\Support\AfterCommit; @@ -22,6 +23,71 @@ final class TaskDispatcher { + /** + * Rows a coalescing scope writes per transaction. The scope commits its + * deferred publishes and summary refreshes in bounded batches instead of + * one autocommit transaction per row (#1231). + */ + private const COALESCED_BATCH_SIZE = 50; + + /** + * Publishes and post-commit writes registered inside the active coalescing + * scope, in commit order. + * + * @var list|null + */ + private static ?array $deferredPublishes = null; + + /** + * Run-summary refreshes owed by tasks published inside the active + * coalescing scope, keyed by run id (#1231). + * + * @var array>|null + */ + private static ?array $deferredRunSummaries = null; + + private static bool $flushingPublishes = false; + + /** + * Publish the tasks committed inside $operation after it returns, and + * refresh each affected run's summary once instead of once per task. + * + * A workflow task that schedules a fan-out commits one task per member. + * Each publish used to rebuild the run's whole summary in its own + * transaction, so the step cost grew with the square of the fan-out width. + * The deferred refresh reads the same committed state the last per-task + * refresh read. Publishing still happens only after the owning commit. + * + * @template T + * @param callable(): T $operation + * @return T + */ + public static function coalescingRunSummaries(callable $operation): mixed + { + if (self::$deferredRunSummaries !== null) { + return $operation(); + } + + self::$deferredPublishes = []; + self::$deferredRunSummaries = []; + + try { + $result = $operation(); + } catch (Throwable $throwable) { + try { + self::flushCoalesced(); + } catch (Throwable) { + // The operation's own failure is the one to report. + } + + throw $throwable; + } + + self::flushCoalesced(); + + return $result; + } + public static function dispatch( WorkflowTask $task, bool $queueTimerWakeup = false, @@ -29,9 +95,10 @@ public static function dispatch( ): void { $connection = $task->getConnection(); if ($connection->transactionLevel() > 0) { + $taskId = (string) $task->id; AfterCommit::on( $connection, - static fn () => self::publish($task->id, $queueTimerWakeup, $boundedWorkflowTaskProjection) + static fn () => self::publishOrDefer($taskId, $queueTimerWakeup, $boundedWorkflowTaskProjection) ); return; @@ -62,6 +129,333 @@ public static function dispatchRepairedPollTask( self::publish($task->id, false, $boundedWorkflowTaskProjection); } + /** + * Run a best-effort projection write after the connection's current + * commit, reporting rather than throwing its failure. Inside a coalescing + * scope the write joins the scope's batched post-commit work, ahead of + * the tasks published after it, instead of committing alone (#1203: one + * child-call row per started child). + * + * @param \Closure(): void $write + */ + public static function afterCommitWrite(Connection $connection, \Closure $write): void + { + AfterCommit::on($connection, static function () use ($connection, $write): void { + if (self::$deferredPublishes !== null && ! WorkflowStub::faked()) { + self::$deferredPublishes[] = static function () use ($connection, $write): void { + try { + // Its own savepoint keeps the batch usable after a failure. + $connection->transaction(static fn () => $write()); + } catch (Throwable $throwable) { + // A deadlock or lost connection ended the whole batch + // transaction; never continue in it. + if (self::endsTransaction($throwable)) { + throw $throwable; + } + + report($throwable); + } + }; + + return; + } + + try { + $write(); + } catch (Throwable $throwable) { + report($throwable); + } + }); + } + + private static function publishOrDefer( + string $taskId, + bool $queueTimerWakeup, + bool $boundedWorkflowTaskProjection, + ): void { + if (self::$deferredPublishes !== null && ! WorkflowStub::faked()) { + self::$deferredPublishes[] = [ + 'task_id' => $taskId, + 'queue_timer_wakeup' => $queueTimerWakeup, + 'bounded' => $boundedWorkflowTaskProjection, + ]; + + return; + } + + self::publish($taskId, $queueTimerWakeup, $boundedWorkflowTaskProjection); + } + + private static function flushCoalesced(): void + { + $deferred = self::$deferredPublishes ?? []; + self::$deferredPublishes = null; + $failure = null; + self::$flushingPublishes = true; + + try { + $admissionTaskId = null; + foreach ($deferred as $item) { + if (is_array($item)) { + $admissionTaskId = $item['task_id']; + + break; + } + } + + if ($admissionTaskId === null) { + // Writes alone keep their own transactions. + foreach ($deferred as $write) { + if ($write instanceof \Closure) { + $write(); + } + } + } else { + foreach (array_chunk($deferred, self::COALESCED_BATCH_SIZE) as $batch) { + // Record the batch's writes and dispatch metadata in one + // transaction, then push its jobs only after it commits, + // so a job never runs ahead of the rows it reads. + [$pushes, $batchFailure] = self::inBatchTransaction( + $admissionTaskId, + array_map( + static fn (array|\Closure $item): \Closure => $item instanceof \Closure + ? static function () use ($item): ?array { + $item(); + + return null; + } + : static fn (): ?array => self::preparePublish( + $item['task_id'], + $item['queue_timer_wakeup'], + $item['bounded'], + ), + $batch, + ), + ); + $failure ??= $batchFailure; + + foreach ($pushes as $push) { + $failure = self::pushPrepared($push) ?? $failure; + } + } + } + } finally { + self::$flushingPublishes = false; + $summaries = self::$deferredRunSummaries ?? []; + self::$deferredRunSummaries = null; + } + + foreach (array_chunk($summaries, self::COALESCED_BATCH_SIZE, true) as $batch) { + $entries = array_values($batch); + [, $batchFailure] = self::inBatchTransaction( + $entries[0][0]['task_id'], + array_map( + static fn (string $runId, array $runEntries): \Closure => static function () use ( + $runId, + $runEntries, + ): ?array { + self::refreshDeferredRunSummary($runId, $runEntries); + + return null; + }, + array_map('strval', array_keys($batch)), + $entries, + ), + ); + $failure ??= $batchFailure; + } + + if ($failure !== null) { + throw $failure; + } + } + + /** + * Run each step in one batch transaction, isolating every step in its own + * savepoint so one failure keeps the others, as separate transactions did. + * + * A failure that ends the transaction itself (a deadlock, lock wait or + * lost connection) rolls back every step, so the batch never continues + * in it: the whole batch runs once more in a fresh transaction, and a + * second such failure is thrown with nothing committed. + * + * @param list<\Closure(): ?array> $steps + * @return array{0: list>, 1: ?Throwable} the steps' + * non-null results and the first step failure + */ + private static function inBatchTransaction(string $admissionTaskId, array $steps): array + { + $connection = ConfiguredV2Models::query('task_model', WorkflowTask::class)->getModel() + ->getConnection(); + + for ($attempt = 1; ; $attempt++) { + $results = []; + $failure = null; + $runSteps = static function (Connection $database) use ($steps, &$results, &$failure): void { + foreach ($steps as $step) { + try { + $result = $database->transaction(static fn () => $step()); + } catch (Throwable $throwable) { + if (self::endsTransaction($throwable)) { + throw $throwable; + } + $failure ??= $throwable; + + continue; + } + + if (is_array($result)) { + $results[] = $result; + } + } + }; + + try { + if ($connection->getDriverName() === 'sqlite') { + // Take SQLite's writer once for the batch. + SqliteTransaction::forTask( + $admissionTaskId, + static fn (Connection $database) => $runSteps($database), + ); + } else { + $connection->transaction(static fn (Connection $database) => $runSteps($database), 1); + } + } catch (Throwable $throwable) { + if ($attempt === 1 && self::endsTransaction($throwable) && $connection->transactionLevel() === 0) { + continue; + } + + throw $throwable; + } + + return [$results, $failure]; + } + } + + /** + * Whether a failure ended the surrounding transaction rather than one + * statement: InnoDB rolls a deadlock back entirely, and a lost connection + * loses everything uncommitted. + */ + private static function endsTransaction(Throwable $throwable): bool + { + return TransactionFailure::endsTransaction($throwable); + } + + /** + * Record one deferred publish inside the batch transaction without + * throwing: the dispatch metadata a successful push leaves, or the + * failure a refused dispatch records. Returns the job to push after the + * batch commits, or null when nothing is pushed. + * + * @return array|null + */ + private static function preparePublish( + string $taskId, + bool $queueTimerWakeup, + bool $boundedWorkflowTaskProjection, + ): ?array { + /** @var WorkflowTask|null $task */ + $task = ConfiguredV2Models::query('task_model', WorkflowTask::class)->find($taskId); + + if (! $task instanceof WorkflowTask || $task->status !== TaskStatus::Ready) { + return null; + } + + $job = self::makeJob($task)->beforeCommit(); + $attemptedAt = now(); + + if (self::isPollMode() && ! $queueTimerWakeup) { + self::markDispatched($task, $attemptedAt, $boundedWorkflowTaskProjection); + + return null; + } + + try { + self::ensureBackendSupportsDispatch($task); + } catch (Throwable $throwable) { + self::markDispatchFailure($task, $attemptedAt, $throwable->getMessage(), $boundedWorkflowTaskProjection); + + return [ + 'failure' => $throwable, + ]; + } + + // Timer wakeups are server infrastructure work, independent of SDK worker compatibility. + $fleetBlockReason = $queueTimerWakeup ? null : self::fleetBlockReason($task); + + if ($fleetBlockReason !== null) { + self::markDispatchFailure($task, $attemptedAt, $fleetBlockReason, $boundedWorkflowTaskProjection); + + return null; + } + + $previous = [ + 'last_dispatched_at' => $task->last_dispatched_at, + ]; + if (array_key_exists('queue_delivery_reference', $task->getAttributes())) { + $previous['queue_delivery_reference'] = $task->queue_delivery_reference; + } + self::markDispatched($task, $attemptedAt, $boundedWorkflowTaskProjection); + + return [ + 'task' => $task, + 'job' => $job, + 'attempted_at' => $attemptedAt, + 'previous' => $previous, + 'bounded' => $boundedWorkflowTaskProjection, + ]; + } + + /** + * Push one prepared job after its batch committed. A failed push records + * the dispatch failure on its own, leaving the task as an unbatched + * failed publish leaves it, and is returned for the caller to throw. + * + * @param array $push + */ + private static function pushPrepared(array $push): ?Throwable + { + if (($push['failure'] ?? null) instanceof Throwable) { + return $push['failure']; + } + + /** @var WorkflowTask $task */ + $task = $push['task']; + $job = $push['job']; + + try { + if (ExecutionTracing::enabled()) { + ExecutionTracing::dispatch( + $task, + $job, + static fn () => app(BusDispatcher::class)->dispatch($job), + true + ); + } else { + app(BusDispatcher::class)->dispatch($job); + } + } catch (Throwable $throwable) { + // Leave what an unbatched failed publish leaves: no dispatch mark. + $task->forceFill($push['previous']); + self::markDispatchFailure($task, $push['attempted_at'], $throwable->getMessage(), $push['bounded']); + + return $throwable; + } + + if (array_key_exists('queue_delivery_reference', $task->getAttributes())) { + $reference = DatabaseTaskDelivery::capture($task, $job); + + if ($reference !== null) { + $task->forceFill([ + 'queue_delivery_reference' => $reference, + ])->save(); + } + } + + return null; + } + private static function publish( string $taskId, bool $queueTimerWakeup = false, @@ -239,6 +633,50 @@ private static function markDispatchFailure( } private static function refreshRunSummary(WorkflowTask $task, bool $boundedWorkflowTaskProjection): void + { + // A publish inside a caller's transaction (atomic poll repair) keeps + // its refresh in that transaction. Scope batches defer theirs. + if ( + self::$deferredRunSummaries !== null + && ! WorkflowStub::faked() + && (self::$flushingPublishes || $task->getConnection()->transactionLevel() === 0) + ) { + self::$deferredRunSummaries[(string) $task->workflow_run_id][] = [ + 'task_id' => (string) $task->id, + 'bounded' => $boundedWorkflowTaskProjection, + ]; + + return; + } + + self::refreshRunSummaryNow($task, $boundedWorkflowTaskProjection); + } + + /** + * @param list $entries + */ + private static function refreshDeferredRunSummary(string $runId, array $entries): void + { + if (count($entries) === 1) { + /** @var WorkflowTask|null $task */ + $task = ConfiguredV2Models::query('task_model', WorkflowTask::class)->find($entries[0]['task_id']); + + if ($task instanceof WorkflowTask) { + self::refreshRunSummaryNow($task, $entries[0]['bounded']); + + return; + } + } + + /** @var WorkflowRun|null $run */ + $run = ConfiguredV2Models::query('run_model', WorkflowRun::class)->find($runId); + + if ($run instanceof WorkflowRun) { + self::historyProjectionRole()->projectRun($run); + } + } + + private static function refreshRunSummaryNow(WorkflowTask $task, bool $boundedWorkflowTaskProjection): void { /** @var WorkflowRun|null $run */ $run = ConfiguredV2Models::query('run_model', WorkflowRun::class)->find($task->workflow_run_id); diff --git a/src/V2/Support/TaskScopeMemo.php b/src/V2/Support/TaskScopeMemo.php new file mode 100644 index 000000000..fd5443f6b --- /dev/null +++ b/src/V2/Support/TaskScopeMemo.php @@ -0,0 +1,92 @@ +|null + */ + private static ?array $values = null; + + /** + * @template T + * @param callable(): T $operation + * @return T + */ + public static function scope(callable $operation): mixed + { + if (self::$values !== null) { + return $operation(); + } + + self::$values = []; + + try { + return $operation(); + } finally { + self::$values = null; + } + } + + /** + * The value remembered under $key in the active task scope, or null. + */ + public static function get(string $key): mixed + { + return self::$values[$key] ?? null; + } + + /** + * Remember $value under $key while a task scope is active. + */ + public static function put(string $key, mixed $value): void + { + if (self::$values !== null + && (array_key_exists($key, self::$values) || count(self::$values) < self::MAX_VALUES)) { + self::$values[$key] = $value; + } + } + + /** + * Remember a non-null value for the active task scope. + * + * @template T + * @param Closure(): T $resolve + * @return T + */ + public static function remember(string $key, Closure $resolve): mixed + { + if (self::$values === null) { + return $resolve(); + } + + if (array_key_exists($key, self::$values)) { + return self::$values[$key]; + } + + $value = $resolve(); + + if ($value !== null && count(self::$values) < self::MAX_VALUES) { + self::$values[$key] = $value; + } + + return $value; + } +} diff --git a/src/V2/Support/TransactionFailure.php b/src/V2/Support/TransactionFailure.php new file mode 100644 index 000000000..b19ec4a60 --- /dev/null +++ b/src/V2/Support/TransactionFailure.php @@ -0,0 +1,35 @@ +getPrevious()) { + if ($detector->causedByConcurrencyError($current) || $detector->causedByLostConnection($current)) { + return true; + } + } + + return false; + } +} diff --git a/src/V2/Support/WorkflowExecutor.php b/src/V2/Support/WorkflowExecutor.php index 5ae5c4567..5cf51fc16 100644 --- a/src/V2/Support/WorkflowExecutor.php +++ b/src/V2/Support/WorkflowExecutor.php @@ -5896,20 +5896,7 @@ private function unserializePayloadWithRun(string $serialized, ?WorkflowRun $run private function activityCompletionEvent(WorkflowRun $run, int $sequence): ?WorkflowHistoryEvent { /** @var WorkflowHistoryEvent|null $event */ - $event = $run->historyEvents->first( - static fn (WorkflowHistoryEvent $event): bool => in_array( - $event->event_type, - [ - HistoryEventType::ActivityCompleted, - HistoryEventType::ActivityFailed, - HistoryEventType::ActivityCancelled, - HistoryEventType::ActivityTimedOut, - ], - true, - ) && ($event->payload['sequence'] ?? null) === $sequence - // The run's own close recorded this; the workflow never saw it (#1117). - && ! ClosingRunOpenWork::settledAtClose($event) - ); + $event = $this->firstActivityEventForSequence($run, 'completion', $sequence); ExecutionTracing::linkOutcome($run, $event); @@ -5919,22 +5906,59 @@ private function activityCompletionEvent(WorkflowRun $run, int $sequence): ?Work private function activityOpenEvent(WorkflowRun $run, int $sequence): ?WorkflowHistoryEvent { /** @var WorkflowHistoryEvent|null $event */ - $event = $run->historyEvents->first( - static fn (WorkflowHistoryEvent $event): bool => in_array( - $event->event_type, - [ - HistoryEventType::ActivityScheduled, - HistoryEventType::ActivityStarted, - HistoryEventType::ActivityHeartbeatRecorded, - HistoryEventType::ActivityRetryScheduled, - ], - true, - ) && ($event->payload['sequence'] ?? null) === $sequence - ); + $event = $this->firstActivityEventForSequence($run, 'open', $sequence); return $event; } + /** + * The first activity resolution or open event for one durable sequence, + * in history order. Replaying a fan-out reads every member's events, so + * the loaded history is indexed as far as each lookup needs instead of + * scanned per member (#1231). + * + * @param 'completion'|'open' $kind + */ + private function firstActivityEventForSequence(WorkflowRun $run, string $kind, int $sequence): mixed + { + $types = $kind === 'completion' + ? [ + HistoryEventType::ActivityCompleted, + HistoryEventType::ActivityFailed, + HistoryEventType::ActivityCancelled, + HistoryEventType::ActivityTimedOut, + ] + : [ + HistoryEventType::ActivityScheduled, + HistoryEventType::ActivityStarted, + HistoryEventType::ActivityHeartbeatRecorded, + HistoryEventType::ActivityRetryScheduled, + ]; + + return HistoryEventIndex::find( + $run->historyEvents, + "activity-{$kind}-event-by-sequence", + $sequence, + static function (mixed $event) use ($kind, $types): array { + if (! $event instanceof WorkflowHistoryEvent || ! in_array($event->event_type, $types, true)) { + return []; + } + + $eventSequence = $event->payload['sequence'] ?? null; + + // The run's own close recorded this; the workflow never saw it (#1117). + if (! is_int($eventSequence) + || ($kind === 'completion' && ClosingRunOpenWork::settledAtClose($event))) { + return []; + } + + return [ + $eventSequence => $event, + ]; + }, + ); + } + private function activityHistoryEvent(WorkflowRun $run, int $sequence): ?WorkflowHistoryEvent { return $this->activityCompletionEvent($run, $sequence) diff --git a/src/V2/Support/WorkflowStepHistory.php b/src/V2/Support/WorkflowStepHistory.php index de8b10944..11f15486e 100644 --- a/src/V2/Support/WorkflowStepHistory.php +++ b/src/V2/Support/WorkflowStepHistory.php @@ -239,19 +239,7 @@ public static function conflictingEventTypesForSequence( ): array { $eventTypes = []; - foreach (self::historyEventsForRun($run)->sortBy('sequence') as $event) { - if (! $event instanceof WorkflowHistoryEvent) { - continue; - } - - if (! self::isWorkflowStepEvent($event)) { - continue; - } - - if (self::intValue($event->payload['sequence'] ?? null) !== $sequence) { - continue; - } - + foreach (self::stepEventsForSequence($run, $sequence) as $event) { if (self::eventMatchesShape($event, $expectedShape)) { continue; } @@ -317,16 +305,8 @@ private static function detailMismatchForSequence( return null; } - foreach (self::historyEventsForRun($run)->sortBy('sequence') as $event) { - if (! $event instanceof WorkflowHistoryEvent) { - continue; - } - - if (! self::isWorkflowStepEvent($event) || ! self::eventMatchesShape($event, $expectedShape)) { - continue; - } - - if (self::intValue($event->payload['sequence'] ?? null) !== $sequence) { + foreach (self::stepEventsForSequence($run, $sequence) as $event) { + if (! self::eventMatchesShape($event, $expectedShape)) { continue; } @@ -637,25 +617,48 @@ private static function workflowStepEventTypesForSequence(WorkflowRun $run, int { $eventTypes = []; - foreach (self::historyEventsForRun($run)->sortBy('sequence') as $event) { - if (! $event instanceof WorkflowHistoryEvent) { - continue; - } - - if (! self::isWorkflowStepEvent($event)) { - continue; - } - - if (self::intValue($event->payload['sequence'] ?? null) !== $sequence) { - continue; - } - + foreach (self::stepEventsForSequence($run, $sequence) as $event) { $eventTypes[] = $event->event_type->value; } return array_values(array_unique($eventTypes)); } + /** + * The run's workflow-step events recorded for one durable sequence, in + * history order. Replay validates every fan-out member, so the events are + * indexed by sequence once per loaded history instead of rescanned per + * member (#1231). + * + * @return list + */ + private static function stepEventsForSequence(WorkflowRun $run, int $sequence): array + { + $index = HistoryEventIndex::remember( + self::historyEventsForRun($run), + 'workflow-step-events-by-sequence', + static function (Collection $events): array { + $index = []; + + foreach ($events->sortBy('sequence') as $event) { + if (! $event instanceof WorkflowHistoryEvent || ! self::isWorkflowStepEvent($event)) { + continue; + } + + $eventSequence = self::intValue($event->payload['sequence'] ?? null); + + if ($eventSequence !== null) { + $index[$eventSequence][] = $event; + } + } + + return $index; + }, + ); + + return $index[$sequence] ?? []; + } + /** * @return Collection */ diff --git a/tests/Feature/V2/V2CoalescedTaskPublishTest.php b/tests/Feature/V2/V2CoalescedTaskPublishTest.php new file mode 100644 index 000000000..7ec759253 --- /dev/null +++ b/tests/Feature/V2/V2CoalescedTaskPublishTest.php @@ -0,0 +1,226 @@ +start(3, 16); + $scheduling = $this->readyWorkflowTask($workflow->runId()); + + $bus = $this->app->make(BusDispatcher::class); + $this->app->instance(BusDispatcher::class, new class($bus) { + public function __construct( + private readonly object $bus, + ) { + } + + public function dispatch(mixed $job): mixed + { + if ($job instanceof RunActivityTask) { + throw new RuntimeException('queue unavailable'); + } + + return $this->bus->dispatch($job); + } + + public function __call(string $method, array $arguments): mixed + { + return $this->bus->{$method}(...$arguments); + } + }); + + try { + $this->app->call([new RunWorkflowTask($scheduling->id), 'handle']); + $this->fail('A failed publish must be reported.'); + } catch (RuntimeException $exception) { + $this->assertSame('queue unavailable', $exception->getMessage()); + } finally { + $this->app->instance(BusDispatcher::class, $bus); + } + + $activityTasks = WorkflowTask::query() + ->where('workflow_run_id', $workflow->runId()) + ->where('task_type', TaskType::Activity->value) + ->get(); + $this->assertCount(3, $activityTasks); + foreach ($activityTasks as $task) { + $this->assertSame(TaskStatus::Ready, $task->status); + $this->assertSame('queue unavailable', $task->last_dispatch_error); + $this->assertNotNull($task->last_dispatch_attempt_at); + $this->assertNotNull($task->repair_available_at); + $this->assertTrue($task->repair_available_at->lte(now()->addSecond())); + $this->assertNull($task->last_dispatched_at); + } + } + + public function testATransactionEndingFailureRestartsTheBatchInsteadOfContinuing(): void + { + Queue::fake(); + $workflow = WorkflowStub::make(TestFanOutPayloadActivityWorkflow::class, 'coalesced-deadlock-restart'); + $workflow->start(1, 16); + $task = $this->readyWorkflowTask($workflow->runId()); + $before = (int) $task->repair_count; + Queue::fake(); + $deadlocks = 1; + + TaskDispatcher::coalescingRunSummaries(function () use ($task, &$deadlocks): void { + $connection = $task->getConnection(); + $connection->transaction(function () use ($connection, $task, &$deadlocks): void { + TaskDispatcher::afterCommitWrite( + $connection, + static fn () => WorkflowTask::query()->whereKey($task->id)->increment('repair_count'), + ); + TaskDispatcher::afterCommitWrite($connection, function () use (&$deadlocks): void { + if ($deadlocks-- > 0) { + throw $this->deadlock(); + } + }); + TaskDispatcher::dispatch($task); + }); + }); + + // The first attempt rolled back whole; the restarted batch applied + // every step exactly once and then pushed the job once. + $this->assertSame($before + 1, (int) $task->fresh()->repair_count); + Queue::assertPushed(RunWorkflowTask::class, 1); + } + + public function testARepeatedTransactionEndingFailureCommitsAndPushesNothing(): void + { + Queue::fake(); + $workflow = WorkflowStub::make(TestFanOutPayloadActivityWorkflow::class, 'coalesced-deadlock-twice'); + $workflow->start(1, 16); + $task = $this->readyWorkflowTask($workflow->runId()); + $before = (int) $task->repair_count; + $attemptedBefore = $task->getRawOriginal('last_dispatch_attempt_at'); + Queue::fake(); + + try { + TaskDispatcher::coalescingRunSummaries(function () use ($task): void { + $connection = $task->getConnection(); + $connection->transaction(function () use ($connection, $task): void { + TaskDispatcher::afterCommitWrite( + $connection, + static fn () => WorkflowTask::query()->whereKey($task->id)->increment('repair_count'), + ); + TaskDispatcher::afterCommitWrite($connection, function (): void { + throw $this->deadlock(); + }); + TaskDispatcher::dispatch($task); + }); + }); + $this->fail('A repeated deadlock must be reported.'); + } catch (Throwable $exception) { + // Laravel rethrows a deadlock as DeadlockException. + $this->assertStringContainsString('Deadlock found', $exception->getMessage()); + } + + $this->assertSame($before, (int) $task->fresh()->repair_count); + $this->assertSame($attemptedBefore, $task->fresh()->getRawOriginal('last_dispatch_attempt_at')); + Queue::assertNothingPushed(); + } + + public function testChildJobsArePushedOnlyAfterTheirCallRowsCommit(): void + { + Queue::fake(); + $workflow = WorkflowStub::make(TestFanOutPayloadChildWorkflow::class, 'coalesced-child-push-after-commit'); + $workflow->start(3, 16); + $scheduling = $this->readyWorkflowTask($workflow->runId()); + $runId = $workflow->runId(); + $observed = new \ArrayObject(); + + $bus = $this->app->make(BusDispatcher::class); + $this->app->instance(BusDispatcher::class, new class($bus, $runId, $observed) { + public function __construct( + private readonly object $bus, + private readonly string $runId, + private readonly \ArrayObject $observed, + ) { + } + + public function dispatch(mixed $job): mixed + { + $this->observed[] = [ + 'transaction_level' => DB::connection()->transactionLevel(), + 'child_calls' => WorkflowChildCall::query() + ->where('parent_workflow_run_id', $this->runId) + ->count(), + ]; + + return $this->bus->dispatch($job); + } + + public function __call(string $method, array $arguments): mixed + { + return $this->bus->{$method}(...$arguments); + } + }); + + try { + $this->app->call([new RunWorkflowTask($scheduling->id), 'handle']); + } finally { + $this->app->instance(BusDispatcher::class, $bus); + } + + $this->assertCount(3, $observed); + foreach ($observed->getArrayCopy() as $push) { + $this->assertSame(0, $push['transaction_level']); + $this->assertSame(3, $push['child_calls']); + } + } + + protected function startQueueWorkersAutomatically(): bool + { + return false; + } + + private function readyWorkflowTask(string $runId): WorkflowTask + { + return WorkflowTask::query() + ->where('workflow_run_id', $runId) + ->where('task_type', TaskType::Workflow->value) + ->where('status', TaskStatus::Ready->value) + ->sole(); + } + + private function deadlock(): QueryException + { + return new QueryException( + DB::connection()->getName(), + 'update workflow_tasks set id = id', + [], + new PDOException('SQLSTATE[40001]: Serialization failure: 1213 Deadlock found when trying to get lock'), + ); + } +} diff --git a/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php b/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php index b081bfba8..8d132c560 100644 --- a/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php +++ b/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php @@ -5,6 +5,7 @@ namespace Tests\Feature\V2; use DateTimeImmutable; +use Illuminate\Database\Eloquent\Collection as EloquentCollection; use Illuminate\Queue\WorkerOptions; use Illuminate\Support\Carbon; use Illuminate\Support\Facades\Artisan; @@ -15,9 +16,11 @@ use Illuminate\Support\Str; use Illuminate\Validation\ValidationException; use Mockery; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\Attributes\PreserveGlobalState; use PHPUnit\Framework\Attributes\RunInSeparateProcess; use Symfony\Component\Uid\Ulid; +use Tests\Fixtures\V2\TestFanOutPayloadActivity; use Tests\Fixtures\V2\TestGreetingActivity; use Tests\Fixtures\V2\TestGreetingWorkflow; use Tests\Fixtures\V2\TestLocalActivityWorkflow; @@ -80,7 +83,9 @@ use Workflow\V2\Models\WorkflowTimer; use Workflow\V2\Models\WorkflowUpdate; use Workflow\V2\StartOptions; +use Workflow\V2\Support\ActivityCall; use Workflow\V2\Support\ActivityCancellationAcknowledgement; +use Workflow\V2\Support\ActivityReplayIdentity; use Workflow\V2\Support\BundleIntegrityVerifier; use Workflow\V2\Support\CancellationScopeDelivery; use Workflow\V2\Support\CancellationScopeHistory; @@ -93,6 +98,7 @@ use Workflow\V2\Support\EmbeddedV2HistoryImport; use Workflow\V2\Support\HistoryExport; use Workflow\V2\Support\MemoPayload; +use Workflow\V2\Support\ParallelChildGroup; use Workflow\V2\Support\QueryStateReplayer; use Workflow\V2\Support\ReplayDiff; use Workflow\V2\Support\RunActivityView; @@ -3192,6 +3198,112 @@ public function testSelectionLoserCorpusIsSettledWhenTheRunCloses(): void } } + /** + * Replaying a fan-out looks up every member by sequence. Each lookup + * family indexes the loaded history once, so the recorded events are read + * a bounded number of times however many members the group has; a scan + * per member read the whole history again for each one (#1203, #1231). + * A live run on the current code records the evidence history unchanged. + * + * @return iterable + */ + public static function fanOutHistoryIndexCorpus(): iterable + { + yield 'activities' => [ + 'fan-out-activity-all-history-index', + WorkflowStepHistory::ACTIVITY, + 'activity_type', + ]; + yield 'children' => [ + 'fan-out-child-all-history-index', + WorkflowStepHistory::CHILD_WORKFLOW, + 'child_workflow_type', + ]; + } + + #[DataProvider('fanOutHistoryIndexCorpus')] + public function testFanOutCorpusReadsRecordedHistoryOncePerLookupFamily( + string $fixtureId, + string $shape, + string $detailField, + ): void { + $path = self::FIXTURE_DIR . "/{$fixtureId}.json"; + if (! is_file($path)) { + $this->markTestSkipped('The fan-out history index evidence is absent.'); + } + /** @var array $fixture */ + $fixture = json_decode((string) file_get_contents($path), true, flags: JSON_THROW_ON_ERROR); + $history = $fixture['history']; + $members = []; + foreach ($history as $event) { + if (in_array($event['event_type'], ['ActivityScheduled', 'ChildWorkflowScheduled'], true)) { + $members[$event['payload']['sequence']] = $event['payload'][$detailField]; + } + } + $this->assertCount(3, $members); + + $run = new WorkflowRun([ + 'id' => 'fan-out-corpus-' . $fixtureId, + ]); + $run->setRelation('historyEvents', new EloquentCollection(array_map( + static fn (array $event): FanOutPayloadReadCountingEvent => (new FanOutPayloadReadCountingEvent()) + ->forceFill([ + ...$event, + 'workflow_run_id' => $run->id, + ]), + $history, + ))); + FanOutPayloadReadCountingEvent::$foreignReads = 0; + $indexed = false; + + foreach ($members as $sequence => $type) { + // The first member's lookups may read the whole history. + FanOutPayloadReadCountingEvent::$member = $indexed ? $sequence : null; + WorkflowStepHistory::assertCompatible($run, $sequence, $shape, [ + $detailField => $type, + ]); + if ($shape === WorkflowStepHistory::ACTIVITY) { + ActivityReplayIdentity::assertCall( + $run, + new ActivityCall(TestFanOutPayloadActivity::class, [$sequence - 1, '']), + $sequence, + ); + } + $this->assertSame( + $sequence, + ParallelChildGroup::metadataPathForSequence($run, $sequence)[0]['parallel_group_base_sequence'] + + ParallelChildGroup::metadataPathForSequence($run, $sequence)[0]['parallel_group_index'], + ); + if ($shape === WorkflowStepHistory::CHILD_WORKFLOW) { + $this->assertSame( + 'ChildWorkflowScheduled', + ChildRunHistory::scheduledEventForSequence($run, $sequence)?->event_type->value, + ); + $this->assertSame( + 'ChildRunStarted', + ChildRunHistory::startedEventForSequence($run, $sequence)?->event_type->value, + ); + $this->assertSame( + 'ChildRunCompleted', + ChildRunHistory::resolutionEventForSequence($run, $sequence)?->event_type->value, + ); + } + $indexed = true; + } + FanOutPayloadReadCountingEvent::$member = null; + + // Indexing reads each recorded event at most once per lookup family, + // so all later members together read fewer foreign events than one + // more pass over the history. A scan per member reads them all again. + $this->assertLessThan( + count($history), + FanOutPayloadReadCountingEvent::$foreignReads, + 'Fan-out replay lookups rescanned other members\' recorded events for each member.', + ); + + $this->assertLiveFanOutRecordsEvidenceHistory($fixture); + } + /** * @param array $fixture */ @@ -3283,6 +3395,100 @@ class_exists(VersionUsageQuery::class), $this->assertSame($history, array_slice($run->historyEvents()->get()->toArray(), 0, count($history))); } + /** + * @param array $fixture + */ + private function assertLiveFanOutRecordsEvidenceHistory(array $fixture): void + { + self::stopWorkers(); + // The evidence was recorded on the redis queue connection. + config([ + 'queue.default' => 'redis', + ]); + Queue::fake(); + $started = array_values(array_filter( + $fixture['history'], + static fn (array $event): bool => $event['event_type'] === 'WorkflowStarted', + ))[0]; + $workflow = WorkflowStub::make($fixture['workflow']['type'], $started['payload']['workflow_instance_id']); + $workflow->start(...$fixture['workflow']['arguments']); + $runId = $workflow->runId(); + + for ($step = 0; $step < 100; $step++) { + /** @var WorkflowTask|null $task */ + $task = WorkflowTask::query() + ->where('status', TaskStatus::Ready->value) + ->orderBy('created_at') + ->orderBy('id') + ->first(); + if ($task === null) { + break; + } + $job = match ($task->task_type) { + TaskType::Workflow => new RunWorkflowTask($task->id), + TaskType::Activity => new RunActivityTask($task->id), + TaskType::Timer => new RunTimerTask($task->id), + }; + $this->app->call([$job, 'handle']); + } + + $this->assertTrue($workflow->refresh()->completed()); + $this->assertSame($fixture['expected']['result'], $workflow->output()); + $recorded = WorkflowHistoryEvent::query() + ->where('workflow_run_id', $runId) + ->orderBy('sequence') + ->get() + ->map(static fn (WorkflowHistoryEvent $event): array => [ + 'sequence' => $event->sequence, + 'event_type' => $event->event_type->value, + // Evidence omits the claimed task and command snapshots. + 'payload' => array_diff_key($event->payload, [ + 'task' => true, + 'command' => true, + ]), + ]) + ->all(); + $evidence = array_map(static fn (array $event): array => [ + 'sequence' => $event['sequence'], + 'event_type' => $event['event_type'], + 'payload' => $event['payload'], + ], $fixture['history']); + + $this->assertSame(self::withoutRunIdentity($evidence), self::withoutRunIdentity($recorded)); + } + + /** + * Mask generated identifiers and timestamps and ignore object key order; + * every other recorded value, key and event position must match. + */ + private static function withoutRunIdentity(mixed $value): mixed + { + if (is_array($value)) { + $value = array_map(self::withoutRunIdentity(...), $value); + // MySQL's JSON column reorders object keys; compare objects by key. + if (! array_is_list($value)) { + ksort($value); + } + + return $value; + } + if (! is_string($value)) { + return $value; + } + if (strlen($value) > 40 && preg_match('/^[A-Za-z0-9+\/]+={0,2}$/', $value) === 1) { + $decoded = base64_decode($value, true); + if ($decoded !== false && preg_match('/01[0-9A-Za-z]{24}/', $decoded) === 1) { + return 'payload-with-identity'; + } + } + + return preg_replace( + ['/01[0-9A-Za-z]{24}/', '/\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:?\d{2})?/'], + ['', '