From 4a39ab5e57def06bb91827aed1d0e243c610a4da Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 19 Nov 2026 16:33:57 +0000 Subject: [PATCH 1/3] Do not rerun a failed activity redispatched before its failure is recorded A v1 activity's failure is recorded by a separately queued Workflow\Exception job, after Laravel has released the activity's unique lock. A workflow replay in that gap dispatched the activity again and the copy executed it, adding an exception row and repeating its side effects (the #1241 flake: the second retry attempt ran twice). The failing job now marks the activity in the cache before its lock is released. Another job for the same index that finds the mark completes without executing or recording a result; the Exception job removes the mark once it has recorded the failure or let it go. History, payloads and replay are unchanged. Fixes #1241. Co-Authored-By: Claude Opus 5.5 --- .github/feature-test-timings.json | 2 + CHANGELOG.md | 10 ++ src/Activity.php | 61 ++++++++- src/Exception.php | 6 + src/Middleware/ActivityMiddleware.php | 5 +- .../Feature/FailedActivityRedispatchTest.php | 126 ++++++++++++++++++ .../TestFailedActivityRedispatchActivity.php | 22 +++ .../TestFailedActivityRedispatchWorkflow.php | 21 +++ 8 files changed, 251 insertions(+), 2 deletions(-) create mode 100644 tests/Feature/FailedActivityRedispatchTest.php create mode 100644 tests/Fixtures/TestFailedActivityRedispatchActivity.php create mode 100644 tests/Fixtures/TestFailedActivityRedispatchWorkflow.php diff --git a/.github/feature-test-timings.json b/.github/feature-test-timings.json index e876481ea..94a9d137b 100644 --- a/.github/feature-test-timings.json +++ b/.github/feature-test-timings.json @@ -17,6 +17,7 @@ "tests/Feature/ExceptionLoggingReplayTest.php": 7.228751, "tests/Feature/ExceptionWorkflowTest.php": 8.416919, "tests/Feature/ExplicitQueueWorkerTest.php": 4.186096, + "tests/Feature/FailedActivityRedispatchTest.php": 6.5, "tests/Feature/FailingWorkflowTest.php": 6.256735, "tests/Feature/HeartbeatWorkflowTest.php": 13.477668, "tests/Feature/MigrationTest.php": 55.40979, @@ -270,6 +271,7 @@ "tests/Feature/ExceptionLoggingReplayTest.php": 8.584815, "tests/Feature/ExceptionWorkflowTest.php": 4.583109, "tests/Feature/ExplicitQueueWorkerTest.php": 0.87937, + "tests/Feature/FailedActivityRedispatchTest.php": 6.0, "tests/Feature/FailingWorkflowTest.php": 3.523026, "tests/Feature/HeartbeatWorkflowTest.php": 13.476006, "tests/Feature/MigrationTest.php": 12.426278, diff --git a/CHANGELOG.md b/CHANGELOG.md index 55987ea73..d7886e03d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,6 +58,16 @@ published package. `docs/business-state-machines.md` describes what it checks. The compensation-scope and state-machine journeys now share `scripts/ci/upgrade-journey-lib.sh`. +- A v1 activity whose job failed no longer runs a second time when the + workflow replays before the failure is recorded (#1241). The failure is + recorded by a separately queued `Workflow\Exception` job after the activity's + unique lock is released, so a wake in between (a signal, timer or sibling + result) dispatched the activity again and the copy executed it, adding an + exception row and repeating its side effects. The failed job now marks the + activity in the cache before its lock is released; a copy that finds the mark + completes without executing or recording a result, and the Exception job + removes the mark once it has recorded the failure or let it go. Recorded + history, job payloads and replay are unchanged. ## 2.33.0 - 2026-11-19 diff --git a/src/Activity.php b/src/Activity.php index f22e488e7..8bca0c9f3 100644 --- a/src/Activity.php +++ b/src/Activity.php @@ -46,6 +46,9 @@ class Activity implements ShouldBeEncrypted, ShouldBeUnique, ShouldQueue private Container $container; + // Set while handling only; an unset typed property is not serialized. + private bool $awaitingRecordedFailure; + public function __construct( public int $index, public string $now, @@ -111,12 +114,39 @@ public function handle() $this->container = App::make(Container::class); + // A failed job's outcome is recorded later by a queued Exception job, + // after this activity's unique lock is gone. A replay in between + // dispatches the activity again; that copy must not run it again. + $failedJob = Cache::get(self::failureKey($this->storedWorkflow->id, $this->index)); + if ($this->storedWorkflow->hasLogByIndex($this->index)) { return; } + if ($failedJob !== null && $failedJob !== $this->queueJobUuid()) { + $this->awaitingRecordedFailure = true; + + return; + } + + // The Exception job forgets the failure only after recording it, so + // logs loaded with this job's payload may predate that record. + if ($failedJob === null && $this->storedWorkflow->relationLoaded('logs') && $this->storedWorkflow->logs() + ->whereIndex($this->index) + ->exists()) { + return; + } + try { - return $this->{'execute'}(...$this->resolveClassMethodDependencies($this->arguments, $this, 'execute')); + $result = $this->{'execute'}( + ...$this->resolveClassMethodDependencies($this->arguments, $this, 'execute') + ); + + if ($failedJob !== null) { + Cache::forget(self::failureKey($this->storedWorkflow->id, $this->index)); + } + + return $result; } catch (\Throwable $throwable) { $this->storedWorkflow->exceptions() ->create([ @@ -124,6 +154,12 @@ public function handle() 'exception' => Serializer::serialize($throwable), ]); + // Recorded before the job can fail and release its unique lock. + $uuid = $this->queueJobUuid(); + if ($uuid !== null) { + Cache::forever(self::failureKey($this->storedWorkflow->id, $this->index), $uuid); + } + if ($throwable instanceof NonRetryableExceptionContract) { $this->fail($throwable); } @@ -132,6 +168,22 @@ public function handle() } } + /** + * @internal + */ + public static function failureKey(int|string $workflowId, int $index): string + { + return 'laravel-workflow-activity-failure:' . $workflowId . ':' . $index; + } + + /** + * @internal True when another job of this activity failed and its outcome is not yet recorded. + */ + public function awaitsRecordedFailure(): bool + { + return $this->awaitingRecordedFailure ?? false; + } + public function middleware() { return [ @@ -183,4 +235,11 @@ public function heartbeat(): void Cache::put($this->key, 1, $this->timeout); } } + + private function queueJobUuid(): ?string + { + $uuid = $this->job?->uuid(); + + return is_string($uuid) && $uuid !== '' ? $uuid : null; + } } diff --git a/src/Exception.php b/src/Exception.php index e5e2bea41..6c1960dd3 100644 --- a/src/Exception.php +++ b/src/Exception.php @@ -91,6 +91,12 @@ public function handle() $this->release(); } } + + // Until here a redispatched copy of the failed activity waits for + // this outcome instead of running again (Activity::handle). + if ($this->job === null || ! $this->job->isReleased()) { + Cache::forget(Activity::failureKey($this->storedWorkflow->id, $this->index)); + } } finally { $lock->release(); } diff --git a/src/Middleware/ActivityMiddleware.php b/src/Middleware/ActivityMiddleware.php index 6e71f1f8f..45c5c772c 100644 --- a/src/Middleware/ActivityMiddleware.php +++ b/src/Middleware/ActivityMiddleware.php @@ -38,7 +38,10 @@ public function handle($job, $next): void try { $this->result = $next($job); - $job->onUnlock = fn (bool $shouldSignal) => $this->onUnlock($shouldSignal); + // A copy skipped for another job's unrecorded failure has no result. + if (! method_exists($job, 'awaitsRecordedFailure') || ! $job->awaitsRecordedFailure()) { + $job->onUnlock = fn (bool $shouldSignal) => $this->onUnlock($shouldSignal); + } } catch (\Throwable $throwable) { $file = new SplFileObject($throwable->getFile()); $iterator = new LimitIterator($file, max(0, $throwable->getLine() - 4), 7); diff --git a/tests/Feature/FailedActivityRedispatchTest.php b/tests/Feature/FailedActivityRedispatchTest.php new file mode 100644 index 000000000..28427d8fe --- /dev/null +++ b/tests/Feature/FailedActivityRedispatchTest.php @@ -0,0 +1,126 @@ +getConnection(); + $key = Queue::connection('redis')->getQueue($queue); + $exceptionLock = null; + + try { + $workflow = WorkflowStub::make(TestFailedActivityRedispatchWorkflow::class); + $workflow->start($directory, new WorkflowOptions(connection: 'redis', queue: $queue)); + $this->work($queue); + $this->assertSame(WorkflowWaitingStatus::class, $workflow->fresh()->status()); + $this->assertSame(1, $redis->llen($key), 'The workflow dispatches its activity.'); + + // Any wake (a signal, timer or sibling result) replays the workflow. + $workflow->resume(); + $this->assertSame(2, $redis->llen($key)); + + // The activity fails its job; the Exception job is queued behind + // the wake and the activity's unique lock is released. + $this->work($queue); + $this->assertSame(1, $this->executions($directory)); + $this->assertSame(1, $workflow->exceptions()->count()); + $this->assertSame(0, $workflow->logs()->count()); + + // The wake replays before the failure is recorded and dispatches + // the activity again behind the Exception job. + $this->work($queue); + $this->assertSame(0, $workflow->logs()->count()); + $this->assertSame(2, $redis->llen($key), 'The replay dispatches the unrecorded activity again.'); + + // The Exception job is still waiting to record the failure. + $exceptionLock = Cache::lock('laravel-workflow-exception:' . $workflow->id(), 15); + $this->assertTrue($exceptionLock->get()); + $this->work($queue); + $this->assertSame(1, $redis->llen($key)); + $this->assertSame(1, $redis->zcard($key . ':delayed')); + + // The redispatched copy runs while the failure is unrecorded. + $this->work($queue); + $this->assertSame(1, $this->executions($directory), 'The failed activity ran again.'); + $this->assertSame(1, $workflow->exceptions()->count()); + + $exceptionLock->release(); + $exceptionLock = null; + $this->drain($queue, $workflow); + + $this->assertSame(WorkflowCompletedStatus::class, $workflow->status()); + $this->assertSame('caught: activity failed', $workflow->output()); + $this->assertSame(1, $this->executions($directory)); + $this->assertSame(1, $workflow->exceptions()->count()); + $this->assertSame([Exception::class], $workflow->logs()->pluck('class')->all()); + $this->assertSame( + [TestFailedActivityRedispatchActivity::class], + $workflow->exceptions() + ->pluck('class') + ->all() + ); + } finally { + $exceptionLock?->release(); + foreach (glob($directory . '/*') ?: [] as $file) { + unlink($file); + } + rmdir($directory); + } + } + + private function work(string $queue): void + { + $process = new Process(['php', __DIR__ . '/../../vendor/bin/testbench', 'queue:work', 'redis', + '--queue=' . $queue, '--once', '--sleep=0'], env: [ + 'WORKFLOW_WATCHDOG_ENABLED' => 'false', + ]); + $process->setTimeout(30); + $process->mustRun(); + } + + private function drain(string $queue, WorkflowStub $workflow): void + { + $deadline = microtime(true) + 30; + + while (! $workflow->fresh()->completed() && ! $workflow->failed()) { + if (microtime(true) > $deadline) { + $this->fail('The workflow did not finish after its failure was recorded.'); + } + + $this->work($queue); + usleep(100000); + } + } + + private function executions(string $directory): int + { + $path = $directory . '/executions'; + + return is_file($path) ? substr_count((string) file_get_contents($path), "\n") : 0; + } +} diff --git a/tests/Fixtures/TestFailedActivityRedispatchActivity.php b/tests/Fixtures/TestFailedActivityRedispatchActivity.php new file mode 100644 index 000000000..8b5dbf760 --- /dev/null +++ b/tests/Fixtures/TestFailedActivityRedispatchActivity.php @@ -0,0 +1,22 @@ +getMessage(); + } + } +} From 2d5c7ebc33a734a698912834923101bd41a9a079 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 19 Nov 2026 16:58:26 +0000 Subject: [PATCH 2/3] Keep the failed-activity mark compatible with mocked jobs; cover it in unit tests Track the Exception job's own release instead of asking the queue job, expose the copy's skip as an unserialized public property, and cover marking, skipping, same-job retries and removal in unit tests. Co-Authored-By: Claude Opus 5.5 --- src/Activity.php | 17 +++----- src/Exception.php | 6 ++- src/Middleware/ActivityMiddleware.php | 2 +- tests/Unit/ActivityTest.php | 61 +++++++++++++++++++++++++++ tests/Unit/ExceptionTest.php | 34 +++++++++++++++ 5 files changed, 107 insertions(+), 13 deletions(-) diff --git a/src/Activity.php b/src/Activity.php index 8bca0c9f3..cb629e0f4 100644 --- a/src/Activity.php +++ b/src/Activity.php @@ -44,10 +44,13 @@ class Activity implements ShouldBeEncrypted, ShouldBeUnique, ShouldQueue public $key = ''; - private Container $container; + /** + * @internal Set while handling, when another job of this activity failed + * and its outcome is not yet recorded. Unset, it is not serialized. + */ + public bool $awaitingRecordedFailure; - // Set while handling only; an unset typed property is not serialized. - private bool $awaitingRecordedFailure; + private Container $container; public function __construct( public int $index, @@ -176,14 +179,6 @@ public static function failureKey(int|string $workflowId, int $index): string return 'laravel-workflow-activity-failure:' . $workflowId . ':' . $index; } - /** - * @internal True when another job of this activity failed and its outcome is not yet recorded. - */ - public function awaitsRecordedFailure(): bool - { - return $this->awaitingRecordedFailure ?? false; - } - public function middleware() { return [ diff --git a/src/Exception.php b/src/Exception.php index 6c1960dd3..3f87b66cf 100644 --- a/src/Exception.php +++ b/src/Exception.php @@ -71,6 +71,8 @@ public function handle() return; } + $released = false; + try { $workflow = $this->storedWorkflow->toWorkflow(); @@ -84,17 +86,19 @@ public function handle() $workflow->next($this->index, $this->now, self::class, $this->exceptionPayload()); } elseif ($probeDecision === self::PROBE_RETRY) { $this->release(self::LOCK_RETRY_DELAY); + $released = true; } } } catch (TransitionNotFound) { if ($workflow->running()) { $this->release(); + $released = true; } } // Until here a redispatched copy of the failed activity waits for // this outcome instead of running again (Activity::handle). - if ($this->job === null || ! $this->job->isReleased()) { + if (! $released) { Cache::forget(Activity::failureKey($this->storedWorkflow->id, $this->index)); } } finally { diff --git a/src/Middleware/ActivityMiddleware.php b/src/Middleware/ActivityMiddleware.php index 45c5c772c..a6517a7e3 100644 --- a/src/Middleware/ActivityMiddleware.php +++ b/src/Middleware/ActivityMiddleware.php @@ -39,7 +39,7 @@ public function handle($job, $next): void $this->result = $next($job); // A copy skipped for another job's unrecorded failure has no result. - if (! method_exists($job, 'awaitsRecordedFailure') || ! $job->awaitsRecordedFailure()) { + if (($job->awaitingRecordedFailure ?? false) !== true) { $job->onUnlock = fn (bool $shouldSignal) => $this->onUnlock($shouldSignal); } } catch (\Throwable $throwable) { diff --git a/tests/Unit/ActivityTest.php b/tests/Unit/ActivityTest.php index 4b11504fa..494b50c15 100644 --- a/tests/Unit/ActivityTest.php +++ b/tests/Unit/ActivityTest.php @@ -6,6 +6,9 @@ use BadMethodCallException; use Exception; +use Illuminate\Contracts\Queue\Job as JobContract; +use Illuminate\Support\Facades\Cache; +use Mockery; use Tests\Fixtures\TestExceptionActivity; use Tests\Fixtures\TestInvalidActivity; use Tests\Fixtures\TestNonRetryableExceptionActivity; @@ -13,6 +16,7 @@ use Tests\Fixtures\TestOtherActivity; use Tests\Fixtures\TestWorkflow; use Tests\TestCase; +use Workflow\Activity; use Workflow\Exceptions\NonRetryableException; use Workflow\Models\StoredWorkflow; use Workflow\Serializers\Serializer; @@ -120,6 +124,52 @@ public function testFailedActivity(): void $this->assertSame(WorkflowCreatedStatus::class, $workflow->status()); } + public function testFailedJobMarksItsActivityBeforeReleasingIt(): void + { + $workflow = WorkflowStub::load(WorkflowStub::make(TestWorkflow::class)->id()); + $activity = new TestExceptionActivity(0, now()->toDateTimeString(), StoredWorkflow::findOrFail( + $workflow->id() + )); + $activity->setJob($this->queueJob('failed-job')); + + try { + $activity->handle(); + $this->fail('The activity should fail.'); + } catch (Exception $exception) { + $this->assertSame('failed', $exception->getMessage()); + } + + $this->assertSame('failed-job', Cache::get(Activity::failureKey($workflow->id(), 0))); + } + + public function testCopyDoesNotRunWhileAnotherJobsFailureIsUnrecorded(): void + { + $workflow = WorkflowStub::load(WorkflowStub::make(TestWorkflow::class)->id()); + Cache::forever(Activity::failureKey($workflow->id(), 0), 'failed-job'); + $activity = new TestExceptionActivity(0, now()->toDateTimeString(), StoredWorkflow::findOrFail( + $workflow->id() + )); + $activity->setJob($this->queueJob('copy-job')); + + $this->assertNull($activity->handle()); + $this->assertTrue($activity->awaitingRecordedFailure); + $this->assertSame(0, $workflow->exceptions()->count()); + $this->assertSame('failed-job', Cache::get(Activity::failureKey($workflow->id(), 0))); + } + + public function testRetriedJobRunsAndClearsItsOwnMark(): void + { + $workflow = WorkflowStub::load(WorkflowStub::make(TestWorkflow::class)->id()); + Cache::forever(Activity::failureKey($workflow->id(), 0), 'retried-job'); + $activity = new TestOtherActivity(0, now()->toDateTimeString(), StoredWorkflow::findOrFail($workflow->id()), [ + 'other', + ]); + $activity->setJob($this->queueJob('retried-job')); + + $this->assertSame(['other'], $activity->handle()); + $this->assertNull(Cache::get(Activity::failureKey($workflow->id(), 0))); + } + public function testActivityAlreadyComplete(): void { $workflow = WorkflowStub::load(WorkflowStub::make(TestWorkflow::class)->id()); @@ -179,4 +229,15 @@ public function testNullArgumentsDoNotShift(): void 'comparing' => 'hours', ], $result); } + + private function queueJob(string $uuid): JobContract + { + $job = Mockery::mock(JobContract::class); + $job->shouldReceive('uuid') + ->andReturn($uuid); + $job->shouldReceive('attempts') + ->andReturn(1); + + return $job; + } } diff --git a/tests/Unit/ExceptionTest.php b/tests/Unit/ExceptionTest.php index 9df287de2..7d1e0f801 100644 --- a/tests/Unit/ExceptionTest.php +++ b/tests/Unit/ExceptionTest.php @@ -7,6 +7,7 @@ use Exception as BaseException; use Illuminate\Contracts\Queue\Job as JobContract; use Illuminate\Support\Facades\Cache; +use Illuminate\Support\Facades\Queue; use InvalidArgumentException; use Mockery; use ReflectionMethod; @@ -22,6 +23,7 @@ use Tests\Fixtures\TestSagaParallelActivityWorkflow; use Tests\Fixtures\TestWorkflow; use Tests\TestCase; +use Workflow\Activity; use Workflow\Exception; use Workflow\Middleware\WithoutOverlappingMiddleware; use Workflow\Models\StoredWorkflow; @@ -76,6 +78,9 @@ public function testHandleResumesWorkflowWhenLogAlreadyExists(): void ->once() ->with('laravel-workflow-exception:123', 15) ->andReturn($lock); + Cache::shouldReceive('forget') + ->once() + ->with(Activity::failureKey(123, 0)); $workflow = Mockery::mock(); $workflow->shouldReceive('resume') @@ -104,6 +109,32 @@ public function testHandleResumesWorkflowWhenLogAlreadyExists(): void Mockery::close(); } + public function testRecordedFailureRemovesTheActivityMark(): void + { + Queue::fake(); + + $workflow = WorkflowStub::load(WorkflowStub::make(TestWorkflow::class)->id()); + $storedWorkflow = StoredWorkflow::findOrFail($workflow->id()); + $storedWorkflow->logs() + ->create([ + 'index' => 0, + 'now' => now() + ->toDateTimeString(), + 'class' => Exception::class, + 'result' => Serializer::serialize([ + 'class' => BaseException::class, + 'message' => 'recorded', + 'code' => 0, + ]), + ]); + Cache::forever(Activity::failureKey($storedWorkflow->id, 0), 'failed-job'); + + $exception = new Exception(0, now()->toDateTimeString(), $storedWorkflow, new BaseException('recorded')); + $exception->handle(); + + $this->assertNull(Cache::get(Activity::failureKey($storedWorkflow->id, 0))); + } + public function testHandleReleasesWhenExceptionLockUnavailable(): void { $workflow = WorkflowStub::load(WorkflowStub::make(TestWorkflow::class)->id()); @@ -341,10 +372,13 @@ public function testRetriesWriteWhenProbeMatchesAfterEarlierPendingWork(): void 'message' => 'child failed: child-2', 'code' => 0, ], sourceClass: TestProbeChildFailureWorkflow::class); + Cache::forever(Activity::failureKey($storedWorkflow->id, 1), 'failed-job'); + $exception->setJob($job); $exception->handle(); $this->assertFalse($storedWorkflow->fresh()->hasLogByIndex(1)); + $this->assertSame('failed-job', Cache::get(Activity::failureKey($storedWorkflow->id, 1))); } public function testPersistsWriteWhenProbeReachesCandidateException(): void From 34bf76c04d2bfbed48b10491d9555179a23fac7e Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 19 Nov 2026 17:53:17 +0000 Subject: [PATCH 3/3] Expire the failed-activity mark and skip copies before reporting them Review of #1245: a mark kept forever stranded a workflow whose Workflow\Exception job was lost (flushed queue, undecryptable payload), because every later copy of the activity skipped. The mark now expires after workflows.activity_failure_mark_seconds (DW_V1_ACTIVITY_FAILURE_MARK_SECONDS, default 3600); after that a wake runs the activity again, as before. The key is top-level so applications with an older published workflows.php receive the default. ActivityMiddleware now checks the mark before ActivityStarted, so a skipped copy fires no activity events. Adds the lost-Exception-job regression and unit coverage; documents that timeout and max-attempts-after-crash failures write no mark. Co-Authored-By: Claude Opus 5.5 --- .github/feature-test-timings.json | 4 +- CHANGELOG.md | 17 +++-- src/Activity.php | 54 ++++++++++---- src/Middleware/ActivityMiddleware.php | 12 ++-- src/config/workflows.php | 9 +++ .../Feature/FailedActivityRedispatchTest.php | 71 ++++++++++++++++++- tests/Unit/ActivityTest.php | 15 ++++ tests/Unit/Config/WorkflowsConfigTest.php | 1 + .../Middleware/ActivityMiddlewareTest.php | 28 ++++++++ 9 files changed, 184 insertions(+), 27 deletions(-) diff --git a/.github/feature-test-timings.json b/.github/feature-test-timings.json index 94a9d137b..08d3e80db 100644 --- a/.github/feature-test-timings.json +++ b/.github/feature-test-timings.json @@ -17,7 +17,7 @@ "tests/Feature/ExceptionLoggingReplayTest.php": 7.228751, "tests/Feature/ExceptionWorkflowTest.php": 8.416919, "tests/Feature/ExplicitQueueWorkerTest.php": 4.186096, - "tests/Feature/FailedActivityRedispatchTest.php": 6.5, + "tests/Feature/FailedActivityRedispatchTest.php": 13.0, "tests/Feature/FailingWorkflowTest.php": 6.256735, "tests/Feature/HeartbeatWorkflowTest.php": 13.477668, "tests/Feature/MigrationTest.php": 55.40979, @@ -271,7 +271,7 @@ "tests/Feature/ExceptionLoggingReplayTest.php": 8.584815, "tests/Feature/ExceptionWorkflowTest.php": 4.583109, "tests/Feature/ExplicitQueueWorkerTest.php": 0.87937, - "tests/Feature/FailedActivityRedispatchTest.php": 6.0, + "tests/Feature/FailedActivityRedispatchTest.php": 12.0, "tests/Feature/FailingWorkflowTest.php": 3.523026, "tests/Feature/HeartbeatWorkflowTest.php": 13.476006, "tests/Feature/MigrationTest.php": 12.426278, diff --git a/CHANGELOG.md b/CHANGELOG.md index d7886e03d..cba3a16e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,10 +64,19 @@ unique lock is released, so a wake in between (a signal, timer or sibling result) dispatched the activity again and the copy executed it, adding an exception row and repeating its side effects. The failed job now marks the - activity in the cache before its lock is released; a copy that finds the mark - completes without executing or recording a result, and the Exception job - removes the mark once it has recorded the failure or let it go. Recorded - history, job payloads and replay are unchanged. + activity in the cache before its lock is released. A copy that finds the mark + completes without executing, recording a result or firing activity events, and + the Exception job removes the mark once it has recorded the failure or let it + go. Recorded history, job payloads and replay are unchanged. + - The mark expires after `workflows.activity_failure_mark_seconds` + (`DW_V1_ACTIVITY_FAILURE_MARK_SECONDS`, default 3600). If the Exception job + is lost (a flushed queue, an undecryptable payload), a later wake then runs + the activity again, as before this release, instead of the workflow waiting + forever. Keep it well above the time an Exception job can wait behind + earlier pending work. + - A job that times out, or exhausts its attempts after a worker crash, fails + without running the activity's catch block, so it writes no mark and the + redispatch race stays open on those paths, as before. ## 2.33.0 - 2026-11-19 diff --git a/src/Activity.php b/src/Activity.php index cb629e0f4..b82d36cdc 100644 --- a/src/Activity.php +++ b/src/Activity.php @@ -45,11 +45,14 @@ class Activity implements ShouldBeEncrypted, ShouldBeUnique, ShouldQueue public $key = ''; /** - * @internal Set while handling, when another job of this activity failed - * and its outcome is not yet recorded. Unset, it is not serialized. + * @internal Whether another job of this activity failed and its outcome + * is not yet recorded, once read for this handling. Unset typed + * properties are not serialized, so payloads are unchanged. */ public bool $awaitingRecordedFailure; + private bool $ownsFailureMark; + private Container $container; public function __construct( @@ -117,24 +120,17 @@ public function handle() $this->container = App::make(Container::class); - // A failed job's outcome is recorded later by a queued Exception job, - // after this activity's unique lock is gone. A replay in between - // dispatches the activity again; that copy must not run it again. - $failedJob = Cache::get(self::failureKey($this->storedWorkflow->id, $this->index)); - - if ($this->storedWorkflow->hasLogByIndex($this->index)) { + if (self::awaitsRecordedFailure($this)) { return; } - if ($failedJob !== null && $failedJob !== $this->queueJobUuid()) { - $this->awaitingRecordedFailure = true; - + if ($this->storedWorkflow->hasLogByIndex($this->index)) { return; } // The Exception job forgets the failure only after recording it, so // logs loaded with this job's payload may predate that record. - if ($failedJob === null && $this->storedWorkflow->relationLoaded('logs') && $this->storedWorkflow->logs() + if ($this->storedWorkflow->relationLoaded('logs') && $this->storedWorkflow->logs() ->whereIndex($this->index) ->exists()) { return; @@ -145,7 +141,7 @@ public function handle() ...$this->resolveClassMethodDependencies($this->arguments, $this, 'execute') ); - if ($failedJob !== null) { + if ($this->ownsFailureMark ?? false) { Cache::forget(self::failureKey($this->storedWorkflow->id, $this->index)); } @@ -158,9 +154,14 @@ public function handle() ]); // Recorded before the job can fail and release its unique lock. + // It expires so that a lost Exception job cannot strand the run. $uuid = $this->queueJobUuid(); if ($uuid !== null) { - Cache::forever(self::failureKey($this->storedWorkflow->id, $this->index), $uuid); + Cache::put( + self::failureKey($this->storedWorkflow->id, $this->index), + $uuid, + self::failureMarkSeconds() + ); } if ($throwable instanceof NonRetryableExceptionContract) { @@ -171,6 +172,31 @@ public function handle() } } + /** + * @internal A failed job's outcome is recorded later by a queued Exception + * job, after this activity's unique lock is gone. A replay in between + * dispatches the activity again; that copy must not run it again. Read + * once per handling, before the activity is reported as started. + */ + public static function awaitsRecordedFailure(self $activity): bool + { + if (! isset($activity->awaitingRecordedFailure)) { + $failedJob = Cache::get(self::failureKey($activity->storedWorkflow->id, $activity->index)); + $activity->ownsFailureMark = $failedJob !== null && $failedJob === $activity->queueJobUuid(); + $activity->awaitingRecordedFailure = $failedJob !== null && ! $activity->ownsFailureMark; + } + + return $activity->awaitingRecordedFailure; + } + + /** + * @internal + */ + public static function failureMarkSeconds(): int + { + return max(1, (int) config('workflows.activity_failure_mark_seconds', 3600)); + } + /** * @internal */ diff --git a/src/Middleware/ActivityMiddleware.php b/src/Middleware/ActivityMiddleware.php index a6517a7e3..95424b966 100644 --- a/src/Middleware/ActivityMiddleware.php +++ b/src/Middleware/ActivityMiddleware.php @@ -7,6 +7,7 @@ use Illuminate\Support\Str; use LimitIterator; use SplFileObject; +use Workflow\Activity; use Workflow\Events\ActivityCompleted; use Workflow\Events\ActivityFailed; use Workflow\Events\ActivityStarted; @@ -22,6 +23,12 @@ final class ActivityMiddleware public function handle($job, $next): void { + // A copy of an activity whose failed job is not yet recorded neither + // runs nor reports anything; the Exception job records the outcome. + if ($job instanceof Activity && Activity::awaitsRecordedFailure($job)) { + return; + } + $this->job = $job; $this->uuid = (string) Str::uuid(); @@ -38,10 +45,7 @@ public function handle($job, $next): void try { $this->result = $next($job); - // A copy skipped for another job's unrecorded failure has no result. - if (($job->awaitingRecordedFailure ?? false) !== true) { - $job->onUnlock = fn (bool $shouldSignal) => $this->onUnlock($shouldSignal); - } + $job->onUnlock = fn (bool $shouldSignal) => $this->onUnlock($shouldSignal); } catch (\Throwable $throwable) { $file = new SplFileObject($throwable->getFile()); $iterator = new LimitIterator($file, max(0, $throwable->getLine() - 4), 7); diff --git a/src/config/workflows.php b/src/config/workflows.php index c35968c10..c1ebeecc1 100644 --- a/src/config/workflows.php +++ b/src/config/workflows.php @@ -47,6 +47,15 @@ // below this release requires draining the runs that recorded v2 first. 'definition_fingerprint_version' => env('DW_DEFINITION_FINGERPRINT_VERSION', 'v1'), + // Seconds a failed v1 activity stays marked while its queued + // Workflow\Exception job records the failure. A copy of the activity + // dispatched by a replay in that window does not run it again. The mark + // expires so that a lost Exception job (a flushed queue, an undecryptable + // payload) cannot leave the workflow waiting forever; a later wake then + // runs the activity again. A top-level key, so an application whose + // published workflows.php predates it still receives the default. + 'activity_failure_mark_seconds' => (int) env('DW_V1_ACTIVITY_FAILURE_MARK_SECONDS', 3600), + 'v1' => [ // Existing installations keep their v1 drain. Disable only for a // fresh v2-only application or after all legacy work is retired. diff --git a/tests/Feature/FailedActivityRedispatchTest.php b/tests/Feature/FailedActivityRedispatchTest.php index 28427d8fe..22959f34b 100644 --- a/tests/Feature/FailedActivityRedispatchTest.php +++ b/tests/Feature/FailedActivityRedispatchTest.php @@ -11,6 +11,7 @@ use Tests\Fixtures\TestFailedActivityRedispatchActivity; use Tests\Fixtures\TestFailedActivityRedispatchWorkflow; use Tests\TestCase; +use Workflow\Activity; use Workflow\Exception; use Workflow\States\WorkflowCompletedStatus; use Workflow\States\WorkflowWaitingStatus; @@ -24,6 +25,8 @@ */ final class FailedActivityRedispatchTest extends TestCase { + private const SHORT_MARK_SECONDS = 2; + public function testActivityRedispatchedBeforeItsFailureIsRecordedDoesNotRunAgain(): void { $queue = 'failed-activity-redispatch-' . Str::uuid(); @@ -93,17 +96,79 @@ public function testActivityRedispatchedBeforeItsFailureIsRecordedDoesNotRunAgai } } - private function work(string $queue): void + public function testLostExceptionJobLeavesTheActivityRunnableAfterTheMarkExpires(): void + { + $queue = 'lost-exception-job-' . Str::uuid(); + $directory = sys_get_temp_dir() . '/' . $queue; + mkdir($directory, 0700); + $redis = Queue::connection('redis')->getConnection(); + $key = Queue::connection('redis')->getQueue($queue); + $environment = [ + 'DW_V1_ACTIVITY_FAILURE_MARK_SECONDS' => (string) self::SHORT_MARK_SECONDS, + ]; + + try { + $workflow = WorkflowStub::make(TestFailedActivityRedispatchWorkflow::class); + $workflow->start($directory, new WorkflowOptions(connection: 'redis', queue: $queue)); + $this->work($queue, $environment); + $this->work($queue, $environment); + $this->assertSame(1, $this->executions($directory)); + $this->assertSame(1, $redis->llen($key), 'The Exception job is queued.'); + $mark = Activity::failureKey($workflow->id(), 0); + $this->assertNotNull(Cache::get($mark)); + + // The Exception job is lost (a flushed queue, an undecryptable payload). + $redis->del($key); + + // While the mark lasts, a wake's copy of the activity does not run. + $workflow->resume(); + $this->work($queue, $environment); + $this->assertSame(1, $redis->llen($key)); + $this->work($queue, $environment); + $this->assertSame(1, $this->executions($directory)); + $this->assertSame(WorkflowWaitingStatus::class, $workflow->fresh()->status()); + + // Once it expires, the next wake runs the activity again. + $deadline = microtime(true) + self::SHORT_MARK_SECONDS + 5; + while (Cache::get($mark) !== null) { + if (microtime(true) > $deadline) { + $this->fail('The failure mark did not expire.'); + } + usleep(100000); + } + $workflow->resume(); + $this->drain($queue, $workflow, $environment); + + $this->assertSame(WorkflowCompletedStatus::class, $workflow->status()); + $this->assertSame('caught: activity failed', $workflow->output()); + $this->assertSame(2, $this->executions($directory)); + $this->assertSame([Exception::class], $workflow->logs()->pluck('class')->all()); + } finally { + foreach (glob($directory . '/*') ?: [] as $file) { + unlink($file); + } + rmdir($directory); + } + } + + /** + * @param array $environment + */ + private function work(string $queue, array $environment = []): void { $process = new Process(['php', __DIR__ . '/../../vendor/bin/testbench', 'queue:work', 'redis', '--queue=' . $queue, '--once', '--sleep=0'], env: [ 'WORKFLOW_WATCHDOG_ENABLED' => 'false', + ...$environment, ]); $process->setTimeout(30); $process->mustRun(); } - private function drain(string $queue, WorkflowStub $workflow): void + /** + * @param array $environment + */ + private function drain(string $queue, WorkflowStub $workflow, array $environment = []): void { $deadline = microtime(true) + 30; @@ -112,7 +177,7 @@ private function drain(string $queue, WorkflowStub $workflow): void $this->fail('The workflow did not finish after its failure was recorded.'); } - $this->work($queue); + $this->work($queue, $environment); usleep(100000); } } diff --git a/tests/Unit/ActivityTest.php b/tests/Unit/ActivityTest.php index 494b50c15..a87785106 100644 --- a/tests/Unit/ActivityTest.php +++ b/tests/Unit/ActivityTest.php @@ -170,6 +170,21 @@ public function testRetriedJobRunsAndClearsItsOwnMark(): void $this->assertNull(Cache::get(Activity::failureKey($workflow->id(), 0))); } + public function testFailureMarkExpiresAfterTheConfiguredSeconds(): void + { + $this->assertSame(3600, Activity::failureMarkSeconds()); + + config([ + 'workflows.activity_failure_mark_seconds' => 120, + ]); + $this->assertSame(120, Activity::failureMarkSeconds()); + + config([ + 'workflows.activity_failure_mark_seconds' => 0, + ]); + $this->assertSame(1, Activity::failureMarkSeconds()); + } + public function testActivityAlreadyComplete(): void { $workflow = WorkflowStub::load(WorkflowStub::make(TestWorkflow::class)->id()); diff --git a/tests/Unit/Config/WorkflowsConfigTest.php b/tests/Unit/Config/WorkflowsConfigTest.php index 9b472842e..06d53a870 100644 --- a/tests/Unit/Config/WorkflowsConfigTest.php +++ b/tests/Unit/Config/WorkflowsConfigTest.php @@ -27,6 +27,7 @@ final class WorkflowsConfigTest extends NonDatabaseTestCase 'DW_V2_COMPATIBILITY_HEARTBEAT_TTL', 'WORKFLOW_V2_COMPATIBILITY_HEARTBEAT_TTL', 'DW_V2_PIN_TO_RECORDED_FINGERPRINT', 'WORKFLOW_V2_PIN_TO_RECORDED_FINGERPRINT', 'DW_DEFINITION_FINGERPRINT_VERSION', + 'DW_V1_ACTIVITY_FAILURE_MARK_SECONDS', 'DW_V2_CONTINUE_AS_NEW_EVENT_THRESHOLD', 'WORKFLOW_V2_CONTINUE_AS_NEW_EVENT_THRESHOLD', 'DW_V2_CONTINUE_AS_NEW_SIZE_BYTES_THRESHOLD', 'WORKFLOW_V2_CONTINUE_AS_NEW_SIZE_BYTES_THRESHOLD', 'DW_V2_HISTORY_EXPORT_SIGNING_KEY', 'WORKFLOW_V2_HISTORY_EXPORT_SIGNING_KEY', diff --git a/tests/Unit/Middleware/ActivityMiddlewareTest.php b/tests/Unit/Middleware/ActivityMiddlewareTest.php index b16ba7300..fd698db00 100644 --- a/tests/Unit/Middleware/ActivityMiddlewareTest.php +++ b/tests/Unit/Middleware/ActivityMiddlewareTest.php @@ -14,6 +14,7 @@ use Tests\Fixtures\TestModelNotFoundWorkflow; use Tests\Fixtures\TestWorkflow; use Tests\TestCase; +use Workflow\Activity; use Workflow\Events\ActivityCompleted; use Workflow\Events\ActivityFailed; use Workflow\Events\ActivityStarted; @@ -68,6 +69,33 @@ public function testMiddleware(): void Queue::assertPushed(TestWorkflow::class, 2); } + public function testCopyAwaitingAnotherJobsRecordedFailureNeitherRunsNorReports(): void + { + Event::fake(); + Queue::fake(); + + $workflow = WorkflowStub::make(TestWorkflow::class); + $workflow->start(); + $storedWorkflow = StoredWorkflow::findOrFail($workflow->id()); + Cache::forever(Activity::failureKey($storedWorkflow->id, 0), 'failed-job'); + + $activity = $this->mock(TestActivity::class); + $activity->index = 0; + $activity->now = now() + ->toDateTimeString(); + $activity->storedWorkflow = $storedWorkflow; + + (new ActivityMiddleware())->handle($activity, function (): void { + $this->fail('The copy must not run.'); + }); + + $this->assertTrue($activity->awaitingRecordedFailure); + $this->assertFalse(isset($activity->onUnlock)); + Event::assertNotDispatched(ActivityStarted::class); + Event::assertNotDispatched(ActivityCompleted::class); + Event::assertNotDispatched(ActivityFailed::class); + } + public function testAlreadyCompleted(): void { Event::fake();