From 5d7b09816d09b09334ea40a3b3832abd64b86b51 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 20 Nov 2026 05:45:05 +0000 Subject: [PATCH 1/5] Keep a managed Worker running through a transient Server or network outage Fixes #448. A Worker process died when native Server answered a workflow poll with 503 {"reason":"storage_unavailable"} while its database warmed up: pollFailurePolicy() retried only typed refusals, and Worker::run() rethrew everything except a stale registration. - Worker\TransientFailure classifies untyped transients: a connection that is refused, reset or timed out; an upstream 502/503/504 with no body; a 503 storage_unavailable or backend_unavailable; and any 5xx or 429 with retryable: true. Never retryable: false, auth failures or other 4xx. - Polls retry them and keep the poll identity (an outage committed nothing), using the shared jittered schedule: 0.25 s doubling to a 4 s cap, full jitter. Typed refusals keep their protocol backoff and identity rules. - Worker::run() absorbs a transient failure that escapes a tick (a completion, for example) with the same schedule, logging worker.transient_failure, and keeps heartbeating while it waits. Fatal failures log worker.fatal_failure with status and reason before run() rethrows. - Tests use a fake transport: 503s then recovery leaves the Worker running; a failure escaping a tick is absorbed; 401, 403, 400 and retryable: false stop the Worker with one fatal diagnostic; classification and jitter bounds. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 11 ++ docs/portal/build/workers.md | 2 + src/Worker.php | 36 ++++- src/Worker/TransientFailure.php | 71 +++++++++ tests/WorkerTransientOutageTest.php | 214 ++++++++++++++++++++++++++++ 5 files changed, 329 insertions(+), 5 deletions(-) create mode 100644 src/Worker/TransientFailure.php create mode 100644 tests/WorkerTransientOutageTest.php diff --git a/CHANGELOG.md b/CHANGELOG.md index b4c4f518..72026408 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,17 @@ project follows [Semantic Versioning](https://semver.org/). ### Fixed +- A managed Worker no longer exits on a transient Server or network outage + (#448). For example, native Server answers `503 storage_unavailable` while + its database warms up. Polls retry untyped transients with the poll's + identity kept, and `Worker::run()` absorbs a transient failure that escapes a + tick (a completion, for example), using the shared jittered schedule (0.25 s + doubling to a 4 s cap). Authentication, authorization, protocol refusals and + explicit `retryable: false` still stop the Worker, after a + `worker.fatal_failure` diagnostic. + +### Fixed + - An activity worker no longer drops a claimed task when the lease renewal at activity start is slow or fails transiently (#445). The worker keeps working under the poll's original lease, charged conservatively from the poll's own diff --git a/docs/portal/build/workers.md b/docs/portal/build/workers.md index 2cc9c8bb..28b35b81 100644 --- a/docs/portal/build/workers.md +++ b/docs/portal/build/workers.md @@ -97,6 +97,8 @@ An activity option such as `['queue' => 'payments-v2']` selects the queue of a n `Worker::run()` registers, negotiates heartbeat cadence, polls workflow/activity/query tasks, renews workflow-task leases, and reports terminal outcomes. Typed transient poll pressure is retried with capped backoff while registration heartbeats and shutdown checks remain responsive. +An untyped outage does not stop the Worker either. That covers a connection that is refused, reset or timed out; a 502/503/504 with no protocol body; a 503 `storage_unavailable` or `backend_unavailable`, for example while the Server's database warms up; and any 5xx or 429 with `retryable: true`. The Worker retries using the schedule all SDKs share: 0.25 s doubling to a 4 s cap, with full random jitter. This applies to polls, and to any request that fails a tick, such as a completion; the Server re-delivers a claimed task when its lease ends. Authentication and authorization failures, other protocol refusals, and explicit `retryable: false` stay fatal. They are reported as a `worker.fatal_failure` diagnostic before `run()` rethrows. + ### Activity continuations Activity continuations are an explicit constructor opt-in: diff --git a/src/Worker.php b/src/Worker.php index da715db1..d073e8a9 100644 --- a/src/Worker.php +++ b/src/Worker.php @@ -22,6 +22,7 @@ use DurableWorkflow\Worker\ActivityLeaseProcess; use DurableWorkflow\Worker\ActivityLeaseAuthority; use DurableWorkflow\Worker\ActivityLeaseMaintenanceException; +use DurableWorkflow\Worker\TransientFailure; use DurableWorkflow\Worker\ActivityExecutionFailure; use DurableWorkflow\Worker\CapabilityManifest; use DurableWorkflow\Worker\CancellationHistory; @@ -484,6 +485,7 @@ public function run(int $pollTimeoutSeconds = 5): void return; } $this->establishRegistration($registration); + $transientAttempt = 0; while (!$this->shutdownRequested || $this->staleRegistrationObserved) { try { if ($this->completionPipeline !== null) { @@ -492,11 +494,28 @@ public function run(int $pollTimeoutSeconds = 5): void $this->tick($pollTimeoutSeconds); } $this->heartbeatIfDue(); + $transientAttempt = 0; } catch (Throwable $exception) { - if (!$this->recoversFromStaleRegistration($exception)) { + if ($this->recoversFromStaleRegistration($exception)) { + $this->staleRegistrationObserved = true; + } elseif (TransientFailure::is($exception)) { + // The Server or network was briefly unavailable (for + // example a database warming up). Keep the Worker + // running; any claimed task is re-delivered by the + // Server when its lease ends. + $delay = TransientFailure::delay(++$transientAttempt); + $this->diagnostic('worker.transient_failure', ['worker_id' => $this->workerId, + 'task_queue' => $this->taskQueue, 'attempt' => $transientAttempt, + 'delay_seconds' => $delay, 'exception' => $exception], 'warning'); + $this->waitForTransientRetry($delay); + } else { + $this->diagnostic('worker.fatal_failure', ['worker_id' => $this->workerId, + 'task_queue' => $this->taskQueue, + 'status' => $exception instanceof ServerException ? $exception->status : null, + 'reason' => $exception instanceof ServerException ? $exception->reason : null, + 'exception' => $exception], 'error'); throw $exception; } - $this->staleRegistrationObserved = true; } if ($this->staleRegistrationObserved && !$this->recoverStaleRegistration()) { break; @@ -1422,7 +1441,12 @@ private function isPollCapacityRefusal(ServerException $error): bool private function waitForPollRetry(string $kind, int $attempt, ServerException $error): void { - $delay = $this->transientRetryDelay($attempt, $error->details['retry_after_seconds'] ?? null); + $retryAfter = $error->details['retry_after_seconds'] ?? null; + // Typed refusals keep their protocol backoff; an untyped outage uses + // the shared jittered schedule so Workers do not retry in step. + $delay = !PollResponse::isTransientFailure($error) && TransientFailure::is($error) + ? max(TransientFailure::delay($attempt), is_int($retryAfter) ? (float) $retryAfter : 0.0) + : $this->transientRetryDelay($attempt, $retryAfter); ($this->transientPollRetryObserver)?->__invoke($kind, $attempt, $delay, $error); $this->diagnostic('worker.retrying', ['worker_id' => $this->workerId, 'operation' => "{$kind}_poll", 'attempt' => $attempt, 'delay_seconds' => $delay, 'exception' => $error], 'warning'); @@ -1439,12 +1463,14 @@ private function pollFailurePolicy(string $kind, string $requestId, ServerExcept 'backend_unavailable' => $database, 'storage_pressure', 'storage_admission_unavailable' => $storage, 'workflow_poll_normalization_pending' => $normalization, - default => PollResponse::isTransientFailure($error), + default => PollResponse::isTransientFailure($error) || TransientFailure::is($error), }; return ['retry' => $retry, 'retain' => $database || $storage || $normalization || $error->isTransientConnectionFailure() || $error->isTransientUpstreamFailure() - || PollResponse::isTransientPayloadFailure($error)]; + || PollResponse::isTransientPayloadFailure($error) + // An untyped outage committed nothing; a typed refusal keeps its own identity rule. + || (!PollResponse::isTransientFailure($error) && TransientFailure::is($error))]; } /** diff --git a/src/Worker/TransientFailure.php b/src/Worker/TransientFailure.php new file mode 100644 index 00000000..f8371995 --- /dev/null +++ b/src/Worker/TransientFailure.php @@ -0,0 +1,71 @@ +isTransientConnectionFailure() || $error->isTransientUpstreamFailure()) { + return true; + } + + return self::transientResponse($error->status, $error->reason, $error->details); + } + if ($error instanceof TransportException) { + if ($error->transientConnectionFailure) { + return true; + } + $reason = is_string($error->response['reason'] ?? null) ? $error->response['reason'] : null; + + return $error->status !== null && self::transientResponse($error->status, $reason, $error->response); + } + + return false; + } + + /** Full-jitter delay for the given 1-based attempt. */ + public static function delay(int $attempt): float + { + $ceiling = min(self::MAX_SECONDS, self::FIRST_SECONDS * 2 ** min(max(0, $attempt - 1), 16)); + + return $ceiling * random_int(0, 1_000_000) / 1_000_000; + } + + /** @param array|null $details */ + private static function transientResponse(int $status, ?string $reason, ?array $details): bool + { + $retryable = is_array($details) ? ($details['retryable'] ?? null) : null; + if ($retryable === false || !($status === 429 || ($status >= 500 && $status <= 599))) { + return false; + } + + return $retryable === true || ($status === 503 && in_array($reason, self::REASONS, true)); + } +} diff --git a/tests/WorkerTransientOutageTest.php b/tests/WorkerTransientOutageTest.php new file mode 100644 index 00000000..dbfd8169 --- /dev/null +++ b/tests/WorkerTransientOutageTest.php @@ -0,0 +1,214 @@ + 'storage_unavailable', 'message' => 'storage_unavailable']; + + return TransportException::fromResponse(503, $body, json_encode($body, JSON_THROW_ON_ERROR)); + } + + /** @return array{0: Worker, 1: list, 2: object} */ + private function worker(\Closure $handler): array + { + $now = 0.0; + $events = []; + $state = new class { public float $slept = 0.0; public int $calls = 0; }; + $worker = new Worker( + new Client('https://server.example', transport: new FakeTransport(handler: $handler)), + 'orders', + workerId: 'worker-1', + clock: static function () use (&$now): float { + return $now; + }, + sleeper: static function (int $microseconds) use (&$now, $state): void { + $now += $microseconds / 1_000_000; + $state->slept += $microseconds / 1_000_000; + }, + diagnosticListener: static function (string $event, array $context) use (&$events): void { + $events[] = $event; + }, + ); + $worker->registerActivity('orders.charge', static function (ActivityContext $context) use ($state): string { + ++$state->calls; + + return 'charged'; + }); + + return [$worker, $events, $state]; + } + + public function testStorageUnavailablePollsAreRetriedWithJitterAndTheWorkerKeepsRunning(): void + { + $workflowPolls = 0; + $completions = 0; + $events = []; + $state = null; + $handler = static function (string $method, string $uri, array $headers, ?array $body) use (&$workflowPolls, &$completions): ?array { + if (str_ends_with($uri, '/api/worker/register')) { + return ['registered' => true]; + } + if (str_ends_with($uri, '/api/worker/workflow-tasks/poll')) { + if (++$workflowPolls <= 3) { + throw self::storageUnavailable(); + } + + return ['task' => null, 'poll_status' => 'empty']; + } + if (str_ends_with($uri, '/api/worker/activity-tasks/poll')) { + return ['poll_status' => 'leased', 'task' => ['task_id' => 'activity-1', 'activity_attempt_id' => 'attempt-1', + 'lease_owner' => 'worker-1', 'activity_type' => 'orders.charge', 'payload_codec' => 'avro']]; + } + if (str_ends_with($uri, '/api/worker/activity-tasks/activity-1/complete')) { + ++$completions; + + return ['completed' => true]; + } + if (str_ends_with($uri, '/api/worker/query-tasks/poll')) { + return ['task' => null, 'poll_status' => 'stopped', 'reason' => 'worker_stopped']; + } + self::fail("Unexpected worker request: {$method} {$uri}"); + }; + [$worker, , $state] = $this->worker($handler); + + $worker->run(0); + + self::assertSame(4, $workflowPolls, 'Three 503 storage_unavailable answers, then a poll that succeeds.'); + self::assertSame(1, $state->calls); + self::assertSame(1, $completions); + // Jittered: each wait is at most 0.25, 0.5 and 1 s. + self::assertLessThanOrEqual(1.75, $state->slept); + } + + public function testATransientFailureThatEscapesATickDoesNotStopTheWorker(): void + { + // The activity's completion and its failure report both meet the + // outage, so the error leaves the tick; the Worker waits and polls on. + $workflowPolls = 0; + $activityPolls = 0; + $reports = []; + $handler = static function (string $method, string $uri, array $headers, ?array $body) use (&$workflowPolls, &$activityPolls, &$reports): ?array { + if (str_ends_with($uri, '/api/worker/register')) { + return ['registered' => true]; + } + if (str_ends_with($uri, '/api/worker/workflow-tasks/poll')) { + ++$workflowPolls; + + return ['task' => null, 'poll_status' => 'empty']; + } + if (str_ends_with($uri, '/api/worker/activity-tasks/poll')) { + if (++$activityPolls === 1) { + return ['poll_status' => 'leased', 'task' => ['task_id' => 'activity-1', 'activity_attempt_id' => 'attempt-1', + 'lease_owner' => 'worker-1', 'activity_type' => 'orders.charge', 'payload_codec' => 'avro']]; + } + + return ['task' => null, 'poll_status' => 'empty']; + } + if (str_contains($uri, '/api/worker/activity-tasks/activity-1/')) { + $reports[] = substr($uri, strrpos($uri, '/') + 1); + throw self::storageUnavailable(); + } + if (str_ends_with($uri, '/api/worker/query-tasks/poll')) { + return $workflowPolls >= 2 ? ['task' => null, 'poll_status' => 'stopped', 'reason' => 'worker_stopped'] + : ['task' => null, 'poll_status' => 'empty']; + } + if (str_ends_with($uri, '/api/worker/heartbeat')) { + return ['worker_id' => 'worker-1', 'acknowledged' => true]; + } + self::fail("Unexpected worker request: {$method} {$uri}"); + }; + $events = []; + $now = 0.0; + $worker = new Worker(new Client('https://server.example', transport: new FakeTransport(handler: $handler)), 'orders', + workerId: 'worker-1', + clock: static function () use (&$now): float { return $now; }, + sleeper: static function (int $microseconds) use (&$now): void { $now += $microseconds / 1_000_000; }, + diagnosticListener: static function (string $event, array $context) use (&$events): void { $events[] = $event; }); + $worker->registerActivity('orders.charge', static fn (ActivityContext $context): string => 'charged'); + + $worker->run(0); + + self::assertContains('complete', $reports); + self::assertGreaterThanOrEqual(2, $workflowPolls, 'The Worker kept polling after the outage.'); + self::assertNotContains('worker.fatal_failure', $events); + self::assertContains('worker.transient_failure', $events, 'The outage left the tick and was absorbed by the run loop.'); + } + + /** @return iterable */ + public static function fatalFailures(): iterable + { + yield 'authentication' => [TransportException::fromResponse(401, ['reason' => 'authentication_failed'], '{}'), 401]; + yield 'authorization' => [TransportException::fromResponse(403, ['reason' => 'forbidden'], '{}'), 403]; + yield 'explicitly not retryable' => [TransportException::fromResponse(503, + ['reason' => 'storage_unavailable', 'retryable' => false], '{}'), 503]; + yield 'protocol refusal' => [TransportException::fromResponse(400, ['reason' => 'unsupported_protocol_version'], '{}'), 400]; + } + + #[DataProvider('fatalFailures')] + public function testAuthenticationAndProtocolFailuresStopTheWorkerAndAreLogged(TransportException $failure, int $status): void + { + $handler = static function (string $method, string $uri) use ($failure): ?array { + if (str_ends_with($uri, '/api/worker/register')) { + return ['registered' => true]; + } + if (str_ends_with($uri, '/api/worker/deregister') || str_contains($uri, '/deregister')) { + return ['deregistered' => true]; + } + throw $failure; + }; + $events = []; + $worker = new Worker(new Client('https://server.example', transport: new FakeTransport(handler: $handler)), 'orders', + workerId: 'worker-1', sleeper: static function (int $microseconds): void {}, + diagnosticListener: static function (string $event, array $context) use (&$events): void { + $events[] = [$event, $context['status'] ?? null, $context['reason'] ?? null]; + }); + + try { + $worker->run(0); + self::fail('A fatal failure must stop the Worker.'); + } catch (ServerException $error) { + self::assertSame($status, $error->status); + } + $fatal = array_values(array_filter($events, static fn (array $event): bool => $event[0] === 'worker.fatal_failure')); + self::assertCount(1, $fatal, json_encode($events)); + self::assertSame($status, $fatal[0][1]); + self::assertSame([], array_values(array_filter($events, static fn (array $event): bool => $event[0] === 'worker.transient_failure'))); + } + + public function testClassificationAndJitterBounds(): void + { + self::assertTrue(TransientFailure::is(self::storageUnavailable())); + self::assertTrue(TransientFailure::is(new ServerException('x', 503, 'backend_unavailable', ['reason' => 'backend_unavailable']))); + self::assertTrue(TransientFailure::is(new ServerException('x', 502, 'anything', ['retryable' => true]))); + self::assertTrue(TransientFailure::is(new TransportException('reset', transientConnectionFailure: true))); + self::assertFalse(TransientFailure::is(new ServerException('x', 503, 'storage_unavailable', ['retryable' => false]))); + self::assertFalse(TransientFailure::is(new ServerException('x', 503, 'backend_lock_pressure', ['reason' => 'backend_lock_pressure']))); + self::assertFalse(TransientFailure::is(new ServerException('x', 401, 'authentication_failed'))); + self::assertFalse(TransientFailure::is(new ServerException('x', 409, 'stale_attempt'))); + self::assertFalse(TransientFailure::is(new \RuntimeException('boom'))); + foreach ([1 => 0.25, 2 => 0.5, 3 => 1.0, 4 => 2.0, 5 => 4.0, 9 => 4.0] as $attempt => $cap) { + for ($i = 0; $i < 50; ++$i) { + $delay = TransientFailure::delay($attempt); + self::assertGreaterThanOrEqual(0.0, $delay); + self::assertLessThanOrEqual($cap, $delay); + } + } + } +} From 6c6e68f184a311ae02e98fd4ea713aed728e988b Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 20 Nov 2026 07:08:49 +0000 Subject: [PATCH 2/5] Retry a completion through an outage; never report an unpublished result Coordinator decisions on #448: - A completion that meets an untyped outage (503 storage_unavailable or backend_unavailable) is retried with the same attempt fence on the shared jittered schedule, like the existing connection-failure retry. With activity lease maintenance the publication fence ends retries at the grant's end; without it, eight attempts do (worker.outcome_retry_exhausted). - Idempotent: verified against published Server 3.17.0. The first completion committed, the Worker saw a 503, and the retry was answered 409 stale_attempt, recorded false, attempt completed. The existing isCommittedActivityCompletion() rule now applies to it, so the run completed with one ActivityCompleted and the activity ran once. - A computed result whose publication an outage defeats is logged (worker.outcome_unpublished) and left to the Server's lease handling, never sent as an activity failure report. - An unclassified retryable answer still never authorizes repeating an acknowledgement, and typed refusals keep their own protocol checks, so an invalid typed retry_after cannot stall the Worker. Server-suggested delays for untyped outages are capped at 60 s. - Docs: completion retry, and the cross-SDK difference (PHP and Rust stop on 401/403 after an error; Python retries polls so credentials can rotate). Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 ++ docs/portal/build/workers.md | 4 + src/Worker.php | 50 ++++++++++- src/Worker/TransientFailure.php | 33 ++++--- tests/WorkerTransientOutageTest.php | 135 ++++++++++++++++++++++++++-- 5 files changed, 206 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72026408..90721d25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,12 @@ project follows [Semantic Versioning](https://semver.org/). doubling to a 4 s cap). Authentication, authorization, protocol refusals and explicit `retryable: false` still stop the Worker, after a `worker.fatal_failure` diagnostic. +- A completion that meets an untyped outage is retried with the same attempt + fence, inside the claim's lease, instead of being reported as an activity + failure. A duplicate of a completion the Server already committed (Server + 3.17.0: `409 stale_attempt`, `recorded: false`, attempt `completed`) counts as + committed. A result that cannot be published in time is logged + (`worker.outcome_unpublished`) and left to the Server's lease handling. ### Fixed diff --git a/docs/portal/build/workers.md b/docs/portal/build/workers.md index 28b35b81..d549f040 100644 --- a/docs/portal/build/workers.md +++ b/docs/portal/build/workers.md @@ -99,6 +99,10 @@ An activity option such as `['queue' => 'payments-v2']` selects the queue of a n An untyped outage does not stop the Worker either. That covers a connection that is refused, reset or timed out; a 502/503/504 with no protocol body; a 503 `storage_unavailable` or `backend_unavailable`, for example while the Server's database warms up; and any 5xx or 429 with `retryable: true`. The Worker retries using the schedule all SDKs share: 0.25 s doubling to a 4 s cap, with full random jitter. This applies to polls, and to any request that fails a tick, such as a completion; the Server re-delivers a claimed task when its lease ends. Authentication and authorization failures, other protocol refusals, and explicit `retryable: false` stay fatal. They are reported as a `worker.fatal_failure` diagnostic before `run()` rethrows. +**Completions.** A completion that meets an outage is retried with the same attempt fence on that schedule. With activity lease maintenance, the retries stop when the claim's grant ends; without it, they stop after a bounded number of attempts. Retrying is safe: the Server records one outcome per attempt. A retry of a completion the Server already committed is answered `409 stale_attempt` with `recorded: false` and the attempt `completed`, and the Worker treats that as its own committed completion. A result that cannot be published before the retries end is logged as `worker.outcome_unpublished` and left to the Server, which re-delivers the attempt when its lease ends. It is never reported as an activity failure. + +**Across SDKs.** PHP and Rust Workers stop on a 401 or 403 after logging it as an error. Python Workers keep retrying polls through a 401 or 403 so that credentials can be rotated without a restart, and log every refusal at error level. + ### Activity continuations Activity continuations are an explicit constructor opt-in: diff --git a/src/Worker.php b/src/Worker.php index d073e8a9..1c41e993 100644 --- a/src/Worker.php +++ b/src/Worker.php @@ -74,6 +74,8 @@ final class Worker { private const DEFAULT_HEARTBEAT_INTERVAL_SECONDS = 30; + /** Untyped-outage retries of one outcome without a maintained lease: up to about 15 s of jittered waits. */ + private const UNTYPED_OUTAGE_MAX_ATTEMPTS = 8; /** A release must land this long before the claim's grant ends. */ private const RELEASE_MARGIN_SECONDS = 0.5; private const WORKFLOW_HISTORY_PAGE_SIZE = 500; @@ -1445,7 +1447,7 @@ private function waitForPollRetry(string $kind, int $attempt, ServerException $e // Typed refusals keep their protocol backoff; an untyped outage uses // the shared jittered schedule so Workers do not retry in step. $delay = !PollResponse::isTransientFailure($error) && TransientFailure::is($error) - ? max(TransientFailure::delay($attempt), is_int($retryAfter) ? (float) $retryAfter : 0.0) + ? TransientFailure::delay($attempt, $retryAfter) : $this->transientRetryDelay($attempt, $retryAfter); ($this->transientPollRetryObserver)?->__invoke($kind, $attempt, $delay, $error); $this->diagnostic('worker.retrying', ['worker_id' => $this->workerId, 'operation' => "{$kind}_poll", @@ -1533,9 +1535,15 @@ private function retryStorageAdmission( $backendUnavailable = $workflowBackendUnavailable || $activityBackendUnavailable; // These requests retain their original lease fence and computed // outcome. An uncertain reply retries the request, never the handler. + // An untyped outage (#448), such as 503 storage_unavailable, is + // just as uncertain: the Server may have committed the request. + $untypedOutage = !$exception->isStorageAdmissionFailure() && !$backendUnavailable + && !$exception->isTransientConnectionFailure() && !$exception->isTransientUpstreamFailure() + && TransientFailure::is($exception, acknowledgement: true); $claimedTaskTransportFailure = ($workflowTaskLease !== null || ($operation === 'activity_complete' && $activityTaskLease !== null)) - && ($exception->isTransientConnectionFailure() || $exception->isTransientUpstreamFailure()); + && ($exception->isTransientConnectionFailure() || $exception->isTransientUpstreamFailure() + || $untypedOutage); if ((!$exception->isStorageAdmissionFailure() && !$backendUnavailable && !$activityDiscoveryUnavailable && !$workflowCompletionDeferred && !$claimedTaskTransportFailure) || $this->shutdownRequested) { @@ -1546,9 +1554,25 @@ private function retryStorageAdmission( } $uncertainActivityCompletion = $uncertainActivityCompletion || $activityBackendUnavailable || ($operation === 'activity_complete' && $claimedTaskTransportFailure); + if ($untypedOutage && $attempt >= $this->untypedOutageAttemptLimit($operation)) { + // Retries stay inside the claim: with lease maintenance the + // publication fence ends them when the grant ends; without + // it, a bounded number of attempts does. The Server then + // settles the attempt (lease expiry and re-delivery). + $this->diagnostic('worker.outcome_retry_exhausted', ['worker_id' => $this->workerId, + 'operation' => $operation, 'attempts' => $attempt + 1, 'exception' => $exception], 'error'); + throw $exception; + } ++$attempt; - $delaySeconds = $this->transientRetryDelay($attempt, $exception->details['retry_after_seconds'] ?? null); + $retryAfter = $exception->details['retry_after_seconds'] ?? null; + // An untyped outage uses the shared jittered schedule; typed + // refusals keep their protocol backoff. The lease fence still + // bounds every retry: once the claim's grant ends, publication + // is refused and the Server settles the attempt. + $delaySeconds = $untypedOutage + ? TransientFailure::delay($attempt, $retryAfter) + : $this->transientRetryDelay($attempt, $retryAfter); if ($this->transientPollRetryObserver !== null) { ($this->transientPollRetryObserver)($operation, $attempt, $delaySeconds, $exception); } @@ -1571,6 +1595,14 @@ private function retryStorageAdmission( } } + /** Retries of one outcome through an untyped outage before the Server's lease handling takes over. */ + private function untypedOutageAttemptLimit(string $operation): int + { + // A maintained activity lease fences publication at the grant's end. + return str_starts_with($operation, 'activity_') && $this->activityLeaseProcess !== null + ? PHP_INT_MAX : self::UNTYPED_OUTAGE_MAX_ATTEMPTS; + } + /** @param array{task_id: string, activity_attempt_id: string, lease_owner: string} $lease */ private function isCommittedActivityCompletion(ServerException $exception, array $lease): bool { @@ -3402,6 +3434,7 @@ private function executeActivityTask(array $task): void $activityType = (string) ($task['activity_type'] ?? ''); $callbackStopped = false; $callbackStarted = false; + $resultComputed = false; try { $this->trackWorkerSessionFromTask($task); $handler = $this->activities[$activityType] ?? null; @@ -3474,6 +3507,7 @@ static function () use (&$callbackStopped): void { $callbackStopped = true; }, $result = $this->invokeActivity($handler, $context, $arguments, ExecutionTelemetry::taskFields($task, $this->client->namespace, $this->taskQueue), authorizedTask: $task); } + $resultComputed = true; $carried = $this->reserveWorkflowPoll($leaseOwner); $continuation = $carried === null ? null : $this->speculateActivityContinuation($task, $result); $completion = $this->retryStorageAdmission( @@ -3539,6 +3573,16 @@ function () use ($taskId, $attemptId, $leaseOwner, $result, $carried, &$continua if ($exception instanceof ServerException && $this->discardClosedActivityAttempt($task, $exception)) { return; } + if ($resultComputed && TransientFailure::is($exception, acknowledgement: true)) { + // An outage outlasted publishing a computed result. It is not + // an activity failure: leave the attempt to the Server, which + // re-delivers it when the lease ends (a committed completion + // is answered as a duplicate then). + $this->diagnostic('worker.outcome_unpublished', ['task_id' => $taskId, + 'activity_attempt_id' => $attemptId, 'task_kind' => 'activity', 'exception' => $exception], 'error'); + + return; + } if ($this->activityLeaseProcess !== null && !$exception instanceof WorkflowClaimRevoked) { try { $this->activityLeaseProcess->check(); diff --git a/src/Worker/TransientFailure.php b/src/Worker/TransientFailure.php index f8371995..4e91cd1b 100644 --- a/src/Worker/TransientFailure.php +++ b/src/Worker/TransientFailure.php @@ -17,10 +17,15 @@ * - an upstream 502/503/504 with no protocol body; * - a 503 whose reason is `storage_unavailable` or `backend_unavailable` * (for example native Server while its database warms up); - * - any 5xx or 429 whose body says `retryable: true`. + * - any 5xx or 429 whose body says `retryable: true` and names no other + * reason. This one does not apply to acknowledgements (completions), + * because an unclassified answer cannot authorize repeating a request the + * Server may have committed. * * An explicit `retryable: false`, authentication and authorization failures, - * and other protocol refusals are never transient. + * and other protocol refusals are never transient. A typed refusal (one that + * names its own reason, such as `backend_lock_pressure`) keeps its protocol + * classification, including its validation of `retry_after_seconds`. */ final class TransientFailure { @@ -28,15 +33,17 @@ final class TransientFailure private const FIRST_SECONDS = 0.25; private const MAX_SECONDS = 4.0; private const REASONS = ['storage_unavailable', 'backend_unavailable']; + /** The longest Server-suggested delay honoured for an untyped outage. */ + private const MAX_RETRY_AFTER_SECONDS = 60; - public static function is(Throwable $error): bool + public static function is(Throwable $error, bool $acknowledgement = false): bool { if ($error instanceof ServerException) { if ($error->isTransientConnectionFailure() || $error->isTransientUpstreamFailure()) { return true; } - return self::transientResponse($error->status, $error->reason, $error->details); + return self::transientResponse($error->status, $error->reason, $error->details, $acknowledgement); } if ($error instanceof TransportException) { if ($error->transientConnectionFailure) { @@ -44,28 +51,34 @@ public static function is(Throwable $error): bool } $reason = is_string($error->response['reason'] ?? null) ? $error->response['reason'] : null; - return $error->status !== null && self::transientResponse($error->status, $reason, $error->response); + return $error->status !== null && self::transientResponse($error->status, $reason, $error->response, $acknowledgement); } return false; } - /** Full-jitter delay for the given 1-based attempt. */ - public static function delay(int $attempt): float + /** Full-jitter delay for the given 1-based attempt, never shorter than a bounded retry_after. */ + public static function delay(int $attempt, mixed $retryAfterSeconds = null): float { $ceiling = min(self::MAX_SECONDS, self::FIRST_SECONDS * 2 ** min(max(0, $attempt - 1), 16)); + $delay = $ceiling * random_int(0, 1_000_000) / 1_000_000; - return $ceiling * random_int(0, 1_000_000) / 1_000_000; + return is_int($retryAfterSeconds) && $retryAfterSeconds > 0 + ? max($delay, (float) min($retryAfterSeconds, self::MAX_RETRY_AFTER_SECONDS)) : $delay; } /** @param array|null $details */ - private static function transientResponse(int $status, ?string $reason, ?array $details): bool + private static function transientResponse(int $status, ?string $reason, ?array $details, bool $acknowledgement): bool { $retryable = is_array($details) ? ($details['retryable'] ?? null) : null; if ($retryable === false || !($status === 429 || ($status >= 500 && $status <= 599))) { return false; } - return $retryable === true || ($status === 503 && in_array($reason, self::REASONS, true)); + if ($status === 503 && in_array($reason, self::REASONS, true)) { + return true; + } + + return !$acknowledgement && $retryable === true && ($reason === null || $reason === ''); } } diff --git a/tests/WorkerTransientOutageTest.php b/tests/WorkerTransientOutageTest.php index dbfd8169..32fd63ff 100644 --- a/tests/WorkerTransientOutageTest.php +++ b/tests/WorkerTransientOutageTest.php @@ -97,10 +97,11 @@ public function testStorageUnavailablePollsAreRetriedWithJitterAndTheWorkerKeeps self::assertLessThanOrEqual(1.75, $state->slept); } - public function testATransientFailureThatEscapesATickDoesNotStopTheWorker(): void + public function testAnOutageThatOutlastsPublishingAResultIsNeverReportedAsAFailure(): void { - // The activity's completion and its failure report both meet the - // outage, so the error leaves the tick; the Worker waits and polls on. + // The activity's completion keeps meeting the outage until its retries + // are spent. The result is left to the Server (lease expiry and + // re-delivery), never turned into an activity failure report. $workflowPolls = 0; $activityPolls = 0; $reports = []; @@ -129,9 +130,6 @@ public function testATransientFailureThatEscapesATickDoesNotStopTheWorker(): voi return $workflowPolls >= 2 ? ['task' => null, 'poll_status' => 'stopped', 'reason' => 'worker_stopped'] : ['task' => null, 'poll_status' => 'empty']; } - if (str_ends_with($uri, '/api/worker/heartbeat')) { - return ['worker_id' => 'worker-1', 'acknowledged' => true]; - } self::fail("Unexpected worker request: {$method} {$uri}"); }; $events = []; @@ -145,10 +143,121 @@ public function testATransientFailureThatEscapesATickDoesNotStopTheWorker(): voi $worker->run(0); - self::assertContains('complete', $reports); + self::assertSame(array_fill(0, 9, 'complete'), $reports, 'The first try and eight bounded retries, never a failure report.'); + self::assertContains('worker.outcome_retry_exhausted', $events); + self::assertContains('worker.outcome_unpublished', $events); self::assertGreaterThanOrEqual(2, $workflowPolls, 'The Worker kept polling after the outage.'); - self::assertNotContains('worker.fatal_failure', $events); + } + + public function testATransientFailureThatEscapesATickDoesNotStopTheWorker(): void + { + // A worker heartbeat meets the outage after a tick; the run loop + // absorbs it, waits with jitter and keeps polling. + $now = 0.0; + $workflowPolls = 0; + $heartbeats = 0; + $handler = static function (string $method, string $uri) use (&$now, &$workflowPolls, &$heartbeats): ?array { + if (str_ends_with($uri, '/api/worker/register')) { + return ['registered' => true, 'heartbeat_interval_seconds' => 1]; + } + if (str_ends_with($uri, '/api/worker/workflow-tasks/poll')) { + ++$workflowPolls; + $now += 2.0; + + return ['task' => null, 'poll_status' => 'empty']; + } + if (str_ends_with($uri, '/api/worker/heartbeat')) { + if (++$heartbeats === 1) { + throw self::storageUnavailable(); + } + + return ['worker_id' => 'worker-1', 'acknowledged' => true, 'heartbeat_interval_seconds' => 1]; + } + if (str_ends_with($uri, '/poll')) { + return $workflowPolls >= 3 && str_ends_with($uri, '/query-tasks/poll') + ? ['task' => null, 'poll_status' => 'stopped', 'reason' => 'worker_stopped'] + : ['task' => null, 'poll_status' => 'empty']; + } + self::fail("Unexpected worker request: {$method} {$uri}"); + }; + $events = []; + $worker = new Worker(new Client('https://server.example', transport: new FakeTransport(handler: $handler)), 'orders', + workerId: 'worker-1', + clock: static function () use (&$now): float { return $now; }, + sleeper: static function (int $microseconds) use (&$now): void { $now += $microseconds / 1_000_000; }, + diagnosticListener: static function (string $event, array $context) use (&$events): void { $events[] = $event; }); + + $worker->run(0); + + self::assertGreaterThanOrEqual(1, $heartbeats); self::assertContains('worker.transient_failure', $events, 'The outage left the tick and was absorbed by the run loop.'); + self::assertNotContains('worker.fatal_failure', $events); + self::assertGreaterThanOrEqual(3, $workflowPolls, 'The Worker kept polling after the outage.'); + } + + /** + * The Server commits the completion but the Worker sees 503 storage_unavailable. + * The retried completion is answered as Server 3.17.0 answers a duplicate, and + * the Worker treats it as its own committed completion: no failure report. + */ + #[DataProvider('completionRetryAnswers')] + public function testACompletionMetByAnOutageIsRetriedAndADuplicateCountsAsCommitted(array|TransportException $second): void + { + $completions = []; + $failures = 0; + $handler = static function (string $method, string $uri, array $headers, ?array $body) use (&$completions, &$failures, $second): ?array { + if (str_ends_with($uri, '/api/worker/register')) { + return ['registered' => true]; + } + if (str_ends_with($uri, '/api/worker/workflow-tasks/poll')) { + return ['task' => null, 'poll_status' => 'empty']; + } + if (str_ends_with($uri, '/api/worker/activity-tasks/poll')) { + return $completions === [] ? ['poll_status' => 'leased', 'task' => ['task_id' => 'activity-1', + 'activity_attempt_id' => 'attempt-1', 'lease_owner' => 'worker-1', 'activity_type' => 'orders.charge', + 'payload_codec' => 'avro']] : ['task' => null, 'poll_status' => 'empty']; + } + if (str_ends_with($uri, '/api/worker/activity-tasks/activity-1/complete')) { + $completions[] = $body; + if (count($completions) === 1) { + throw self::storageUnavailable(); + } + if ($second instanceof TransportException) { + throw $second; + } + + return $second; + } + if (str_ends_with($uri, '/api/worker/activity-tasks/activity-1/fail')) { + ++$failures; + + return ['recorded' => true]; + } + if (str_ends_with($uri, '/api/worker/query-tasks/poll')) { + return ['task' => null, 'poll_status' => 'stopped', 'reason' => 'worker_stopped']; + } + self::fail("Unexpected worker request: {$method} {$uri}"); + }; + [$worker, , $state] = $this->worker($handler); + + $worker->run(0); + + self::assertSame(1, $state->calls, 'The handler ran once; only the request was retried.'); + self::assertCount(2, $completions); + self::assertSame($completions[0], $completions[1], 'The same completion, with the same attempt fence.'); + self::assertSame(0, $failures, 'An outage never turns a computed result into a failure report.'); + } + + /** @return iterable|TransportException}> */ + public static function completionRetryAnswers(): iterable + { + yield 'first answer was lost; retry recorded' => [['task_id' => 'activity-1', 'activity_attempt_id' => 'attempt-1', + 'outcome' => 'completed', 'recorded' => true, 'reason' => null]]; + $duplicate = ['task_id' => 'activity-1', 'activity_attempt_id' => 'attempt-1', 'outcome' => 'completed', + 'recorded' => false, 'reason' => 'stale_attempt', 'next_task_id' => null, 'lease_owner' => 'worker-1', + 'activity_status' => 'completed', 'attempt_status' => 'completed', 'task_status' => 'completed']; + yield 'first was committed; duplicate answered as Server 3.17.0 does' => [ + TransportException::fromResponse(409, $duplicate, json_encode($duplicate, JSON_THROW_ON_ERROR))]; } /** @return iterable */ @@ -196,7 +305,15 @@ public function testClassificationAndJitterBounds(): void { self::assertTrue(TransientFailure::is(self::storageUnavailable())); self::assertTrue(TransientFailure::is(new ServerException('x', 503, 'backend_unavailable', ['reason' => 'backend_unavailable']))); - self::assertTrue(TransientFailure::is(new ServerException('x', 502, 'anything', ['retryable' => true]))); + self::assertTrue(TransientFailure::is(new ServerException('x', 502, null, ['retryable' => true]))); + // An unclassified answer cannot authorize repeating an acknowledgement. + self::assertFalse(TransientFailure::is(new ServerException('x', 503, null, ['retryable' => true]), acknowledgement: true)); + self::assertTrue(TransientFailure::is(self::storageUnavailable(), acknowledgement: true)); + // A typed refusal keeps its own protocol classification. + self::assertFalse(TransientFailure::is(new ServerException('x', 503, 'backend_lock_pressure', + ['reason' => 'backend_lock_pressure', 'retryable' => true, 'retry_after_seconds' => PHP_INT_MAX]))); + self::assertLessThanOrEqual(60.0, TransientFailure::delay(1, PHP_INT_MAX)); + self::assertGreaterThanOrEqual(3.0, TransientFailure::delay(1, 3)); self::assertTrue(TransientFailure::is(new TransportException('reset', transientConnectionFailure: true))); self::assertFalse(TransientFailure::is(new ServerException('x', 503, 'storage_unavailable', ['retryable' => false]))); self::assertFalse(TransientFailure::is(new ServerException('x', 503, 'backend_lock_pressure', ['reason' => 'backend_lock_pressure']))); From 3c97391a36d4b8b39c39e277a25279b70424955c Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 20 Nov 2026 07:58:54 +0000 Subject: [PATCH 3/5] Survive outages at registration, in retry waits, and committed-first answers Review HIGHs on #449, found live against Server 3.17.0 through a fault proxy: 1. Registration (the original #448 incident) retried only typed refusals, so 503 storage_unavailable during warm-up still ended the Worker. It now uses the shared transient classification and jittered backoff, and so do the cluster-info reads registration makes in query-authority and cooperative modes. 2. The worker heartbeat inside a retry wait rethrew an untyped outage, so a 12 s outage after registration ended run(). The heartbeat now defers it on the shared schedule like its other transient failures, so no network call in the wait path can end the run. 3. cURL can resend a completion whose reply was lost on a reused connection, so the first answer the Worker sees may already be the Server's duplicate: 409 stale_attempt, recorded false, our owner and attempt, every status completed. That exact shape is accepted as our committed completion on the first answer (worker.completion_already_committed), not only after a seen failure. Tests reproduce all three with a fake transport and fail on the previous head. A live check against durableworkflow/server:3.17.0 with injected faults (three registration 503s, then a 12 s outage of polls and heartbeats) shows no Worker error, and workflows started before and after both complete. Co-Authored-By: Claude Opus 5.5 --- src/Worker.php | 73 +++++++++++++-- tests/WorkerTransientOutageTest.php | 138 +++++++++++++++++++++++++++- 2 files changed, 196 insertions(+), 15 deletions(-) diff --git a/src/Worker.php b/src/Worker.php index 1c41e993..1a111fa8 100644 --- a/src/Worker.php +++ b/src/Worker.php @@ -901,13 +901,15 @@ private function registerWithRetry(): ?array if ($this->client->workerProtocolVersion !== '1.20' || !$this->client->supportsSupervisedQueryRequests()) { throw new WorkflowClaimAborted('Query authority requires protocol 1.20 and bounded Worker transport.'); } - $protocol = $this->client->withBoundedWorkerRequests()->workerClusterInfo()->raw['worker_protocol'] ?? null; + $protocol = $this->retryUntypedOutage('cluster_info', + fn (): array => $this->client->withBoundedWorkerRequests()->workerClusterInfo()->raw)['worker_protocol'] ?? null; if (!is_array($protocol) || ($protocol['server_capabilities']['query_task_authority'] ?? null) !== true) { throw new WorkflowClaimAborted('The runtime does not advertise query task authority.'); } } if ($this->enableCooperativeCancellation) { - $protocol = $this->client->workerClusterInfo()->raw['worker_protocol'] ?? null; + $protocol = $this->retryUntypedOutage('cluster_info', + fn (): array => $this->client->workerClusterInfo()->raw)['worker_protocol'] ?? null; if (!is_array($protocol) || !is_string($protocol['version'] ?? null) || !Version::supportsCooperativeCancellation($protocol['version']) || ($protocol['server_capabilities']['cooperative_cancellation'] ?? null) !== true) { @@ -996,15 +998,18 @@ private function registerWithRetry(): ?array workflowDefinitionFingerprints: $this->workflowDefinitionFingerprints(), ); } catch (ServerException $exception) { - if (!$this->isTransientRegistrationFailure($exception)) { + // A typed registration refusal keeps its protocol backoff; an + // untyped outage (a Server warming up, #448) uses the shared + // jittered schedule. + $untypedOutage = !$this->isTransientRegistrationFailure($exception) && TransientFailure::is($exception); + if (!$untypedOutage && !$this->isTransientRegistrationFailure($exception)) { throw $exception; } ++$attempt; - $delaySeconds = $this->transientRetryDelay( - $attempt, - $exception->details['retry_after_seconds'] ?? null, - ); + $delaySeconds = $untypedOutage + ? TransientFailure::delay($attempt, $exception->details['retry_after_seconds'] ?? null) + : $this->transientRetryDelay($attempt, $exception->details['retry_after_seconds'] ?? null); if ($this->transientPollRetryObserver !== null) { ($this->transientPollRetryObserver)('registration', $attempt, $delaySeconds, $exception); } @@ -1528,8 +1533,18 @@ private function retryStorageAdmission( $activityTaskLease['activity_attempt_id'], $activityTaskLease['lease_owner'], ); - if ($uncertainActivityCompletion && $activityTaskLease !== null + // Only this Worker can complete this attempt as its lease owner, + // so this exact refusal proves our completion was committed, + // even on the first answer: cURL can resend a POST whose reply + // was lost on a reused connection before the Worker sees it. + if ($operation === 'activity_complete' && $activityTaskLease !== null && $this->isCommittedActivityCompletion($exception, $activityTaskLease)) { + if (!$uncertainActivityCompletion) { + $this->diagnostic('worker.completion_already_committed', ['worker_id' => $this->workerId, + 'task_id' => $activityTaskLease['task_id'], + 'activity_attempt_id' => $activityTaskLease['activity_attempt_id']], 'info'); + } + return null; } $backendUnavailable = $workflowBackendUnavailable || $activityBackendUnavailable; @@ -1595,6 +1610,35 @@ private function retryStorageAdmission( } } + /** + * Repeat an idempotent read through an untyped outage, using the shared + * jittered schedule, until it succeeds, fails otherwise, or shutdown. + * + * @template T + * @param \Closure(): T $call + * @return T + */ + private function retryUntypedOutage(string $operation, \Closure $call): mixed + { + $attempt = 0; + while (true) { + try { + return $call(); + } catch (ServerException $exception) { + if (!TransientFailure::is($exception) || $this->shutdownRequested) { + throw $exception; + } + $delay = TransientFailure::delay(++$attempt, $exception->details['retry_after_seconds'] ?? null); + $this->diagnostic('worker.retrying', ['worker_id' => $this->workerId, 'operation' => $operation, + 'attempt' => $attempt, 'delay_seconds' => $delay, 'exception' => $exception], 'warning'); + $this->waitForTransientRetry($delay); + if ($this->shutdownRequested) { + throw $exception; + } + } + } + } + /** Retries of one outcome through an untyped outage before the Server's lease handling takes over. */ private function untypedOutageAttemptLimit(string $operation): int { @@ -4870,7 +4914,13 @@ private function heartbeat(?RequestBudget $budget = null): void 'activity_available' => $this->carriedActivityPoll === null ? 1 : 0, ], $budget); } catch (ServerException $exception) { - if (!$exception->isTransientConnectionFailure() + $untypedOutage = !$exception->isTransientConnectionFailure() + && !$exception->isTransientUpstreamFailure() + && !$this->isTransientDatabaseFailure($exception, 'heartbeat_worker') + && !$exception->isStorageAdmissionFailure() + && TransientFailure::is($exception); + if (!$untypedOutage + && !$exception->isTransientConnectionFailure() && !$exception->isTransientUpstreamFailure() && !$this->isTransientDatabaseFailure($exception, 'heartbeat_worker') && !$exception->isStorageAdmissionFailure()) { @@ -4879,8 +4929,11 @@ private function heartbeat(?RequestBudget $budget = null): void // Do not enter the heartbeat-aware wait recursively or delay an // already leased task. The next poll/wait services this deadline. + // An outage during a retry wait therefore never ends run() (#448). $attempt = ++$this->heartbeatRetryAttempt; - $delaySeconds = $this->transientRetryDelay($attempt, $exception->details['retry_after_seconds'] ?? null); + $delaySeconds = $untypedOutage + ? TransientFailure::delay($attempt, $exception->details['retry_after_seconds'] ?? null) + : $this->transientRetryDelay($attempt, $exception->details['retry_after_seconds'] ?? null); $this->heartbeatRetryAt = $this->now() + $delaySeconds; if ($this->transientPollRetryObserver !== null) { ($this->transientPollRetryObserver)('heartbeat', $attempt, $delaySeconds, $exception); diff --git a/tests/WorkerTransientOutageTest.php b/tests/WorkerTransientOutageTest.php index 32fd63ff..48b629ee 100644 --- a/tests/WorkerTransientOutageTest.php +++ b/tests/WorkerTransientOutageTest.php @@ -149,10 +149,10 @@ public function testAnOutageThatOutlastsPublishingAResultIsNeverReportedAsAFailu self::assertGreaterThanOrEqual(2, $workflowPolls, 'The Worker kept polling after the outage.'); } - public function testATransientFailureThatEscapesATickDoesNotStopTheWorker(): void + public function testAHeartbeatOutageBetweenTicksIsDeferredAndTheWorkerKeepsRunning(): void { - // A worker heartbeat meets the outage after a tick; the run loop - // absorbs it, waits with jitter and keeps polling. + // A worker heartbeat meets the outage after a tick. It is deferred on + // the shared schedule; it never ends run(). $now = 0.0; $workflowPolls = 0; $heartbeats = 0; @@ -189,12 +189,140 @@ public function testATransientFailureThatEscapesATickDoesNotStopTheWorker(): voi $worker->run(0); - self::assertGreaterThanOrEqual(1, $heartbeats); - self::assertContains('worker.transient_failure', $events, 'The outage left the tick and was absorbed by the run loop.'); + self::assertGreaterThanOrEqual(2, $heartbeats, 'The deferred heartbeat was sent again.'); + self::assertContains('worker.retrying', $events); self::assertNotContains('worker.fatal_failure', $events); self::assertGreaterThanOrEqual(3, $workflowPolls, 'The Worker kept polling after the outage.'); } + /** Review HIGH 1: the original incident, a Server warming up while the Worker registers. */ + public function testStorageUnavailableDuringRegistrationIsRetried(): void + { + $registrations = 0; + $handler = static function (string $method, string $uri) use (&$registrations): ?array { + if (str_ends_with($uri, '/api/worker/register')) { + if (++$registrations <= 3) { + throw self::storageUnavailable(); + } + + return ['registered' => true]; + } + if (str_ends_with($uri, '/query-tasks/poll')) { + return ['task' => null, 'poll_status' => 'stopped', 'reason' => 'worker_stopped']; + } + if (str_ends_with($uri, '/poll')) { + return ['task' => null, 'poll_status' => 'empty']; + } + self::fail("Unexpected worker request: {$method} {$uri}"); + }; + [$worker, , $state] = $this->worker($handler); + + $worker->run(0); + + self::assertSame(4, $registrations, 'Three 503 storage_unavailable answers, then registration succeeds.'); + self::assertLessThanOrEqual(1.75, $state->slept, 'Jittered waits of at most 0.25, 0.5 and 1 s.'); + } + + /** Review HIGH 2: a failing heartbeat during a poll retry wait must not end run(). */ + public function testAnOutageThatAlsoFailsTheHeartbeatDuringARetryWaitDoesNotEndTheRun(): void + { + $now = 0.0; + $workflowPolls = 0; + $heartbeats = 0; + $handler = static function (string $method, string $uri) use (&$workflowPolls, &$heartbeats): ?array { + if (str_ends_with($uri, '/api/worker/register')) { + return ['registered' => true, 'heartbeat_interval_seconds' => 1]; + } + if (str_ends_with($uri, '/api/worker/heartbeat')) { + if (++$heartbeats <= 3) { + throw self::storageUnavailable(); + } + + return ['worker_id' => 'worker-1', 'acknowledged' => true, 'heartbeat_interval_seconds' => 1]; + } + if (str_ends_with($uri, '/workflow-tasks/poll')) { + // A 12-attempt outage: long enough for heartbeats to fall due inside the retry waits. + if (++$workflowPolls <= 12) { + throw self::storageUnavailable(); + } + + return ['task' => null, 'poll_status' => 'empty']; + } + if (str_ends_with($uri, '/query-tasks/poll')) { + return ['task' => null, 'poll_status' => 'stopped', 'reason' => 'worker_stopped']; + } + if (str_ends_with($uri, '/poll')) { + return ['task' => null, 'poll_status' => 'empty']; + } + self::fail("Unexpected worker request: {$method} {$uri}"); + }; + $events = []; + $worker = new Worker(new Client('https://server.example', transport: new FakeTransport(handler: $handler)), 'orders', + workerId: 'worker-1', + clock: static function () use (&$now): float { return $now; }, + sleeper: static function (int $microseconds) use (&$now): void { $now += $microseconds / 1_000_000; }, + diagnosticListener: static function (string $event, array $context) use (&$events): void { $events[] = $event; }); + + $worker->run(0); + + self::assertSame(13, $workflowPolls); + self::assertGreaterThanOrEqual(1, $heartbeats, 'A heartbeat fell due inside a retry wait and met the outage.'); + self::assertNotContains('worker.fatal_failure', $events); + } + + /** + * Review HIGH 3: cURL resent a completion whose reply was lost on a reused + * connection, and the first answer the Worker sees is the Server's + * duplicate: 409 stale_attempt, recorded false, every status completed. + * That is our own committed completion, not an error. + */ + public function testACommittedCompletionReportedOnTheFirstAnswerIsOurs(): void + { + $completions = 0; + $failures = 0; + $duplicate = ['task_id' => 'activity-1', 'activity_attempt_id' => 'attempt-1', 'outcome' => 'completed', + 'recorded' => false, 'reason' => 'stale_attempt', 'next_task_id' => null, 'lease_owner' => 'worker-1', + 'activity_status' => 'completed', 'attempt_status' => 'completed', 'task_status' => 'completed']; + $handler = static function (string $method, string $uri) use (&$completions, &$failures, $duplicate): ?array { + if (str_ends_with($uri, '/api/worker/register')) { + return ['registered' => true]; + } + if (str_ends_with($uri, '/activity-tasks/poll')) { + return $completions === 0 ? ['poll_status' => 'leased', 'task' => ['task_id' => 'activity-1', + 'activity_attempt_id' => 'attempt-1', 'lease_owner' => 'worker-1', 'activity_type' => 'orders.charge', + 'payload_codec' => 'avro']] : ['task' => null, 'poll_status' => 'empty']; + } + if (str_ends_with($uri, '/activity-1/complete')) { + ++$completions; + throw TransportException::fromResponse(409, $duplicate, json_encode($duplicate, JSON_THROW_ON_ERROR)); + } + if (str_ends_with($uri, '/activity-1/fail')) { + ++$failures; + + return ['recorded' => true]; + } + if (str_ends_with($uri, '/query-tasks/poll')) { + return ['task' => null, 'poll_status' => 'stopped', 'reason' => 'worker_stopped']; + } + if (str_ends_with($uri, '/poll')) { + return ['task' => null, 'poll_status' => 'empty']; + } + self::fail("Unexpected worker request: {$method} {$uri}"); + }; + $events = []; + $worker = new Worker(new Client('https://server.example', transport: new FakeTransport(handler: $handler)), 'orders', + workerId: 'worker-1', sleeper: static function (int $microseconds): void {}, + diagnosticListener: static function (string $event, array $context) use (&$events): void { $events[] = $event; }); + $worker->registerActivity('orders.charge', static fn (ActivityContext $context): string => 'charged'); + + $worker->run(0); + + self::assertSame(1, $completions, 'Not resent: the answer already proves the completion committed.'); + self::assertSame(0, $failures); + self::assertContains('worker.completion_already_committed', $events); + self::assertNotContains('worker.fatal_failure', $events); + } + /** * The Server commits the completion but the Worker sees 503 storage_unavailable. * The retried completion is answered as Server 3.17.0 answers a duplicate, and From f870b55a488066527d71d7df48deb59adf2f07e3 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 20 Nov 2026 08:16:56 +0000 Subject: [PATCH 4/5] Registration retries a definite outage answer, never a lost connection The registration retry also retried connection failures, so the existing contract that an ambiguous registration (the Server may have registered) is not blindly repeated looped until the CI timeout. Registration now retries only the Server's definite outage answer, 503 storage_unavailable or backend_unavailable (the #448 warm-up incident), on the shared jittered schedule. Full suite: 9216 tests, 0 failures. Co-Authored-By: Claude Opus 5.5 --- src/Worker.php | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/Worker.php b/src/Worker.php index 1a111fa8..d8228021 100644 --- a/src/Worker.php +++ b/src/Worker.php @@ -998,10 +998,14 @@ private function registerWithRetry(): ?array workflowDefinitionFingerprints: $this->workflowDefinitionFingerprints(), ); } catch (ServerException $exception) { - // A typed registration refusal keeps its protocol backoff; an - // untyped outage (a Server warming up, #448) uses the shared - // jittered schedule. - $untypedOutage = !$this->isTransientRegistrationFailure($exception) && TransientFailure::is($exception); + // A typed registration refusal keeps its protocol backoff. A + // definite outage answer (503 storage_unavailable or + // backend_unavailable: a Server warming up, #448) uses the + // shared jittered schedule. A lost connection stays ambiguous + // (the Server may have registered) and is not repeated. + $untypedOutage = !$this->isTransientRegistrationFailure($exception) + && !$exception->isTransientConnectionFailure() && !$exception->isTransientUpstreamFailure() + && TransientFailure::is($exception, acknowledgement: true); if (!$untypedOutage && !$this->isTransientRegistrationFailure($exception)) { throw $exception; } From 8e383aa77adc4cbfbfa5ebc55e208c36d73ee5f1 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 20 Nov 2026 09:10:21 +0000 Subject: [PATCH 5/5] Wait for the Server at startup; report lasting outages as errors Re-check follow-up on #449: a Worker started during an outage still exited on a refused connection, an empty reply or a bodiless 502/503/504 at registration. - Registration retries every outage on the shared jittered schedule: refused or reset connections, empty replies, bodiless 502/503/504 from a load balancer, and 503 storage_unavailable / backend_unavailable. Registration is idempotent: against durableworkflow/server:3.17.0, registering the same worker ID twice answers registered: true both times. The two tests that forbade the repeat are replaced by ones asserting it keeps the original identity. - An outage lasting 60 s is reported once as a worker.outage_persisting error, then every 5 minutes, until a request succeeds (registration, poll retries and the run loop). - Tests: server down at startup (refused connection, empty reply), a load balancer answering bare 502 and 504, a bodiless 503 behind a load balancer, and the 60 s / 5 min escalation on a simulated clock. Live: a Worker started with the Server stopped for about 15 s registered once it returned and completed a workflow. Full suite: 9221 tests, 0 failures. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 7 +++ docs/portal/build/workers.md | 2 + src/Worker.php | 47 +++++++++++++-- tests/WorkerConnectionRecoveryTest.php | 28 +++++---- tests/WorkerTransientOutageTest.php | 82 ++++++++++++++++++++++++++ tests/WorkerUpstreamRecoveryTest.php | 32 ++++++---- 6 files changed, 169 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90721d25..f89d35d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,13 @@ project follows [Semantic Versioning](https://semver.org/). doubling to a 4 s cap). Authentication, authorization, protocol refusals and explicit `retryable: false` still stop the Worker, after a `worker.fatal_failure` diagnostic. +- A Worker started while the Server is down or warming up waits for it. + Registration retries refused or reset connections, empty replies, bodiless + 502/503/504 answers from a load balancer, and `503 storage_unavailable` / + `backend_unavailable`. Registration is idempotent: Server 3.17.0 accepts the + same worker ID registering again. An outage lasting 60 s is reported once as + a `worker.outage_persisting` error, then every 5 minutes until a request + succeeds. - A completion that meets an untyped outage is retried with the same attempt fence, inside the claim's lease, instead of being reported as an activity failure. A duplicate of a completion the Server already committed (Server diff --git a/docs/portal/build/workers.md b/docs/portal/build/workers.md index d549f040..aa7b3215 100644 --- a/docs/portal/build/workers.md +++ b/docs/portal/build/workers.md @@ -99,6 +99,8 @@ An activity option such as `['queue' => 'payments-v2']` selects the queue of a n An untyped outage does not stop the Worker either. That covers a connection that is refused, reset or timed out; a 502/503/504 with no protocol body; a 503 `storage_unavailable` or `backend_unavailable`, for example while the Server's database warms up; and any 5xx or 429 with `retryable: true`. The Worker retries using the schedule all SDKs share: 0.25 s doubling to a 4 s cap, with full random jitter. This applies to polls, and to any request that fails a tick, such as a completion; the Server re-delivers a claimed task when its lease ends. Authentication and authorization failures, other protocol refusals, and explicit `retryable: false` stay fatal. They are reported as a `worker.fatal_failure` diagnostic before `run()` rethrows. +**Startup and long outages.** A Worker started while the Server or its load balancer is unavailable keeps retrying registration on the same schedule instead of exiting. Registration is safe to repeat: the Server accepts the same worker ID registering again. An outage that lasts 60 s is reported as a `worker.outage_persisting` error, then every 5 minutes until a request succeeds. + **Completions.** A completion that meets an outage is retried with the same attempt fence on that schedule. With activity lease maintenance, the retries stop when the claim's grant ends; without it, they stop after a bounded number of attempts. Retrying is safe: the Server records one outcome per attempt. A retry of a completion the Server already committed is answered `409 stale_attempt` with `recorded: false` and the attempt `completed`, and the Worker treats that as its own committed completion. A result that cannot be published before the retries end is logged as `worker.outcome_unpublished` and left to the Server, which re-delivers the attempt when its lease ends. It is never reported as an activity failure. **Across SDKs.** PHP and Rust Workers stop on a 401 or 403 after logging it as an error. Python Workers keep retrying polls through a 401 or 403 so that credentials can be rotated without a restart, and log every refusal at error level. diff --git a/src/Worker.php b/src/Worker.php index d8228021..c9885489 100644 --- a/src/Worker.php +++ b/src/Worker.php @@ -76,6 +76,9 @@ final class Worker private const DEFAULT_HEARTBEAT_INTERVAL_SECONDS = 30; /** Untyped-outage retries of one outcome without a maintained lease: up to about 15 s of jittered waits. */ private const UNTYPED_OUTAGE_MAX_ATTEMPTS = 8; + /** A lasting outage is reported as an error after this long, then at this interval. */ + private const OUTAGE_ERROR_AFTER_SECONDS = 60.0; + private const OUTAGE_ERROR_EVERY_SECONDS = 300.0; /** A release must land this long before the claim's grant ends. */ private const RELEASE_MARGIN_SECONDS = 0.5; private const WORKFLOW_HISTORY_PAGE_SIZE = 500; @@ -144,6 +147,8 @@ final class Worker private ?ActivityLeaseProcess $activityLeaseHelper = null; /** @var array|null The activity claim whose lease maintenance is starting. */ private ?array $claimingActivityTask = null; + private ?float $outageStartedAt = null; + private ?float $outageErrorLoggedAt = null; /** Monotonic end of that claim's conservative poll grant. */ private float $claimingActivityDeadline = 0; /** @var array{0: array, 1: string, 2: float}|null A stopped claim to release once its lease binding is gone. */ @@ -486,6 +491,7 @@ public function run(int $pollTimeoutSeconds = 5): void if ($registration === null) { return; } + $this->clearOutage(); $this->establishRegistration($registration); $transientAttempt = 0; while (!$this->shutdownRequested || $this->staleRegistrationObserved) { @@ -497,6 +503,7 @@ public function run(int $pollTimeoutSeconds = 5): void } $this->heartbeatIfDue(); $transientAttempt = 0; + $this->clearOutage(); } catch (Throwable $exception) { if ($this->recoversFromStaleRegistration($exception)) { $this->staleRegistrationObserved = true; @@ -509,6 +516,7 @@ public function run(int $pollTimeoutSeconds = 5): void $this->diagnostic('worker.transient_failure', ['worker_id' => $this->workerId, 'task_queue' => $this->taskQueue, 'attempt' => $transientAttempt, 'delay_seconds' => $delay, 'exception' => $exception], 'warning'); + $this->observeOutage('run', $exception); $this->waitForTransientRetry($delay); } else { $this->diagnostic('worker.fatal_failure', ['worker_id' => $this->workerId, @@ -998,13 +1006,14 @@ private function registerWithRetry(): ?array workflowDefinitionFingerprints: $this->workflowDefinitionFingerprints(), ); } catch (ServerException $exception) { - // A typed registration refusal keeps its protocol backoff. A - // definite outage answer (503 storage_unavailable or - // backend_unavailable: a Server warming up, #448) uses the - // shared jittered schedule. A lost connection stays ambiguous - // (the Server may have registered) and is not repeated. + // A typed registration refusal keeps its protocol backoff. An + // outage (a Server warming up or down, a refused or reset + // connection, an empty reply, a bodiless 502/503/504 from a + // load balancer, #448) uses the shared jittered schedule. + // Registration is idempotent: Server 3.17.0 accepts the same + // worker ID registering again, so repeating it after a lost + // reply is safe. $untypedOutage = !$this->isTransientRegistrationFailure($exception) - && !$exception->isTransientConnectionFailure() && !$exception->isTransientUpstreamFailure() && TransientFailure::is($exception, acknowledgement: true); if (!$untypedOutage && !$this->isTransientRegistrationFailure($exception)) { throw $exception; @@ -1024,6 +1033,7 @@ private function registerWithRetry(): ?array 'delay_seconds' => $delaySeconds, 'exception' => $exception, ], 'warning'); + $this->observeOutage('registration', $exception); $this->waitForTransientRetry($delaySeconds); } } @@ -1031,6 +1041,30 @@ private function registerWithRetry(): ?array return null; } + /** + * Escalate a lasting outage: one `worker.outage_persisting` error once it + * has lasted 60 s, then one every 5 minutes, until a request succeeds. + */ + private function observeOutage(string $operation, Throwable $exception): void + { + $now = $this->now(); + $this->outageStartedAt ??= $now; + $lasted = $now - $this->outageStartedAt; + if ($lasted < self::OUTAGE_ERROR_AFTER_SECONDS || ($this->outageErrorLoggedAt !== null + && $now - $this->outageErrorLoggedAt < self::OUTAGE_ERROR_EVERY_SECONDS)) { + return; + } + $this->outageErrorLoggedAt = $now; + $this->diagnostic('worker.outage_persisting', ['worker_id' => $this->workerId, 'task_queue' => $this->taskQueue, + 'operation' => $operation, 'outage_seconds' => round($lasted, 1), 'exception' => $exception], 'error'); + } + + private function clearOutage(): void + { + $this->outageStartedAt = null; + $this->outageErrorLoggedAt = null; + } + private function isTransientRegistrationFailure(ServerException $exception): bool { if ($this->isTransientDatabaseFailure($exception, 'register_worker') @@ -1461,6 +1495,7 @@ private function waitForPollRetry(string $kind, int $attempt, ServerException $e ($this->transientPollRetryObserver)?->__invoke($kind, $attempt, $delay, $error); $this->diagnostic('worker.retrying', ['worker_id' => $this->workerId, 'operation' => "{$kind}_poll", 'attempt' => $attempt, 'delay_seconds' => $delay, 'exception' => $error], 'warning'); + $this->observeOutage("{$kind}_poll", $error); $this->waitForTransientRetry($delay); } diff --git a/tests/WorkerConnectionRecoveryTest.php b/tests/WorkerConnectionRecoveryTest.php index a48cfe3a..fb3986e0 100644 --- a/tests/WorkerConnectionRecoveryTest.php +++ b/tests/WorkerConnectionRecoveryTest.php @@ -354,18 +354,26 @@ public static function terminalFailures(): iterable } } - public function testAmbiguousRegistrationIsNotRepeated(): void + /** + * Registration is idempotent: Server 3.17.0 accepts the same worker ID + * registering again (both answers `registered: true`), so a registration + * whose reply was lost is repeated with its original identity (#448). + */ + public function testRegistrationAfterALostConnectionIsRepeatedWithItsIdentity(): void { - $transport = new FakeTransport([self::disconnected()]); - $worker = new Worker(new Client('https://server.example', transport: $transport), 'orders'); + $transport = new FakeTransport([ + self::disconnected(), + ['registered' => true], + ['task' => null, 'poll_status' => 'stopped', 'reason' => 'worker_stopped'], + ]); + $worker = new Worker(new Client('https://server.example', transport: $transport), 'orders', + sleeper: static function (int $microseconds): void {}); - try { - $worker->run(0); - self::fail('Registration requires explicit reconciliation, not blind retry.'); - } catch (ServerException $exception) { - self::assertTrue($exception->isTransientConnectionFailure()); - } - self::assertCount(1, $transport->requests); + $worker->run(0); + + self::assertStringEndsWith('/api/worker/register', $transport->requests[0]['uri']); + self::assertStringEndsWith('/api/worker/register', $transport->requests[1]['uri']); + self::assertSame($transport->requests[0]['body'], $transport->requests[1]['body']); } public function testAnActivityCompletionConnectionFailureDoesNotRepeatTheHandler(): void diff --git a/tests/WorkerTransientOutageTest.php b/tests/WorkerTransientOutageTest.php index 48b629ee..023bfd97 100644 --- a/tests/WorkerTransientOutageTest.php +++ b/tests/WorkerTransientOutageTest.php @@ -223,6 +223,88 @@ public function testStorageUnavailableDuringRegistrationIsRetried(): void self::assertLessThanOrEqual(1.75, $state->slept, 'Jittered waits of at most 0.25, 0.5 and 1 s.'); } + /** @return iterable */ + public static function startupOutages(): iterable + { + yield 'server down: connection refused' => [new TransportException('Connection refused', transientConnectionFailure: true)]; + yield 'empty reply from the server' => [new TransportException('Empty reply from server', transientConnectionFailure: true)]; + yield 'load balancer: bare 502' => [TransportException::fromResponse(502, null, '')]; + yield 'load balancer: bare 504' => [TransportException::fromResponse(504, null, '')]; + } + + /** A Worker started while the Server (or its load balancer) is down waits for it instead of exiting. */ + #[DataProvider('startupOutages')] + public function testAWorkerStartedDuringAnOutageWaitsForTheServer(TransportException $outage): void + { + $registrations = []; + $handler = static function (string $method, string $uri, array $headers, ?array $body) use (&$registrations, $outage): ?array { + if (str_ends_with($uri, '/api/worker/register')) { + $registrations[] = $body; + if (count($registrations) <= 4) { + throw $outage; + } + + return ['registered' => true]; + } + if (str_ends_with($uri, '/query-tasks/poll')) { + return ['task' => null, 'poll_status' => 'stopped', 'reason' => 'worker_stopped']; + } + if (str_ends_with($uri, '/poll')) { + return ['task' => null, 'poll_status' => 'empty']; + } + self::fail("Unexpected worker request: {$method} {$uri}"); + }; + [$worker] = $this->worker($handler); + + $worker->run(0); + + self::assertCount(5, $registrations); + self::assertCount(1, array_unique(array_map('serialize', $registrations)), 'Repeated with its original identity.'); + } + + public function testALastingOutageIsReportedAsAnErrorAfterAMinuteThenEveryFiveMinutes(): void + { + $now = 0.0; + $registrations = 0; + $handler = static function (string $method, string $uri) use (&$now, &$registrations): ?array { + if (str_ends_with($uri, '/api/worker/register')) { + ++$registrations; + // Down for about 12 minutes of simulated time. + if ($now < 720.0) { + $now += 5.0; + throw new TransportException('Connection refused', transientConnectionFailure: true); + } + + return ['registered' => true]; + } + if (str_ends_with($uri, '/query-tasks/poll')) { + return ['task' => null, 'poll_status' => 'stopped', 'reason' => 'worker_stopped']; + } + if (str_ends_with($uri, '/poll')) { + return ['task' => null, 'poll_status' => 'empty']; + } + self::fail("Unexpected worker request: {$method} {$uri}"); + }; + $errors = []; + $worker = new Worker(new Client('https://server.example', transport: new FakeTransport(handler: $handler)), 'orders', + workerId: 'worker-1', + clock: static function () use (&$now): float { return $now; }, + sleeper: static function (int $microseconds) use (&$now): void { $now += $microseconds / 1_000_000; }, + diagnosticListener: static function (string $event, array $context) use (&$errors): void { + if ($event === 'worker.outage_persisting') { + $errors[] = $context['outage_seconds']; + } + }); + + $worker->run(0); + + self::assertCount(3, $errors, 'At about 60 s, then about every 5 minutes: '.json_encode($errors)); + self::assertGreaterThanOrEqual(60.0, $errors[0]); + self::assertLessThan(75.0, $errors[0]); + self::assertGreaterThanOrEqual(300.0, $errors[1] - $errors[0]); + self::assertGreaterThanOrEqual(300.0, $errors[2] - $errors[1]); + } + /** Review HIGH 2: a failing heartbeat during a poll retry wait must not end run(). */ public function testAnOutageThatAlsoFailsTheHeartbeatDuringARetryWaitDoesNotEndTheRun(): void { diff --git a/tests/WorkerUpstreamRecoveryTest.php b/tests/WorkerUpstreamRecoveryTest.php index 6134534a..41bcd8f8 100644 --- a/tests/WorkerUpstreamRecoveryTest.php +++ b/tests/WorkerUpstreamRecoveryTest.php @@ -222,21 +222,27 @@ public static function terminalResponses(): iterable yield 'unknown structured error' => [503, '{"error":"refused"}']; } - public function testRegistrationDoesNotBlindlyRepeatAnUncertainRequest(): void + /** + * A load balancer answering a bodiless 503 at registration is an outage, + * and registration is idempotent (Server 3.17.0 accepts the same worker ID + * registering again), so the Worker waits and registers once the Server + * answers (#448). + */ + public function testRegistrationBehindAnUnavailableLoadBalancerIsRepeated(): void { - $calls = 0; - $worker = new Worker(self::client(static function () use (&$calls): ResponseInterface { - ++$calls; - return new Response(503, [], 'Unavailable'); + $registrations = 0; + $worker = new Worker(self::client(static function (RequestInterface $request) use (&$registrations): ResponseInterface { + $path = $request->getUri()->getPath(); + if (str_ends_with($path, '/worker/register')) { + return ++$registrations <= 2 ? new Response(503, [], 'Unavailable') : self::json(['registered' => true]); + } + + return self::json(['task' => null, 'poll_status' => 'stopped', 'reason' => 'worker_stopped']); }), 'orders'); - try { - $worker->run(0); - self::fail('Registration requires reconciliation, not blind retry.'); - } catch (ServerException $exception) { - self::assertSame(503, $exception->status); - self::assertTrue($exception->isTransientUpstreamFailure()); - self::assertSame(1, $calls); - } + + $worker->run(0); + + self::assertSame(3, $registrations); } #[DataProvider('upstreamStatuses')]