diff --git a/CHANGELOG.md b/CHANGELOG.md index b4c4f518..f89d35d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,30 @@ project follows [Semantic Versioning](https://semver.org/). ### Fixed +- A managed Worker no longer exits on a transient Server or network outage + (#448). For example, native Server answers `503 storage_unavailable` while + its database warms up. Polls retry untyped transients with the poll's + identity kept, and `Worker::run()` absorbs a transient failure that escapes a + tick (a completion, for example), using the shared jittered schedule (0.25 s + doubling to a 4 s cap). Authentication, authorization, protocol refusals and + explicit `retryable: false` still stop the Worker, after a + `worker.fatal_failure` diagnostic. +- A Worker started while the Server is down or warming up waits for it. + Registration retries refused or reset connections, empty replies, bodiless + 502/503/504 answers from a load balancer, and `503 storage_unavailable` / + `backend_unavailable`. Registration is idempotent: Server 3.17.0 accepts the + same worker ID registering again. An outage lasting 60 s is reported once as + a `worker.outage_persisting` error, then every 5 minutes until a request + succeeds. +- A completion that meets an untyped outage is retried with the same attempt + fence, inside the claim's lease, instead of being reported as an activity + failure. A duplicate of a completion the Server already committed (Server + 3.17.0: `409 stale_attempt`, `recorded: false`, attempt `completed`) counts as + committed. A result that cannot be published in time is logged + (`worker.outcome_unpublished`) and left to the Server's lease handling. + +### Fixed + - An activity worker no longer drops a claimed task when the lease renewal at activity start is slow or fails transiently (#445). The worker keeps working under the poll's original lease, charged conservatively from the poll's own diff --git a/docs/portal/build/workers.md b/docs/portal/build/workers.md index 2cc9c8bb..aa7b3215 100644 --- a/docs/portal/build/workers.md +++ b/docs/portal/build/workers.md @@ -97,6 +97,14 @@ An activity option such as `['queue' => 'payments-v2']` selects the queue of a n `Worker::run()` registers, negotiates heartbeat cadence, polls workflow/activity/query tasks, renews workflow-task leases, and reports terminal outcomes. Typed transient poll pressure is retried with capped backoff while registration heartbeats and shutdown checks remain responsive. +An untyped outage does not stop the Worker either. That covers a connection that is refused, reset or timed out; a 502/503/504 with no protocol body; a 503 `storage_unavailable` or `backend_unavailable`, for example while the Server's database warms up; and any 5xx or 429 with `retryable: true`. The Worker retries using the schedule all SDKs share: 0.25 s doubling to a 4 s cap, with full random jitter. This applies to polls, and to any request that fails a tick, such as a completion; the Server re-delivers a claimed task when its lease ends. Authentication and authorization failures, other protocol refusals, and explicit `retryable: false` stay fatal. They are reported as a `worker.fatal_failure` diagnostic before `run()` rethrows. + +**Startup and long outages.** A Worker started while the Server or its load balancer is unavailable keeps retrying registration on the same schedule instead of exiting. Registration is safe to repeat: the Server accepts the same worker ID registering again. An outage that lasts 60 s is reported as a `worker.outage_persisting` error, then every 5 minutes until a request succeeds. + +**Completions.** A completion that meets an outage is retried with the same attempt fence on that schedule. With activity lease maintenance, the retries stop when the claim's grant ends; without it, they stop after a bounded number of attempts. Retrying is safe: the Server records one outcome per attempt. A retry of a completion the Server already committed is answered `409 stale_attempt` with `recorded: false` and the attempt `completed`, and the Worker treats that as its own committed completion. A result that cannot be published before the retries end is logged as `worker.outcome_unpublished` and left to the Server, which re-delivers the attempt when its lease ends. It is never reported as an activity failure. + +**Across SDKs.** PHP and Rust Workers stop on a 401 or 403 after logging it as an error. Python Workers keep retrying polls through a 401 or 403 so that credentials can be rotated without a restart, and log every refusal at error level. + ### Activity continuations Activity continuations are an explicit constructor opt-in: diff --git a/src/Worker.php b/src/Worker.php index da715db1..c9885489 100644 --- a/src/Worker.php +++ b/src/Worker.php @@ -22,6 +22,7 @@ use DurableWorkflow\Worker\ActivityLeaseProcess; use DurableWorkflow\Worker\ActivityLeaseAuthority; use DurableWorkflow\Worker\ActivityLeaseMaintenanceException; +use DurableWorkflow\Worker\TransientFailure; use DurableWorkflow\Worker\ActivityExecutionFailure; use DurableWorkflow\Worker\CapabilityManifest; use DurableWorkflow\Worker\CancellationHistory; @@ -73,6 +74,11 @@ final class Worker { private const DEFAULT_HEARTBEAT_INTERVAL_SECONDS = 30; + /** Untyped-outage retries of one outcome without a maintained lease: up to about 15 s of jittered waits. */ + private const UNTYPED_OUTAGE_MAX_ATTEMPTS = 8; + /** A lasting outage is reported as an error after this long, then at this interval. */ + private const OUTAGE_ERROR_AFTER_SECONDS = 60.0; + private const OUTAGE_ERROR_EVERY_SECONDS = 300.0; /** A release must land this long before the claim's grant ends. */ private const RELEASE_MARGIN_SECONDS = 0.5; private const WORKFLOW_HISTORY_PAGE_SIZE = 500; @@ -141,6 +147,8 @@ final class Worker private ?ActivityLeaseProcess $activityLeaseHelper = null; /** @var array|null The activity claim whose lease maintenance is starting. */ private ?array $claimingActivityTask = null; + private ?float $outageStartedAt = null; + private ?float $outageErrorLoggedAt = null; /** Monotonic end of that claim's conservative poll grant. */ private float $claimingActivityDeadline = 0; /** @var array{0: array, 1: string, 2: float}|null A stopped claim to release once its lease binding is gone. */ @@ -483,7 +491,9 @@ public function run(int $pollTimeoutSeconds = 5): void if ($registration === null) { return; } + $this->clearOutage(); $this->establishRegistration($registration); + $transientAttempt = 0; while (!$this->shutdownRequested || $this->staleRegistrationObserved) { try { if ($this->completionPipeline !== null) { @@ -492,11 +502,30 @@ public function run(int $pollTimeoutSeconds = 5): void $this->tick($pollTimeoutSeconds); } $this->heartbeatIfDue(); + $transientAttempt = 0; + $this->clearOutage(); } catch (Throwable $exception) { - if (!$this->recoversFromStaleRegistration($exception)) { + if ($this->recoversFromStaleRegistration($exception)) { + $this->staleRegistrationObserved = true; + } elseif (TransientFailure::is($exception)) { + // The Server or network was briefly unavailable (for + // example a database warming up). Keep the Worker + // running; any claimed task is re-delivered by the + // Server when its lease ends. + $delay = TransientFailure::delay(++$transientAttempt); + $this->diagnostic('worker.transient_failure', ['worker_id' => $this->workerId, + 'task_queue' => $this->taskQueue, 'attempt' => $transientAttempt, + 'delay_seconds' => $delay, 'exception' => $exception], 'warning'); + $this->observeOutage('run', $exception); + $this->waitForTransientRetry($delay); + } else { + $this->diagnostic('worker.fatal_failure', ['worker_id' => $this->workerId, + 'task_queue' => $this->taskQueue, + 'status' => $exception instanceof ServerException ? $exception->status : null, + 'reason' => $exception instanceof ServerException ? $exception->reason : null, + 'exception' => $exception], 'error'); throw $exception; } - $this->staleRegistrationObserved = true; } if ($this->staleRegistrationObserved && !$this->recoverStaleRegistration()) { break; @@ -880,13 +909,15 @@ private function registerWithRetry(): ?array if ($this->client->workerProtocolVersion !== '1.20' || !$this->client->supportsSupervisedQueryRequests()) { throw new WorkflowClaimAborted('Query authority requires protocol 1.20 and bounded Worker transport.'); } - $protocol = $this->client->withBoundedWorkerRequests()->workerClusterInfo()->raw['worker_protocol'] ?? null; + $protocol = $this->retryUntypedOutage('cluster_info', + fn (): array => $this->client->withBoundedWorkerRequests()->workerClusterInfo()->raw)['worker_protocol'] ?? null; if (!is_array($protocol) || ($protocol['server_capabilities']['query_task_authority'] ?? null) !== true) { throw new WorkflowClaimAborted('The runtime does not advertise query task authority.'); } } if ($this->enableCooperativeCancellation) { - $protocol = $this->client->workerClusterInfo()->raw['worker_protocol'] ?? null; + $protocol = $this->retryUntypedOutage('cluster_info', + fn (): array => $this->client->workerClusterInfo()->raw)['worker_protocol'] ?? null; if (!is_array($protocol) || !is_string($protocol['version'] ?? null) || !Version::supportsCooperativeCancellation($protocol['version']) || ($protocol['server_capabilities']['cooperative_cancellation'] ?? null) !== true) { @@ -975,15 +1006,23 @@ private function registerWithRetry(): ?array workflowDefinitionFingerprints: $this->workflowDefinitionFingerprints(), ); } catch (ServerException $exception) { - if (!$this->isTransientRegistrationFailure($exception)) { + // A typed registration refusal keeps its protocol backoff. An + // outage (a Server warming up or down, a refused or reset + // connection, an empty reply, a bodiless 502/503/504 from a + // load balancer, #448) uses the shared jittered schedule. + // Registration is idempotent: Server 3.17.0 accepts the same + // worker ID registering again, so repeating it after a lost + // reply is safe. + $untypedOutage = !$this->isTransientRegistrationFailure($exception) + && TransientFailure::is($exception, acknowledgement: true); + if (!$untypedOutage && !$this->isTransientRegistrationFailure($exception)) { throw $exception; } ++$attempt; - $delaySeconds = $this->transientRetryDelay( - $attempt, - $exception->details['retry_after_seconds'] ?? null, - ); + $delaySeconds = $untypedOutage + ? TransientFailure::delay($attempt, $exception->details['retry_after_seconds'] ?? null) + : $this->transientRetryDelay($attempt, $exception->details['retry_after_seconds'] ?? null); if ($this->transientPollRetryObserver !== null) { ($this->transientPollRetryObserver)('registration', $attempt, $delaySeconds, $exception); } @@ -994,6 +1033,7 @@ private function registerWithRetry(): ?array 'delay_seconds' => $delaySeconds, 'exception' => $exception, ], 'warning'); + $this->observeOutage('registration', $exception); $this->waitForTransientRetry($delaySeconds); } } @@ -1001,6 +1041,30 @@ private function registerWithRetry(): ?array return null; } + /** + * Escalate a lasting outage: one `worker.outage_persisting` error once it + * has lasted 60 s, then one every 5 minutes, until a request succeeds. + */ + private function observeOutage(string $operation, Throwable $exception): void + { + $now = $this->now(); + $this->outageStartedAt ??= $now; + $lasted = $now - $this->outageStartedAt; + if ($lasted < self::OUTAGE_ERROR_AFTER_SECONDS || ($this->outageErrorLoggedAt !== null + && $now - $this->outageErrorLoggedAt < self::OUTAGE_ERROR_EVERY_SECONDS)) { + return; + } + $this->outageErrorLoggedAt = $now; + $this->diagnostic('worker.outage_persisting', ['worker_id' => $this->workerId, 'task_queue' => $this->taskQueue, + 'operation' => $operation, 'outage_seconds' => round($lasted, 1), 'exception' => $exception], 'error'); + } + + private function clearOutage(): void + { + $this->outageStartedAt = null; + $this->outageErrorLoggedAt = null; + } + private function isTransientRegistrationFailure(ServerException $exception): bool { if ($this->isTransientDatabaseFailure($exception, 'register_worker') @@ -1422,10 +1486,16 @@ private function isPollCapacityRefusal(ServerException $error): bool private function waitForPollRetry(string $kind, int $attempt, ServerException $error): void { - $delay = $this->transientRetryDelay($attempt, $error->details['retry_after_seconds'] ?? null); + $retryAfter = $error->details['retry_after_seconds'] ?? null; + // Typed refusals keep their protocol backoff; an untyped outage uses + // the shared jittered schedule so Workers do not retry in step. + $delay = !PollResponse::isTransientFailure($error) && TransientFailure::is($error) + ? TransientFailure::delay($attempt, $retryAfter) + : $this->transientRetryDelay($attempt, $retryAfter); ($this->transientPollRetryObserver)?->__invoke($kind, $attempt, $delay, $error); $this->diagnostic('worker.retrying', ['worker_id' => $this->workerId, 'operation' => "{$kind}_poll", 'attempt' => $attempt, 'delay_seconds' => $delay, 'exception' => $error], 'warning'); + $this->observeOutage("{$kind}_poll", $error); $this->waitForTransientRetry($delay); } @@ -1439,12 +1509,14 @@ private function pollFailurePolicy(string $kind, string $requestId, ServerExcept 'backend_unavailable' => $database, 'storage_pressure', 'storage_admission_unavailable' => $storage, 'workflow_poll_normalization_pending' => $normalization, - default => PollResponse::isTransientFailure($error), + default => PollResponse::isTransientFailure($error) || TransientFailure::is($error), }; return ['retry' => $retry, 'retain' => $database || $storage || $normalization || $error->isTransientConnectionFailure() || $error->isTransientUpstreamFailure() - || PollResponse::isTransientPayloadFailure($error)]; + || PollResponse::isTransientPayloadFailure($error) + // An untyped outage committed nothing; a typed refusal keeps its own identity rule. + || (!PollResponse::isTransientFailure($error) && TransientFailure::is($error))]; } /** @@ -1500,16 +1572,32 @@ private function retryStorageAdmission( $activityTaskLease['activity_attempt_id'], $activityTaskLease['lease_owner'], ); - if ($uncertainActivityCompletion && $activityTaskLease !== null + // Only this Worker can complete this attempt as its lease owner, + // so this exact refusal proves our completion was committed, + // even on the first answer: cURL can resend a POST whose reply + // was lost on a reused connection before the Worker sees it. + if ($operation === 'activity_complete' && $activityTaskLease !== null && $this->isCommittedActivityCompletion($exception, $activityTaskLease)) { + if (!$uncertainActivityCompletion) { + $this->diagnostic('worker.completion_already_committed', ['worker_id' => $this->workerId, + 'task_id' => $activityTaskLease['task_id'], + 'activity_attempt_id' => $activityTaskLease['activity_attempt_id']], 'info'); + } + return null; } $backendUnavailable = $workflowBackendUnavailable || $activityBackendUnavailable; // These requests retain their original lease fence and computed // outcome. An uncertain reply retries the request, never the handler. + // An untyped outage (#448), such as 503 storage_unavailable, is + // just as uncertain: the Server may have committed the request. + $untypedOutage = !$exception->isStorageAdmissionFailure() && !$backendUnavailable + && !$exception->isTransientConnectionFailure() && !$exception->isTransientUpstreamFailure() + && TransientFailure::is($exception, acknowledgement: true); $claimedTaskTransportFailure = ($workflowTaskLease !== null || ($operation === 'activity_complete' && $activityTaskLease !== null)) - && ($exception->isTransientConnectionFailure() || $exception->isTransientUpstreamFailure()); + && ($exception->isTransientConnectionFailure() || $exception->isTransientUpstreamFailure() + || $untypedOutage); if ((!$exception->isStorageAdmissionFailure() && !$backendUnavailable && !$activityDiscoveryUnavailable && !$workflowCompletionDeferred && !$claimedTaskTransportFailure) || $this->shutdownRequested) { @@ -1520,9 +1608,25 @@ private function retryStorageAdmission( } $uncertainActivityCompletion = $uncertainActivityCompletion || $activityBackendUnavailable || ($operation === 'activity_complete' && $claimedTaskTransportFailure); + if ($untypedOutage && $attempt >= $this->untypedOutageAttemptLimit($operation)) { + // Retries stay inside the claim: with lease maintenance the + // publication fence ends them when the grant ends; without + // it, a bounded number of attempts does. The Server then + // settles the attempt (lease expiry and re-delivery). + $this->diagnostic('worker.outcome_retry_exhausted', ['worker_id' => $this->workerId, + 'operation' => $operation, 'attempts' => $attempt + 1, 'exception' => $exception], 'error'); + throw $exception; + } ++$attempt; - $delaySeconds = $this->transientRetryDelay($attempt, $exception->details['retry_after_seconds'] ?? null); + $retryAfter = $exception->details['retry_after_seconds'] ?? null; + // An untyped outage uses the shared jittered schedule; typed + // refusals keep their protocol backoff. The lease fence still + // bounds every retry: once the claim's grant ends, publication + // is refused and the Server settles the attempt. + $delaySeconds = $untypedOutage + ? TransientFailure::delay($attempt, $retryAfter) + : $this->transientRetryDelay($attempt, $retryAfter); if ($this->transientPollRetryObserver !== null) { ($this->transientPollRetryObserver)($operation, $attempt, $delaySeconds, $exception); } @@ -1545,6 +1649,43 @@ private function retryStorageAdmission( } } + /** + * Repeat an idempotent read through an untyped outage, using the shared + * jittered schedule, until it succeeds, fails otherwise, or shutdown. + * + * @template T + * @param \Closure(): T $call + * @return T + */ + private function retryUntypedOutage(string $operation, \Closure $call): mixed + { + $attempt = 0; + while (true) { + try { + return $call(); + } catch (ServerException $exception) { + if (!TransientFailure::is($exception) || $this->shutdownRequested) { + throw $exception; + } + $delay = TransientFailure::delay(++$attempt, $exception->details['retry_after_seconds'] ?? null); + $this->diagnostic('worker.retrying', ['worker_id' => $this->workerId, 'operation' => $operation, + 'attempt' => $attempt, 'delay_seconds' => $delay, 'exception' => $exception], 'warning'); + $this->waitForTransientRetry($delay); + if ($this->shutdownRequested) { + throw $exception; + } + } + } + } + + /** Retries of one outcome through an untyped outage before the Server's lease handling takes over. */ + private function untypedOutageAttemptLimit(string $operation): int + { + // A maintained activity lease fences publication at the grant's end. + return str_starts_with($operation, 'activity_') && $this->activityLeaseProcess !== null + ? PHP_INT_MAX : self::UNTYPED_OUTAGE_MAX_ATTEMPTS; + } + /** @param array{task_id: string, activity_attempt_id: string, lease_owner: string} $lease */ private function isCommittedActivityCompletion(ServerException $exception, array $lease): bool { @@ -3376,6 +3517,7 @@ private function executeActivityTask(array $task): void $activityType = (string) ($task['activity_type'] ?? ''); $callbackStopped = false; $callbackStarted = false; + $resultComputed = false; try { $this->trackWorkerSessionFromTask($task); $handler = $this->activities[$activityType] ?? null; @@ -3448,6 +3590,7 @@ static function () use (&$callbackStopped): void { $callbackStopped = true; }, $result = $this->invokeActivity($handler, $context, $arguments, ExecutionTelemetry::taskFields($task, $this->client->namespace, $this->taskQueue), authorizedTask: $task); } + $resultComputed = true; $carried = $this->reserveWorkflowPoll($leaseOwner); $continuation = $carried === null ? null : $this->speculateActivityContinuation($task, $result); $completion = $this->retryStorageAdmission( @@ -3513,6 +3656,16 @@ function () use ($taskId, $attemptId, $leaseOwner, $result, $carried, &$continua if ($exception instanceof ServerException && $this->discardClosedActivityAttempt($task, $exception)) { return; } + if ($resultComputed && TransientFailure::is($exception, acknowledgement: true)) { + // An outage outlasted publishing a computed result. It is not + // an activity failure: leave the attempt to the Server, which + // re-delivers it when the lease ends (a committed completion + // is answered as a duplicate then). + $this->diagnostic('worker.outcome_unpublished', ['task_id' => $taskId, + 'activity_attempt_id' => $attemptId, 'task_kind' => 'activity', 'exception' => $exception], 'error'); + + return; + } if ($this->activityLeaseProcess !== null && !$exception instanceof WorkflowClaimRevoked) { try { $this->activityLeaseProcess->check(); @@ -4800,7 +4953,13 @@ private function heartbeat(?RequestBudget $budget = null): void 'activity_available' => $this->carriedActivityPoll === null ? 1 : 0, ], $budget); } catch (ServerException $exception) { - if (!$exception->isTransientConnectionFailure() + $untypedOutage = !$exception->isTransientConnectionFailure() + && !$exception->isTransientUpstreamFailure() + && !$this->isTransientDatabaseFailure($exception, 'heartbeat_worker') + && !$exception->isStorageAdmissionFailure() + && TransientFailure::is($exception); + if (!$untypedOutage + && !$exception->isTransientConnectionFailure() && !$exception->isTransientUpstreamFailure() && !$this->isTransientDatabaseFailure($exception, 'heartbeat_worker') && !$exception->isStorageAdmissionFailure()) { @@ -4809,8 +4968,11 @@ private function heartbeat(?RequestBudget $budget = null): void // Do not enter the heartbeat-aware wait recursively or delay an // already leased task. The next poll/wait services this deadline. + // An outage during a retry wait therefore never ends run() (#448). $attempt = ++$this->heartbeatRetryAttempt; - $delaySeconds = $this->transientRetryDelay($attempt, $exception->details['retry_after_seconds'] ?? null); + $delaySeconds = $untypedOutage + ? TransientFailure::delay($attempt, $exception->details['retry_after_seconds'] ?? null) + : $this->transientRetryDelay($attempt, $exception->details['retry_after_seconds'] ?? null); $this->heartbeatRetryAt = $this->now() + $delaySeconds; if ($this->transientPollRetryObserver !== null) { ($this->transientPollRetryObserver)('heartbeat', $attempt, $delaySeconds, $exception); diff --git a/src/Worker/TransientFailure.php b/src/Worker/TransientFailure.php new file mode 100644 index 00000000..4e91cd1b --- /dev/null +++ b/src/Worker/TransientFailure.php @@ -0,0 +1,84 @@ +isTransientConnectionFailure() || $error->isTransientUpstreamFailure()) { + return true; + } + + return self::transientResponse($error->status, $error->reason, $error->details, $acknowledgement); + } + if ($error instanceof TransportException) { + if ($error->transientConnectionFailure) { + return true; + } + $reason = is_string($error->response['reason'] ?? null) ? $error->response['reason'] : null; + + return $error->status !== null && self::transientResponse($error->status, $reason, $error->response, $acknowledgement); + } + + return false; + } + + /** Full-jitter delay for the given 1-based attempt, never shorter than a bounded retry_after. */ + public static function delay(int $attempt, mixed $retryAfterSeconds = null): float + { + $ceiling = min(self::MAX_SECONDS, self::FIRST_SECONDS * 2 ** min(max(0, $attempt - 1), 16)); + $delay = $ceiling * random_int(0, 1_000_000) / 1_000_000; + + return is_int($retryAfterSeconds) && $retryAfterSeconds > 0 + ? max($delay, (float) min($retryAfterSeconds, self::MAX_RETRY_AFTER_SECONDS)) : $delay; + } + + /** @param array|null $details */ + private static function transientResponse(int $status, ?string $reason, ?array $details, bool $acknowledgement): bool + { + $retryable = is_array($details) ? ($details['retryable'] ?? null) : null; + if ($retryable === false || !($status === 429 || ($status >= 500 && $status <= 599))) { + return false; + } + + if ($status === 503 && in_array($reason, self::REASONS, true)) { + return true; + } + + return !$acknowledgement && $retryable === true && ($reason === null || $reason === ''); + } +} diff --git a/tests/WorkerConnectionRecoveryTest.php b/tests/WorkerConnectionRecoveryTest.php index a48cfe3a..fb3986e0 100644 --- a/tests/WorkerConnectionRecoveryTest.php +++ b/tests/WorkerConnectionRecoveryTest.php @@ -354,18 +354,26 @@ public static function terminalFailures(): iterable } } - public function testAmbiguousRegistrationIsNotRepeated(): void + /** + * Registration is idempotent: Server 3.17.0 accepts the same worker ID + * registering again (both answers `registered: true`), so a registration + * whose reply was lost is repeated with its original identity (#448). + */ + public function testRegistrationAfterALostConnectionIsRepeatedWithItsIdentity(): void { - $transport = new FakeTransport([self::disconnected()]); - $worker = new Worker(new Client('https://server.example', transport: $transport), 'orders'); + $transport = new FakeTransport([ + self::disconnected(), + ['registered' => true], + ['task' => null, 'poll_status' => 'stopped', 'reason' => 'worker_stopped'], + ]); + $worker = new Worker(new Client('https://server.example', transport: $transport), 'orders', + sleeper: static function (int $microseconds): void {}); - try { - $worker->run(0); - self::fail('Registration requires explicit reconciliation, not blind retry.'); - } catch (ServerException $exception) { - self::assertTrue($exception->isTransientConnectionFailure()); - } - self::assertCount(1, $transport->requests); + $worker->run(0); + + self::assertStringEndsWith('/api/worker/register', $transport->requests[0]['uri']); + self::assertStringEndsWith('/api/worker/register', $transport->requests[1]['uri']); + self::assertSame($transport->requests[0]['body'], $transport->requests[1]['body']); } public function testAnActivityCompletionConnectionFailureDoesNotRepeatTheHandler(): void diff --git a/tests/WorkerTransientOutageTest.php b/tests/WorkerTransientOutageTest.php new file mode 100644 index 00000000..023bfd97 --- /dev/null +++ b/tests/WorkerTransientOutageTest.php @@ -0,0 +1,541 @@ + 'storage_unavailable', 'message' => 'storage_unavailable']; + + return TransportException::fromResponse(503, $body, json_encode($body, JSON_THROW_ON_ERROR)); + } + + /** @return array{0: Worker, 1: list, 2: object} */ + private function worker(\Closure $handler): array + { + $now = 0.0; + $events = []; + $state = new class { public float $slept = 0.0; public int $calls = 0; }; + $worker = new Worker( + new Client('https://server.example', transport: new FakeTransport(handler: $handler)), + 'orders', + workerId: 'worker-1', + clock: static function () use (&$now): float { + return $now; + }, + sleeper: static function (int $microseconds) use (&$now, $state): void { + $now += $microseconds / 1_000_000; + $state->slept += $microseconds / 1_000_000; + }, + diagnosticListener: static function (string $event, array $context) use (&$events): void { + $events[] = $event; + }, + ); + $worker->registerActivity('orders.charge', static function (ActivityContext $context) use ($state): string { + ++$state->calls; + + return 'charged'; + }); + + return [$worker, $events, $state]; + } + + public function testStorageUnavailablePollsAreRetriedWithJitterAndTheWorkerKeepsRunning(): void + { + $workflowPolls = 0; + $completions = 0; + $events = []; + $state = null; + $handler = static function (string $method, string $uri, array $headers, ?array $body) use (&$workflowPolls, &$completions): ?array { + if (str_ends_with($uri, '/api/worker/register')) { + return ['registered' => true]; + } + if (str_ends_with($uri, '/api/worker/workflow-tasks/poll')) { + if (++$workflowPolls <= 3) { + throw self::storageUnavailable(); + } + + return ['task' => null, 'poll_status' => 'empty']; + } + if (str_ends_with($uri, '/api/worker/activity-tasks/poll')) { + return ['poll_status' => 'leased', 'task' => ['task_id' => 'activity-1', 'activity_attempt_id' => 'attempt-1', + 'lease_owner' => 'worker-1', 'activity_type' => 'orders.charge', 'payload_codec' => 'avro']]; + } + if (str_ends_with($uri, '/api/worker/activity-tasks/activity-1/complete')) { + ++$completions; + + return ['completed' => true]; + } + if (str_ends_with($uri, '/api/worker/query-tasks/poll')) { + return ['task' => null, 'poll_status' => 'stopped', 'reason' => 'worker_stopped']; + } + self::fail("Unexpected worker request: {$method} {$uri}"); + }; + [$worker, , $state] = $this->worker($handler); + + $worker->run(0); + + self::assertSame(4, $workflowPolls, 'Three 503 storage_unavailable answers, then a poll that succeeds.'); + self::assertSame(1, $state->calls); + self::assertSame(1, $completions); + // Jittered: each wait is at most 0.25, 0.5 and 1 s. + self::assertLessThanOrEqual(1.75, $state->slept); + } + + public function testAnOutageThatOutlastsPublishingAResultIsNeverReportedAsAFailure(): void + { + // The activity's completion keeps meeting the outage until its retries + // are spent. The result is left to the Server (lease expiry and + // re-delivery), never turned into an activity failure report. + $workflowPolls = 0; + $activityPolls = 0; + $reports = []; + $handler = static function (string $method, string $uri, array $headers, ?array $body) use (&$workflowPolls, &$activityPolls, &$reports): ?array { + if (str_ends_with($uri, '/api/worker/register')) { + return ['registered' => true]; + } + if (str_ends_with($uri, '/api/worker/workflow-tasks/poll')) { + ++$workflowPolls; + + return ['task' => null, 'poll_status' => 'empty']; + } + if (str_ends_with($uri, '/api/worker/activity-tasks/poll')) { + if (++$activityPolls === 1) { + return ['poll_status' => 'leased', 'task' => ['task_id' => 'activity-1', 'activity_attempt_id' => 'attempt-1', + 'lease_owner' => 'worker-1', 'activity_type' => 'orders.charge', 'payload_codec' => 'avro']]; + } + + return ['task' => null, 'poll_status' => 'empty']; + } + if (str_contains($uri, '/api/worker/activity-tasks/activity-1/')) { + $reports[] = substr($uri, strrpos($uri, '/') + 1); + throw self::storageUnavailable(); + } + if (str_ends_with($uri, '/api/worker/query-tasks/poll')) { + return $workflowPolls >= 2 ? ['task' => null, 'poll_status' => 'stopped', 'reason' => 'worker_stopped'] + : ['task' => null, 'poll_status' => 'empty']; + } + self::fail("Unexpected worker request: {$method} {$uri}"); + }; + $events = []; + $now = 0.0; + $worker = new Worker(new Client('https://server.example', transport: new FakeTransport(handler: $handler)), 'orders', + workerId: 'worker-1', + clock: static function () use (&$now): float { return $now; }, + sleeper: static function (int $microseconds) use (&$now): void { $now += $microseconds / 1_000_000; }, + diagnosticListener: static function (string $event, array $context) use (&$events): void { $events[] = $event; }); + $worker->registerActivity('orders.charge', static fn (ActivityContext $context): string => 'charged'); + + $worker->run(0); + + self::assertSame(array_fill(0, 9, 'complete'), $reports, 'The first try and eight bounded retries, never a failure report.'); + self::assertContains('worker.outcome_retry_exhausted', $events); + self::assertContains('worker.outcome_unpublished', $events); + self::assertGreaterThanOrEqual(2, $workflowPolls, 'The Worker kept polling after the outage.'); + } + + public function testAHeartbeatOutageBetweenTicksIsDeferredAndTheWorkerKeepsRunning(): void + { + // A worker heartbeat meets the outage after a tick. It is deferred on + // the shared schedule; it never ends run(). + $now = 0.0; + $workflowPolls = 0; + $heartbeats = 0; + $handler = static function (string $method, string $uri) use (&$now, &$workflowPolls, &$heartbeats): ?array { + if (str_ends_with($uri, '/api/worker/register')) { + return ['registered' => true, 'heartbeat_interval_seconds' => 1]; + } + if (str_ends_with($uri, '/api/worker/workflow-tasks/poll')) { + ++$workflowPolls; + $now += 2.0; + + return ['task' => null, 'poll_status' => 'empty']; + } + if (str_ends_with($uri, '/api/worker/heartbeat')) { + if (++$heartbeats === 1) { + throw self::storageUnavailable(); + } + + return ['worker_id' => 'worker-1', 'acknowledged' => true, 'heartbeat_interval_seconds' => 1]; + } + if (str_ends_with($uri, '/poll')) { + return $workflowPolls >= 3 && str_ends_with($uri, '/query-tasks/poll') + ? ['task' => null, 'poll_status' => 'stopped', 'reason' => 'worker_stopped'] + : ['task' => null, 'poll_status' => 'empty']; + } + self::fail("Unexpected worker request: {$method} {$uri}"); + }; + $events = []; + $worker = new Worker(new Client('https://server.example', transport: new FakeTransport(handler: $handler)), 'orders', + workerId: 'worker-1', + clock: static function () use (&$now): float { return $now; }, + sleeper: static function (int $microseconds) use (&$now): void { $now += $microseconds / 1_000_000; }, + diagnosticListener: static function (string $event, array $context) use (&$events): void { $events[] = $event; }); + + $worker->run(0); + + self::assertGreaterThanOrEqual(2, $heartbeats, 'The deferred heartbeat was sent again.'); + self::assertContains('worker.retrying', $events); + self::assertNotContains('worker.fatal_failure', $events); + self::assertGreaterThanOrEqual(3, $workflowPolls, 'The Worker kept polling after the outage.'); + } + + /** Review HIGH 1: the original incident, a Server warming up while the Worker registers. */ + public function testStorageUnavailableDuringRegistrationIsRetried(): void + { + $registrations = 0; + $handler = static function (string $method, string $uri) use (&$registrations): ?array { + if (str_ends_with($uri, '/api/worker/register')) { + if (++$registrations <= 3) { + throw self::storageUnavailable(); + } + + return ['registered' => true]; + } + if (str_ends_with($uri, '/query-tasks/poll')) { + return ['task' => null, 'poll_status' => 'stopped', 'reason' => 'worker_stopped']; + } + if (str_ends_with($uri, '/poll')) { + return ['task' => null, 'poll_status' => 'empty']; + } + self::fail("Unexpected worker request: {$method} {$uri}"); + }; + [$worker, , $state] = $this->worker($handler); + + $worker->run(0); + + self::assertSame(4, $registrations, 'Three 503 storage_unavailable answers, then registration succeeds.'); + self::assertLessThanOrEqual(1.75, $state->slept, 'Jittered waits of at most 0.25, 0.5 and 1 s.'); + } + + /** @return iterable */ + public static function startupOutages(): iterable + { + yield 'server down: connection refused' => [new TransportException('Connection refused', transientConnectionFailure: true)]; + yield 'empty reply from the server' => [new TransportException('Empty reply from server', transientConnectionFailure: true)]; + yield 'load balancer: bare 502' => [TransportException::fromResponse(502, null, '')]; + yield 'load balancer: bare 504' => [TransportException::fromResponse(504, null, '')]; + } + + /** A Worker started while the Server (or its load balancer) is down waits for it instead of exiting. */ + #[DataProvider('startupOutages')] + public function testAWorkerStartedDuringAnOutageWaitsForTheServer(TransportException $outage): void + { + $registrations = []; + $handler = static function (string $method, string $uri, array $headers, ?array $body) use (&$registrations, $outage): ?array { + if (str_ends_with($uri, '/api/worker/register')) { + $registrations[] = $body; + if (count($registrations) <= 4) { + throw $outage; + } + + return ['registered' => true]; + } + if (str_ends_with($uri, '/query-tasks/poll')) { + return ['task' => null, 'poll_status' => 'stopped', 'reason' => 'worker_stopped']; + } + if (str_ends_with($uri, '/poll')) { + return ['task' => null, 'poll_status' => 'empty']; + } + self::fail("Unexpected worker request: {$method} {$uri}"); + }; + [$worker] = $this->worker($handler); + + $worker->run(0); + + self::assertCount(5, $registrations); + self::assertCount(1, array_unique(array_map('serialize', $registrations)), 'Repeated with its original identity.'); + } + + public function testALastingOutageIsReportedAsAnErrorAfterAMinuteThenEveryFiveMinutes(): void + { + $now = 0.0; + $registrations = 0; + $handler = static function (string $method, string $uri) use (&$now, &$registrations): ?array { + if (str_ends_with($uri, '/api/worker/register')) { + ++$registrations; + // Down for about 12 minutes of simulated time. + if ($now < 720.0) { + $now += 5.0; + throw new TransportException('Connection refused', transientConnectionFailure: true); + } + + return ['registered' => true]; + } + if (str_ends_with($uri, '/query-tasks/poll')) { + return ['task' => null, 'poll_status' => 'stopped', 'reason' => 'worker_stopped']; + } + if (str_ends_with($uri, '/poll')) { + return ['task' => null, 'poll_status' => 'empty']; + } + self::fail("Unexpected worker request: {$method} {$uri}"); + }; + $errors = []; + $worker = new Worker(new Client('https://server.example', transport: new FakeTransport(handler: $handler)), 'orders', + workerId: 'worker-1', + clock: static function () use (&$now): float { return $now; }, + sleeper: static function (int $microseconds) use (&$now): void { $now += $microseconds / 1_000_000; }, + diagnosticListener: static function (string $event, array $context) use (&$errors): void { + if ($event === 'worker.outage_persisting') { + $errors[] = $context['outage_seconds']; + } + }); + + $worker->run(0); + + self::assertCount(3, $errors, 'At about 60 s, then about every 5 minutes: '.json_encode($errors)); + self::assertGreaterThanOrEqual(60.0, $errors[0]); + self::assertLessThan(75.0, $errors[0]); + self::assertGreaterThanOrEqual(300.0, $errors[1] - $errors[0]); + self::assertGreaterThanOrEqual(300.0, $errors[2] - $errors[1]); + } + + /** Review HIGH 2: a failing heartbeat during a poll retry wait must not end run(). */ + public function testAnOutageThatAlsoFailsTheHeartbeatDuringARetryWaitDoesNotEndTheRun(): void + { + $now = 0.0; + $workflowPolls = 0; + $heartbeats = 0; + $handler = static function (string $method, string $uri) use (&$workflowPolls, &$heartbeats): ?array { + if (str_ends_with($uri, '/api/worker/register')) { + return ['registered' => true, 'heartbeat_interval_seconds' => 1]; + } + if (str_ends_with($uri, '/api/worker/heartbeat')) { + if (++$heartbeats <= 3) { + throw self::storageUnavailable(); + } + + return ['worker_id' => 'worker-1', 'acknowledged' => true, 'heartbeat_interval_seconds' => 1]; + } + if (str_ends_with($uri, '/workflow-tasks/poll')) { + // A 12-attempt outage: long enough for heartbeats to fall due inside the retry waits. + if (++$workflowPolls <= 12) { + throw self::storageUnavailable(); + } + + return ['task' => null, 'poll_status' => 'empty']; + } + if (str_ends_with($uri, '/query-tasks/poll')) { + return ['task' => null, 'poll_status' => 'stopped', 'reason' => 'worker_stopped']; + } + if (str_ends_with($uri, '/poll')) { + return ['task' => null, 'poll_status' => 'empty']; + } + self::fail("Unexpected worker request: {$method} {$uri}"); + }; + $events = []; + $worker = new Worker(new Client('https://server.example', transport: new FakeTransport(handler: $handler)), 'orders', + workerId: 'worker-1', + clock: static function () use (&$now): float { return $now; }, + sleeper: static function (int $microseconds) use (&$now): void { $now += $microseconds / 1_000_000; }, + diagnosticListener: static function (string $event, array $context) use (&$events): void { $events[] = $event; }); + + $worker->run(0); + + self::assertSame(13, $workflowPolls); + self::assertGreaterThanOrEqual(1, $heartbeats, 'A heartbeat fell due inside a retry wait and met the outage.'); + self::assertNotContains('worker.fatal_failure', $events); + } + + /** + * Review HIGH 3: cURL resent a completion whose reply was lost on a reused + * connection, and the first answer the Worker sees is the Server's + * duplicate: 409 stale_attempt, recorded false, every status completed. + * That is our own committed completion, not an error. + */ + public function testACommittedCompletionReportedOnTheFirstAnswerIsOurs(): void + { + $completions = 0; + $failures = 0; + $duplicate = ['task_id' => 'activity-1', 'activity_attempt_id' => 'attempt-1', 'outcome' => 'completed', + 'recorded' => false, 'reason' => 'stale_attempt', 'next_task_id' => null, 'lease_owner' => 'worker-1', + 'activity_status' => 'completed', 'attempt_status' => 'completed', 'task_status' => 'completed']; + $handler = static function (string $method, string $uri) use (&$completions, &$failures, $duplicate): ?array { + if (str_ends_with($uri, '/api/worker/register')) { + return ['registered' => true]; + } + if (str_ends_with($uri, '/activity-tasks/poll')) { + return $completions === 0 ? ['poll_status' => 'leased', 'task' => ['task_id' => 'activity-1', + 'activity_attempt_id' => 'attempt-1', 'lease_owner' => 'worker-1', 'activity_type' => 'orders.charge', + 'payload_codec' => 'avro']] : ['task' => null, 'poll_status' => 'empty']; + } + if (str_ends_with($uri, '/activity-1/complete')) { + ++$completions; + throw TransportException::fromResponse(409, $duplicate, json_encode($duplicate, JSON_THROW_ON_ERROR)); + } + if (str_ends_with($uri, '/activity-1/fail')) { + ++$failures; + + return ['recorded' => true]; + } + if (str_ends_with($uri, '/query-tasks/poll')) { + return ['task' => null, 'poll_status' => 'stopped', 'reason' => 'worker_stopped']; + } + if (str_ends_with($uri, '/poll')) { + return ['task' => null, 'poll_status' => 'empty']; + } + self::fail("Unexpected worker request: {$method} {$uri}"); + }; + $events = []; + $worker = new Worker(new Client('https://server.example', transport: new FakeTransport(handler: $handler)), 'orders', + workerId: 'worker-1', sleeper: static function (int $microseconds): void {}, + diagnosticListener: static function (string $event, array $context) use (&$events): void { $events[] = $event; }); + $worker->registerActivity('orders.charge', static fn (ActivityContext $context): string => 'charged'); + + $worker->run(0); + + self::assertSame(1, $completions, 'Not resent: the answer already proves the completion committed.'); + self::assertSame(0, $failures); + self::assertContains('worker.completion_already_committed', $events); + self::assertNotContains('worker.fatal_failure', $events); + } + + /** + * The Server commits the completion but the Worker sees 503 storage_unavailable. + * The retried completion is answered as Server 3.17.0 answers a duplicate, and + * the Worker treats it as its own committed completion: no failure report. + */ + #[DataProvider('completionRetryAnswers')] + public function testACompletionMetByAnOutageIsRetriedAndADuplicateCountsAsCommitted(array|TransportException $second): void + { + $completions = []; + $failures = 0; + $handler = static function (string $method, string $uri, array $headers, ?array $body) use (&$completions, &$failures, $second): ?array { + if (str_ends_with($uri, '/api/worker/register')) { + return ['registered' => true]; + } + if (str_ends_with($uri, '/api/worker/workflow-tasks/poll')) { + return ['task' => null, 'poll_status' => 'empty']; + } + if (str_ends_with($uri, '/api/worker/activity-tasks/poll')) { + return $completions === [] ? ['poll_status' => 'leased', 'task' => ['task_id' => 'activity-1', + 'activity_attempt_id' => 'attempt-1', 'lease_owner' => 'worker-1', 'activity_type' => 'orders.charge', + 'payload_codec' => 'avro']] : ['task' => null, 'poll_status' => 'empty']; + } + if (str_ends_with($uri, '/api/worker/activity-tasks/activity-1/complete')) { + $completions[] = $body; + if (count($completions) === 1) { + throw self::storageUnavailable(); + } + if ($second instanceof TransportException) { + throw $second; + } + + return $second; + } + if (str_ends_with($uri, '/api/worker/activity-tasks/activity-1/fail')) { + ++$failures; + + return ['recorded' => true]; + } + if (str_ends_with($uri, '/api/worker/query-tasks/poll')) { + return ['task' => null, 'poll_status' => 'stopped', 'reason' => 'worker_stopped']; + } + self::fail("Unexpected worker request: {$method} {$uri}"); + }; + [$worker, , $state] = $this->worker($handler); + + $worker->run(0); + + self::assertSame(1, $state->calls, 'The handler ran once; only the request was retried.'); + self::assertCount(2, $completions); + self::assertSame($completions[0], $completions[1], 'The same completion, with the same attempt fence.'); + self::assertSame(0, $failures, 'An outage never turns a computed result into a failure report.'); + } + + /** @return iterable|TransportException}> */ + public static function completionRetryAnswers(): iterable + { + yield 'first answer was lost; retry recorded' => [['task_id' => 'activity-1', 'activity_attempt_id' => 'attempt-1', + 'outcome' => 'completed', 'recorded' => true, 'reason' => null]]; + $duplicate = ['task_id' => 'activity-1', 'activity_attempt_id' => 'attempt-1', 'outcome' => 'completed', + 'recorded' => false, 'reason' => 'stale_attempt', 'next_task_id' => null, 'lease_owner' => 'worker-1', + 'activity_status' => 'completed', 'attempt_status' => 'completed', 'task_status' => 'completed']; + yield 'first was committed; duplicate answered as Server 3.17.0 does' => [ + TransportException::fromResponse(409, $duplicate, json_encode($duplicate, JSON_THROW_ON_ERROR))]; + } + + /** @return iterable */ + public static function fatalFailures(): iterable + { + yield 'authentication' => [TransportException::fromResponse(401, ['reason' => 'authentication_failed'], '{}'), 401]; + yield 'authorization' => [TransportException::fromResponse(403, ['reason' => 'forbidden'], '{}'), 403]; + yield 'explicitly not retryable' => [TransportException::fromResponse(503, + ['reason' => 'storage_unavailable', 'retryable' => false], '{}'), 503]; + yield 'protocol refusal' => [TransportException::fromResponse(400, ['reason' => 'unsupported_protocol_version'], '{}'), 400]; + } + + #[DataProvider('fatalFailures')] + public function testAuthenticationAndProtocolFailuresStopTheWorkerAndAreLogged(TransportException $failure, int $status): void + { + $handler = static function (string $method, string $uri) use ($failure): ?array { + if (str_ends_with($uri, '/api/worker/register')) { + return ['registered' => true]; + } + if (str_ends_with($uri, '/api/worker/deregister') || str_contains($uri, '/deregister')) { + return ['deregistered' => true]; + } + throw $failure; + }; + $events = []; + $worker = new Worker(new Client('https://server.example', transport: new FakeTransport(handler: $handler)), 'orders', + workerId: 'worker-1', sleeper: static function (int $microseconds): void {}, + diagnosticListener: static function (string $event, array $context) use (&$events): void { + $events[] = [$event, $context['status'] ?? null, $context['reason'] ?? null]; + }); + + try { + $worker->run(0); + self::fail('A fatal failure must stop the Worker.'); + } catch (ServerException $error) { + self::assertSame($status, $error->status); + } + $fatal = array_values(array_filter($events, static fn (array $event): bool => $event[0] === 'worker.fatal_failure')); + self::assertCount(1, $fatal, json_encode($events)); + self::assertSame($status, $fatal[0][1]); + self::assertSame([], array_values(array_filter($events, static fn (array $event): bool => $event[0] === 'worker.transient_failure'))); + } + + public function testClassificationAndJitterBounds(): void + { + self::assertTrue(TransientFailure::is(self::storageUnavailable())); + self::assertTrue(TransientFailure::is(new ServerException('x', 503, 'backend_unavailable', ['reason' => 'backend_unavailable']))); + self::assertTrue(TransientFailure::is(new ServerException('x', 502, null, ['retryable' => true]))); + // An unclassified answer cannot authorize repeating an acknowledgement. + self::assertFalse(TransientFailure::is(new ServerException('x', 503, null, ['retryable' => true]), acknowledgement: true)); + self::assertTrue(TransientFailure::is(self::storageUnavailable(), acknowledgement: true)); + // A typed refusal keeps its own protocol classification. + self::assertFalse(TransientFailure::is(new ServerException('x', 503, 'backend_lock_pressure', + ['reason' => 'backend_lock_pressure', 'retryable' => true, 'retry_after_seconds' => PHP_INT_MAX]))); + self::assertLessThanOrEqual(60.0, TransientFailure::delay(1, PHP_INT_MAX)); + self::assertGreaterThanOrEqual(3.0, TransientFailure::delay(1, 3)); + self::assertTrue(TransientFailure::is(new TransportException('reset', transientConnectionFailure: true))); + self::assertFalse(TransientFailure::is(new ServerException('x', 503, 'storage_unavailable', ['retryable' => false]))); + self::assertFalse(TransientFailure::is(new ServerException('x', 503, 'backend_lock_pressure', ['reason' => 'backend_lock_pressure']))); + self::assertFalse(TransientFailure::is(new ServerException('x', 401, 'authentication_failed'))); + self::assertFalse(TransientFailure::is(new ServerException('x', 409, 'stale_attempt'))); + self::assertFalse(TransientFailure::is(new \RuntimeException('boom'))); + foreach ([1 => 0.25, 2 => 0.5, 3 => 1.0, 4 => 2.0, 5 => 4.0, 9 => 4.0] as $attempt => $cap) { + for ($i = 0; $i < 50; ++$i) { + $delay = TransientFailure::delay($attempt); + self::assertGreaterThanOrEqual(0.0, $delay); + self::assertLessThanOrEqual($cap, $delay); + } + } + } +} diff --git a/tests/WorkerUpstreamRecoveryTest.php b/tests/WorkerUpstreamRecoveryTest.php index 6134534a..41bcd8f8 100644 --- a/tests/WorkerUpstreamRecoveryTest.php +++ b/tests/WorkerUpstreamRecoveryTest.php @@ -222,21 +222,27 @@ public static function terminalResponses(): iterable yield 'unknown structured error' => [503, '{"error":"refused"}']; } - public function testRegistrationDoesNotBlindlyRepeatAnUncertainRequest(): void + /** + * A load balancer answering a bodiless 503 at registration is an outage, + * and registration is idempotent (Server 3.17.0 accepts the same worker ID + * registering again), so the Worker waits and registers once the Server + * answers (#448). + */ + public function testRegistrationBehindAnUnavailableLoadBalancerIsRepeated(): void { - $calls = 0; - $worker = new Worker(self::client(static function () use (&$calls): ResponseInterface { - ++$calls; - return new Response(503, [], 'Unavailable'); + $registrations = 0; + $worker = new Worker(self::client(static function (RequestInterface $request) use (&$registrations): ResponseInterface { + $path = $request->getUri()->getPath(); + if (str_ends_with($path, '/worker/register')) { + return ++$registrations <= 2 ? new Response(503, [], 'Unavailable') : self::json(['registered' => true]); + } + + return self::json(['task' => null, 'poll_status' => 'stopped', 'reason' => 'worker_stopped']); }), 'orders'); - try { - $worker->run(0); - self::fail('Registration requires reconciliation, not blind retry.'); - } catch (ServerException $exception) { - self::assertSame(503, $exception->status); - self::assertTrue($exception->isTransientUpstreamFailure()); - self::assertSame(1, $calls); - } + + $worker->run(0); + + self::assertSame(3, $registrations); } #[DataProvider('upstreamStatuses')]