diff --git a/CHANGELOG.md b/CHANGELOG.md index 90483e89..66b9840f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,25 @@ project follows [Semantic Versioning](https://semver.org/). ### Fixed +- An outage of `GET /api/cluster/info` no longer stops a managed Worker. + Previously a plain 500 from it, read before a workflow-task completion for + the payload policy, ended `Worker::run()`. Cluster info is read-only, so a + refused, reset or timed-out connection and any 5xx or 429 (unless + `retryable: false`) are retried with the shared jittered schedule and + reported as `worker.outage_persisting` when they last. Startup discovery + waits like registration. The transport keeps the last discovered payload + policy through an outage and tries discovery again later. Without one, the + request that needed it is not sent and is retried. Authentication and + authorization refusals still stop the Worker. +- A completion refused for its size (`413 payload_too_large`) no longer stops + the Worker. The Server's inline limit can be lowered while the Worker holds + an older payload policy. The Worker drops that policy, reads it again, + encodes the same outcome under it (externalising what the new limit + requires) and sends it once more, without recomputing the result. A + discovery outage then is retried under the outage rules. An activity result + that is still too large is reported as a non-retryable failure whose + `failure.type` and `failure.class` are `ActivityResultTooLarge`, as in the + other SDKs. - A managed Worker no longer exits on a transient Server or network outage (#448). For example, native Server answers `503 storage_unavailable` while its database warms up. Polls retry untyped transients with the poll's diff --git a/docs/portal/build/workers.md b/docs/portal/build/workers.md index aa7b3215..e9fe045c 100644 --- a/docs/portal/build/workers.md +++ b/docs/portal/build/workers.md @@ -101,6 +101,10 @@ An untyped outage does not stop the Worker either. That covers a connection that **Startup and long outages.** A Worker started while the Server or its load balancer is unavailable keeps retrying registration on the same schedule instead of exiting. Registration is safe to repeat: the Server accepts the same worker ID registering again. An outage that lasts 60 s is reported as a `worker.outage_persisting` error, then every 5 minutes until a request succeeds. +**Cluster info.** Reads of `GET /api/cluster/info` are read-only, so any outage of them is retried: a refused, reset or timed-out connection, or any 5xx or 429 unless it says `retryable: false`. At startup, a Worker that needs discovery waits for it as it waits for registration. While running, the transport keeps the last payload policy it discovered and tries discovery again later. If it has none yet, the request that needed it is not sent and is retried. A lasting outage is reported as `worker.outage_persisting`. Authentication and authorization refusals stay fatal. + +**A lowered payload limit.** If the Server refuses a completion for its size (`413 payload_too_large`), for example because its inline limit was lowered after the Worker read it, the Worker reads the payload policy again, encodes the same outcome under it (externalising large values when storage is configured) and sends it once more. The result is never recomputed. An activity result that is still too large fails the attempt as a non-retryable `ActivityResultTooLarge`; the Worker keeps running. + **Completions.** A completion that meets an outage is retried with the same attempt fence on that schedule. With activity lease maintenance, the retries stop when the claim's grant ends; without it, they stop after a bounded number of attempts. Retrying is safe: the Server records one outcome per attempt. A retry of a completion the Server already committed is answered `409 stale_attempt` with `recorded: false` and the attempt `completed`, and the Worker treats that as its own committed completion. A result that cannot be published before the retries end is logged as `worker.outcome_unpublished` and left to the Server, which re-delivers the attempt when its lease ends. It is never reported as an activity failure. **Across SDKs.** PHP and Rust Workers stop on a 401 or 403 after logging it as an error. Python Workers keep retrying polls through a 401 or 403 so that credentials can be rotated without a restart, and log every refusal at error level. diff --git a/src/Client.php b/src/Client.php index fc49804e..53361c71 100644 --- a/src/Client.php +++ b/src/Client.php @@ -3472,24 +3472,37 @@ private function performRequest( $poll = str_ends_with($path, '/poll') && is_int($body['timeout_seconds'] ?? null); $pollSeconds = $poll ? max(0, min(60, $body['timeout_seconds'])) : 0; $budget ??= $worker && $this->boundedWorkerRequests ? new RequestBudget($pollSeconds + 5) : null; - if ($body !== null) { - $body = $this->failureReports->request($path, $body, false); - $body = $this->payloadUploads->request($body, $method, $path, $worker, $headers, $budget, - $this->failureReports); - $body = $this->failureReports->request($path, $body); - } - $activityLease = $worker ? $this->activityLeaseBinding->current() : null; - if ($activityLease !== null && $activityLease->governs($path)) { - // Uploading an external result is not publication authority. - // Check again before its final Worker POST and bound the send - // by the latest acknowledged grant, not an old cached lease. - $activityLease->checkRequest($body); - $budget?->restrictAuthorityDeadline($activityLease->publicationDeadline()); - } + $sourceBody = $body; + $body = $this->encodeRequestBody($sourceBody, $method, $path, $worker, $headers, $budget); try { - $response = $this->sendTransportRequest($method, $path, $worker, $headers, $body, $budget); - } finally { - $observation?->captureStatus($this->transport); + try { + $response = $this->sendTransportRequest($method, $path, $worker, $headers, $body, $budget); + } finally { + $observation?->captureStatus($this->transport); + } + } catch (TransportException $refusal) { + if (!$worker || $method !== 'POST' || $sourceBody === null || !self::isCompletion($path) + || !self::isPayloadSizeRefusal($refusal)) { + throw $refusal; + } + // The Server refused the body's size before reading it, so + // nothing was recorded. The payload policy this transport held + // may be older than the Server's limit: read it again and + // encode the same outcome under it (externalising what the new + // limit requires). Send once more only if that changes the + // body; otherwise the refusal stands. A discovery outage here + // fails as not sent, and the Worker repeats the request. + $this->payloadUploads->forgetPolicies(); + $reencoded = $this->encodeRequestBody($sourceBody, $method, $path, $worker, $headers, $budget); + if ($reencoded === $body) { + throw $refusal; + } + $body = $reencoded; + try { + $response = $this->sendTransportRequest($method, $path, $worker, $headers, $body, $budget); + } finally { + $observation?->captureStatus($this->transport); + } } $budget?->remainingSeconds(); @@ -3532,6 +3545,48 @@ private function performRequest( } } + /** + * Apply failure-report limits and the payload policy to a request body, + * then check an activity outcome against its current lease. + * + * @param array|null $body + * @param array $headers + * @return array|null + */ + private function encodeRequestBody(?array $body, string $method, string $path, bool $worker, array $headers, ?RequestBudget $budget): ?array + { + if ($body !== null) { + $body = $this->failureReports->request($path, $body, false); + $body = $this->payloadUploads->request($body, $method, $path, $worker, $headers, $budget, + $this->failureReports); + $body = $this->failureReports->request($path, $body); + } + $activityLease = $worker ? $this->activityLeaseBinding->current() : null; + if ($activityLease !== null && $activityLease->governs($path)) { + // Uploading an external result is not publication authority. + // Check again before its final Worker POST and bound the send + // by the latest acknowledged grant, not an old cached lease. + $activityLease->checkRequest($body); + $budget?->restrictAuthorityDeadline($activityLease->publicationDeadline()); + } + + return $body; + } + + /** Activity and workflow-task completions, whose outcome is already computed. */ + private static function isCompletion(string $path): bool + { + return preg_match('~\A/worker/(?:workflow-tasks/[^/]+/complete|activity-tasks/[^/]+/(?:continuation/)?complete)\z~', $path) === 1; + } + + /** The Server's structural size refusal (413 payload_too_large), answered before the body is read. */ + private static function isPayloadSizeRefusal(TransportException $refusal): bool + { + $details = $refusal->response; + + return $refusal->status === 413 && is_array($details) && ($details['reason'] ?? null) === 'payload_too_large'; + } + /** * Server 3.6.0 bounds these requests by its workflow-task completion limit, and * records each in one history transaction bounded by its history budget. diff --git a/src/Exception/ActivityResultTooLarge.php b/src/Exception/ActivityResultTooLarge.php new file mode 100644 index 00000000..999c8dc3 --- /dev/null +++ b/src/Exception/ActivityResultTooLarge.php @@ -0,0 +1,28 @@ +getMessage(), 0, $refusal); + } + + /** A size refusal of an activity result: the Server's 413, or an encoding the policy cannot carry. */ + public static function matches(\Throwable $error): bool + { + return $error instanceof ServerException && !$error instanceof WorkflowTaskCompletionTooLarge + && $error->status === 413 && in_array($error->reason, ['payload_too_large', 'external_payload_oversized'], true); + } +} diff --git a/src/Exception/ServerException.php b/src/Exception/ServerException.php index c4908875..27f8ac3c 100644 --- a/src/Exception/ServerException.php +++ b/src/Exception/ServerException.php @@ -159,6 +159,19 @@ public function isActivityTaskBackendUnavailable(string $taskId, string $attempt && $response['retry_after_seconds'] > 0; } + /** Runtime discovery (cluster info) failed before this request was sent, so it may be repeated. */ + public function isPayloadDiscoveryUnavailable(): bool + { + $details = $this->details; + + return $this instanceof ExternalPayloadException + && $this->status === 503 + && $this->reason === 'payload_discovery_unavailable' + && $details !== null && !array_is_list($details) + && ($details['request_admitted'] ?? null) === false + && ($details['retryable'] ?? null) === true; + } + /** Discovery failed before any activity-completion request was submitted. */ public function isActivityCompletionPayloadDiscoveryUnavailable(string $taskId, string $attemptId, string $leaseOwner): bool { diff --git a/src/Transport/RuntimePayloadUploads.php b/src/Transport/RuntimePayloadUploads.php index 4bfdee97..5801f685 100644 --- a/src/Transport/RuntimePayloadUploads.php +++ b/src/Transport/RuntimePayloadUploads.php @@ -8,6 +8,7 @@ use DurableWorkflow\Exception\TransportException; use DurableWorkflow\Version; use DurableWorkflow\Failure\FailureReportLimiter; +use DurableWorkflow\Worker\TransientFailure; /** Namespace-scoped discovery and request-local, content-addressed uploads. * @phpstan-type PayloadSlot array{path: list, blob: string} @@ -16,6 +17,9 @@ */ final class RuntimePayloadUploads { + /** How long a policy kept through a discovery outage is used before discovery is tried again. */ + private const STALE_POLICY_RETRY_SECONDS = 10; + private const COMPLETION_SCHEMA = 'durable-workflow.v2.payload-completion-context.v1'; private const PREPARED_COMPLETION_SCHEMA = 'durable-workflow.v2.payload-completion-context.v2'; private const COMPLETION_HEADER = 'X-Durable-Workflow-Payload-Completion'; @@ -143,6 +147,12 @@ private function continuationUploads(array $body, array $payloads, array $policy return $results; } + /** @internal Forget discovered policies; the next request reads cluster info again. */ + public function forgetPolicies(): void + { + $this->policies = []; + } + /** @param array $body * @param array $headers * @return UploadPolicy @@ -152,6 +162,15 @@ private function requestPolicy(array $body, string $path, bool $worker, array $h try { return $this->policy($headers, $worker, $budget); } catch (TransportException $exception) { + $key = (int) $worker; + if (isset($this->policies[$key]) && TransientFailure::isReadOutage($exception)) { + // Discovery is read-only and the Server validates every request, + // so an outage of it keeps the last policy this transport read. + // The read is tried again later, not before every request. + $this->policies[$key]['expires'] = time() + self::STALE_POLICY_RETRY_SECONDS; + + return $this->policies[$key]['policy']; + } $discovery = $exception->response; if ($worker && preg_match('~\A/worker/activity-tasks/([^/]+)/complete\z~', $path, $matches) === 1 @@ -184,6 +203,15 @@ private function requestPolicy(array $body, string $path, bool $worker, array $h $exception, ); } + if ($worker && TransientFailure::isReadOutage($exception)) { + // Nothing has been sent: the Worker may repeat this request + // once discovery answers again. + $retryAfter = is_array($discovery) ? ($discovery['retry_after_seconds'] ?? null) : null; + throw new ExternalPayloadException('Runtime payload policy discovery is temporarily unavailable.', 503, + 'payload_discovery_unavailable', ['reason' => 'payload_discovery_unavailable', 'operation' => 'cluster_info', + 'request_admitted' => false, 'retryable' => true, + 'retry_after_seconds' => is_int($retryAfter) && $retryAfter > 0 ? $retryAfter : 1], $exception); + } throw $exception; } diff --git a/src/Worker.php b/src/Worker.php index c9885489..8433ccec 100644 --- a/src/Worker.php +++ b/src/Worker.php @@ -6,6 +6,7 @@ use DurableWorkflow\Exception\ActivityCancelled; use DurableWorkflow\Exception\ActivityFailed; +use DurableWorkflow\Exception\ActivityResultTooLarge; use DurableWorkflow\Exception\CodecException; use DurableWorkflow\Exception\FailureReportRefused; use DurableWorkflow\Exception\InvalidLocalActivityReport; @@ -147,8 +148,10 @@ final class Worker private ?ActivityLeaseProcess $activityLeaseHelper = null; /** @var array|null The activity claim whose lease maintenance is starting. */ private ?array $claimingActivityTask = null; - private ?float $outageStartedAt = null; - private ?float $outageErrorLoggedAt = null; + /** @var array When each tracked outage began, by tracker. */ + private array $outageStartedAt = []; + /** @var array */ + private array $outageErrorLoggedAt = []; /** Monotonic end of that claim's conservative poll grant. */ private float $claimingActivityDeadline = 0; /** @var array{0: array, 1: string, 2: float}|null A stopped claim to release once its lease binding is gone. */ @@ -1045,24 +1048,28 @@ private function registerWithRetry(): ?array * Escalate a lasting outage: one `worker.outage_persisting` error once it * has lasted 60 s, then one every 5 minutes, until a request succeeds. */ - private function observeOutage(string $operation, Throwable $exception): void + private function observeOutage(string $operation, Throwable $exception, string $tracker = 'server'): void { $now = $this->now(); - $this->outageStartedAt ??= $now; - $lasted = $now - $this->outageStartedAt; - if ($lasted < self::OUTAGE_ERROR_AFTER_SECONDS || ($this->outageErrorLoggedAt !== null - && $now - $this->outageErrorLoggedAt < self::OUTAGE_ERROR_EVERY_SECONDS)) { + $this->outageStartedAt[$tracker] ??= $now; + $lasted = $now - $this->outageStartedAt[$tracker]; + if ($lasted < self::OUTAGE_ERROR_AFTER_SECONDS || (isset($this->outageErrorLoggedAt[$tracker]) + && $now - $this->outageErrorLoggedAt[$tracker] < self::OUTAGE_ERROR_EVERY_SECONDS)) { return; } - $this->outageErrorLoggedAt = $now; + $this->outageErrorLoggedAt[$tracker] = $now; $this->diagnostic('worker.outage_persisting', ['worker_id' => $this->workerId, 'task_queue' => $this->taskQueue, 'operation' => $operation, 'outage_seconds' => round($lasted, 1), 'exception' => $exception], 'error'); } - private function clearOutage(): void + /** + * End a tracked outage. Ordinary requests succeeding end the `server` + * outage; only a request that needed cluster info ends that one, because + * polls keep succeeding while discovery alone is down. + */ + private function clearOutage(string $tracker = 'server'): void { - $this->outageStartedAt = null; - $this->outageErrorLoggedAt = null; + unset($this->outageStartedAt[$tracker], $this->outageErrorLoggedAt[$tracker]); } private function isTransientRegistrationFailure(ServerException $exception): bool @@ -1543,7 +1550,13 @@ private function retryStorageAdmission( $this->assertCancellationDeadline(); } try { - return $request(); + $result = $request(); + if (str_ends_with($operation, '_complete')) { + // A completion got through, so its payload policy was available. + $this->clearOutage('cluster_info'); + } + + return $result; } catch (ServerException $exception) { $workflowCompletionDeferred = $operation === 'workflow_complete' && $workflowTaskLease !== null && ($exception->isWorkflowTaskCompletionDeferred( @@ -1592,8 +1605,12 @@ private function retryStorageAdmission( // An untyped outage (#448), such as 503 storage_unavailable, is // just as uncertain: the Server may have committed the request. $untypedOutage = !$exception->isStorageAdmissionFailure() && !$backendUnavailable + && !$activityDiscoveryUnavailable && !$exception->isTransientConnectionFailure() && !$exception->isTransientUpstreamFailure() && TransientFailure::is($exception, acknowledgement: true); + // Cluster info failed before this request was sent (see + // RuntimePayloadUploads); repeating it is safe. + $discoveryOutage = $untypedOutage && $exception->isPayloadDiscoveryUnavailable(); $claimedTaskTransportFailure = ($workflowTaskLease !== null || ($operation === 'activity_complete' && $activityTaskLease !== null)) && ($exception->isTransientConnectionFailure() || $exception->isTransientUpstreamFailure() @@ -1632,11 +1649,14 @@ private function retryStorageAdmission( } $this->diagnostic('worker.retrying', [ 'worker_id' => $this->workerId, - 'operation' => $operation, + 'operation' => $discoveryOutage ? 'cluster_info' : $operation, 'attempt' => $attempt, 'delay_seconds' => $delaySeconds, 'exception' => $exception, ], 'warning'); + if ($discoveryOutage) { + $this->observeOutage('cluster_info', $exception, 'cluster_info'); + } $this->waitForTransientRetry($delaySeconds); if ($this->shutdownRequested) { if ($workflowCompletionDeferred) { @@ -1650,8 +1670,9 @@ private function retryStorageAdmission( } /** - * Repeat an idempotent read through an untyped outage, using the shared - * jittered schedule, until it succeeds, fails otherwise, or shutdown. + * Repeat a read-only request (cluster info) through an outage, using the + * shared jittered schedule and reporting a lasting outage, until it + * succeeds, fails otherwise, or shutdown. * * @template T * @param \Closure(): T $call @@ -1662,14 +1683,19 @@ private function retryUntypedOutage(string $operation, \Closure $call): mixed $attempt = 0; while (true) { try { - return $call(); - } catch (ServerException $exception) { - if (!TransientFailure::is($exception) || $this->shutdownRequested) { + $result = $call(); + $this->clearOutage('cluster_info'); + + return $result; + } catch (ServerException|TransportException $exception) { + if (!TransientFailure::isReadOutage($exception) || $this->shutdownRequested) { throw $exception; } - $delay = TransientFailure::delay(++$attempt, $exception->details['retry_after_seconds'] ?? null); + $details = $exception instanceof ServerException ? $exception->details : $exception->response; + $delay = TransientFailure::delay(++$attempt, $details['retry_after_seconds'] ?? null); $this->diagnostic('worker.retrying', ['worker_id' => $this->workerId, 'operation' => $operation, 'attempt' => $attempt, 'delay_seconds' => $delay, 'exception' => $exception], 'warning'); + $this->observeOutage($operation, $exception, 'cluster_info'); $this->waitForTransientRetry($delay); if ($this->shutdownRequested) { throw $exception; @@ -3666,6 +3692,12 @@ function () use ($taskId, $attemptId, $leaseOwner, $result, $carried, &$continua return; } + if ($resultComputed && $exception instanceof ServerException && ActivityResultTooLarge::matches($exception)) { + // The Server cannot accept this result even under the payload + // policy read again after its refusal. That is this attempt's + // failure, not the Worker's: report it and keep running. + $exception = new ActivityResultTooLarge($exception); + } if ($this->activityLeaseProcess !== null && !$exception instanceof WorkflowClaimRevoked) { try { $this->activityLeaseProcess->check(); @@ -3706,10 +3738,16 @@ function (Throwable $failure) use ($taskId, $attemptId, $leaseOwner): void { $attemptId, $leaseOwner, $failure->getMessage(), - $failure instanceof ActivityExecutionFailure ? $failure->originalType : $failure::class, - $failure instanceof ActivityCancelled || ($failure instanceof ActivityExecutionFailure && $failure->cancelled), + match (true) { + $failure instanceof ActivityExecutionFailure => $failure->originalType, + $failure instanceof ActivityResultTooLarge => ActivityResultTooLarge::FAILURE_TYPE, + default => $failure::class, + }, + $failure instanceof ActivityCancelled || $failure instanceof ActivityResultTooLarge + || ($failure instanceof ActivityExecutionFailure && $failure->cancelled), stackTrace: $failure instanceof ActivityExecutionFailure ? $failure->originalStackTrace : $failure->getTraceAsString(), diagnostics: $failure instanceof ActivityExecutionFailure ? $failure->originalDiagnostics : null, + failureClass: $failure instanceof ActivityResultTooLarge ? ActivityResultTooLarge::FAILURE_TYPE : null, ); }, ); diff --git a/src/Worker/TransientFailure.php b/src/Worker/TransientFailure.php index 4e91cd1b..5bfb92e4 100644 --- a/src/Worker/TransientFailure.php +++ b/src/Worker/TransientFailure.php @@ -38,6 +38,10 @@ final class TransientFailure public static function is(Throwable $error, bool $acknowledgement = false): bool { + if ($error instanceof ServerException && $error->isPayloadDiscoveryUnavailable()) { + // Discovery failed before the request was sent: repeating it is safe. + return true; + } if ($error instanceof ServerException) { if ($error->isTransientConnectionFailure() || $error->isTransientUpstreamFailure()) { return true; @@ -57,6 +61,34 @@ public static function is(Throwable $error, bool $acknowledgement = false): bool return false; } + /** + * An outage of a read-only request (cluster info), which is always safe to + * repeat: a connection that could not be made, was reset or timed out, or + * any 5xx or 429 answer unless it says `retryable: false`. Authentication, + * authorization and other 4xx refusals are not outages. + */ + public static function isReadOutage(Throwable $error): bool + { + if ($error instanceof ServerException) { + if ($error->isTransientConnectionFailure() || $error->isPayloadDiscoveryUnavailable()) { + return true; + } + $status = $error->status; + $details = $error->details; + } elseif ($error instanceof TransportException) { + if ($error->transientConnectionFailure) { + return true; + } + $status = $error->status; + $details = $error->response; + } else { + return false; + } + + return $status !== null && ($status === 429 || ($status >= 500 && $status <= 599)) + && (!is_array($details) || ($details['retryable'] ?? null) !== false); + } + /** Full-jitter delay for the given 1-based attempt, never shorter than a bounded retry_after. */ public static function delay(int $attempt, mixed $retryAfterSeconds = null): float { diff --git a/tests/RuntimePayloadUploadTest.php b/tests/RuntimePayloadUploadTest.php index f1e5881c..83d8ad4d 100644 --- a/tests/RuntimePayloadUploadTest.php +++ b/tests/RuntimePayloadUploadTest.php @@ -57,13 +57,58 @@ public function uploadPayload(string $uri, array $headers, string $blob, int $ti ->completeActivityTask('task', 'attempt', 'worker', 'small'); self::fail("{$name}: expected discovery failure."); } catch (ServerException $exception) { - self::assertSame($status, $exception->status, $name); self::assertFalse($exception->isActivityCompletionPayloadDiscoveryUnavailable('task', 'attempt', 'worker'), $name); self::assertSame(1, $transport->requests, $name); + // A discovery outage (any 5xx) is a generic not-admitted failure the + // Worker may repeat; a refusal (401) is surfaced as itself. + self::assertSame($status >= 500, $exception->isPayloadDiscoveryUnavailable(), $name); + $cause = $exception->getPrevious(); + self::assertSame($status, $cause instanceof TransportException ? $cause->status : $exception->status, $name); } } } + public function testADiscoveryOutageKeepsTheLastPolicyAndTriesDiscoveryAgainLater(): void + { + $transport = new class implements PayloadUploadTransport { + public int $discoveries = 0; + public int $completions = 0; + public bool $down = false; + public function send(string $method, string $uri, array $headers, ?array $body = null): ?array + { + if (str_ends_with($uri, '/api/cluster/info')) { + ++$this->discoveries; + if ($this->down) { + throw TransportException::fromResponse(500, ['message' => 'Server Error'], ''); + } + + return ['limits' => ['max_payload_bytes' => 2097152]]; + } + ++$this->completions; + + return ['recorded' => true, 'outcome' => 'completed']; + } + public function uploadPayload(string $uri, array $headers, string $blob, int $timeoutSeconds): array + { + throw new \LogicException('No upload is needed.'); + } + }; + $client = new Client('https://runtime.test', transport: $transport); + $client->completeActivityTask('task', 'attempt', 'worker', 'small'); + self::assertSame([1, 1], [$transport->discoveries, $transport->completions]); + + // The cached policy expires while discovery is down. + $transport->down = true; + $uploads = (new \ReflectionProperty(Client::class, 'payloadUploads'))->getValue($client); + $policies = new \ReflectionProperty($uploads, 'policies'); + $policies->setValue($uploads, array_map(static fn (array $entry): array => ['expires' => 0] + $entry, $policies->getValue($uploads))); + + $client->completeActivityTask('task', 'attempt', 'worker', 'small'); + $client->completeActivityTask('task', 'attempt', 'worker', 'small'); + self::assertSame(3, $transport->completions, 'Completions proceed with the last policy.'); + self::assertSame(2, $transport->discoveries, 'Discovery is tried again later, not before every request.'); + } + public function testClientUploadsEncodedBytesWithNamespaceAndClientCredential(): void { [$client, $http] = $this->client(); @@ -203,7 +248,8 @@ public function testExplicitCompletionLimitAllowsInlineValuesWithoutConfiguredSt self::assertSame(strlen((string) $http->requests[1]->getBody()), $error->sizeBytes); self::assertGreaterThan($limit, $error->sizeBytes); } - self::assertCount(2, $http->requests, 'Discover once, submit once, with no upload or retry.'); + // A refusal re-reads the policy once; an unchanged body is not sent again. + self::assertCount($overLimit ? 3 : 2, $http->requests, 'Discover once, submit once, with no upload or retry.'); self::assertSame($commands, $this->body($http->requests[1])['commands']); } @@ -254,11 +300,13 @@ public function testContinuationUsesTheAdvertisedAggregateWithoutStorage(string } catch (WorkflowTaskCompletionTooLarge $error) { self::assertTrue($overLimit); self::assertSame($limit, $error->limitBytes); - self::assertCount(2, $http->requests); + self::assertCount(3, $http->requests, 'The policy is read again; the unchanged body is not sent again.'); + self::assertStringEndsWith('/cluster/info', $http->requests[2]->getUri()->getPath()); self::assertSame(strlen((string) $http->requests[1]->getBody()), $error->sizeBytes); self::assertGreaterThan($limit, $error->sizeBytes); } - self::assertCount(2, $http->requests, 'Discover once, submit once; no upload or retry.'); + // A refusal re-reads the policy once; an unchanged body is not sent again. + self::assertCount($overLimit ? 3 : 2, $http->requests, 'Discover once, submit once; no upload or retry.'); $sent = $this->body($http->requests[1]); self::assertSame($continuation, $sent['continuation']); self::assertSame(['codec' => 'avro', 'blob' => $blob], $sent['result']); @@ -335,7 +383,7 @@ public function testExplicitSmallerCompletionCapIsHonoredOnBothRoutes(): void self::assertTrue($continuation); self::assertSame(64, $error->limitBytes); } - self::assertCount($continuation ? 2 : 1, $http->requests); + self::assertCount($continuation ? 3 : 1, $http->requests); } } @@ -423,7 +471,7 @@ public function testImpossibleContinuationRewriteSubmitsOriginalOnceWithoutParti } catch (WorkflowTaskCompletionTooLarge $error) { self::assertSame(500, $error->limitBytes); } - self::assertCount(2, $http->requests); + self::assertCount(3, $http->requests, 'The policy is read again; the unchanged body is not sent again.'); $sent = $this->body($http->requests[1]); self::assertSame($continuation, $sent['continuation']); self::assertSame(['codec' => 'avro', 'blob' => $blob], $sent['result']); diff --git a/tests/WorkerClusterInfoOutageTest.php b/tests/WorkerClusterInfoOutageTest.php new file mode 100644 index 00000000..b16aa85c --- /dev/null +++ b/tests/WorkerClusterInfoOutageTest.php @@ -0,0 +1,178 @@ + */ + public static function outages(): iterable + { + yield 'a 500 with no reason' => ['500']; + yield 'a refused connection' => ['refused']; + yield 'a bodiless 502 from a load balancer' => ['502']; + } + + /** + * A Server whose cluster info fails `$failures` times with `$outage`. + * + * @param array $capabilities + */ + private static function server(string $outage, int $failures, array $capabilities = []): object + { + return new class ($outage, $failures, $capabilities) implements ClientInterface { + /** @var list */ + public array $paths = []; + public int $clusterInfoReads = 0; + public int $workflowPolls = 0; + public bool $completed = false; + public float $now = 0.0; + + /** @param array $capabilities */ + public function __construct(private string $outage, private int $failures, private array $capabilities) + { + } + + public function sendRequest(RequestInterface $request): ResponseInterface + { + $path = $request->getUri()->getPath(); + $this->paths[] = $path; + if ($path === '/api/cluster/info' && ++$this->clusterInfoReads <= $this->failures) { + $this->now += 5.0; + if ($this->outage === 'refused') { + throw new ConnectException('Connection refused', $request); + } + + return $this->outage === '502' ? new Response(502, ['Content-Type' => 'text/html'], 'Bad gateway') + : new Response(500, ['Content-Type' => 'application/json'], '{"message":"Server Error"}'); + } + $body = match (true) { + $path === '/api/cluster/info' => ['version' => '3.17.0', 'limits' => ['max_payload_bytes' => 2097152], + 'worker_protocol' => ['version' => '1.20', 'server_capabilities' => $this->capabilities]], + str_ends_with($path, '/worker/register') => ['registered' => true], + // Until it is completed, the task is offered again (as after a lease expiry). + str_ends_with($path, '/workflow-tasks/poll') && !$this->completed => ['poll_status' => 'leased', + 'protocol_version' => '1.20', 'task' => ['task_id' => 'workflow-task', + 'workflow_task_attempt' => ++$this->workflowPolls, + 'lease_owner' => 'worker-1', 'workflow_id' => 'workflow', 'run_id' => 'run', + 'workflow_type' => 'orders.flow', 'payload_codec' => 'avro', 'history_events' => [], + 'lease_expires_at' => gmdate('Y-m-d\\TH:i:s.000000\\Z', time() + 30)], + 'server_time' => gmdate('Y-m-d\\TH:i:s.000000\\Z')], + str_ends_with($path, '/workflow-tasks/workflow-task/complete') => ['completed' => $this->completed = true], + str_ends_with($path, '/query-tasks/poll') && ($this->completed || count($this->paths) > 600) => ['task' => null, + 'poll_status' => 'stopped', 'reason' => 'worker_stopped'], + default => ['task' => null, 'poll_status' => 'empty'], + }; + + return new Response(200, ['Content-Type' => 'application/json'], json_encode($body, JSON_THROW_ON_ERROR)); + } + + /** A Guzzle handler: a refused connection is reported as cURL reports it (errno 7). */ + public function client(): GuzzleClient + { + return new GuzzleClient(['handler' => function (RequestInterface $request, array $options): PromiseInterface { + try { + $response = $this->sendRequest($request); + } catch (ConnectException $refused) { + ($options['on_stats'] ?? static fn () => null)(new TransferStats($request, null, 0.0, 7)); + + return Create::rejectionFor($refused); + } + ($options['on_stats'] ?? static fn () => null)(new TransferStats($request, $response)); + + return Create::promiseFor($response); + }]); + } + }; + } + + /** @param \ArrayObject $events */ + private static function worker(object $server, \ArrayObject $events, bool $cooperativeCancellation = false): Worker + { + $transport = new Psr18Transport(client: $server->client()); + $worker = new Worker( + new Client('https://runtime.test', transport: $transport, workerToken: 'w', workerProtocolVersion: '1.20'), + 'orders', + workerId: 'worker-1', + enableCooperativeCancellation: $cooperativeCancellation, + clock: static fn (): float => $server->now, + sleeper: static function (int $microseconds) use ($server): void { + $server->now += $microseconds / 1_000_000; + }, + diagnosticListener: static function (string $event, array $context, string $level = 'info') use ($events): void { + $events[] = [$event, $context['operation'] ?? '']; + }, + ); + $worker->registerWorkflow('orders.flow', static fn (WorkflowContext $context): string => 'done'); + + return $worker; + } + + #[DataProvider('outages')] + public function testAWorkflowTaskCompletionWaitsOutAClusterInfoOutageAndIsSentOnce(string $outage): void + { + $server = self::server($outage, 30); + self::worker($server, $log = new \ArrayObject())->run(0); + $events = $log->getArrayCopy(); + + $completions = array_keys($server->paths, '/api/worker/workflow-tasks/workflow-task/complete'); + self::assertCount(1, $completions, 'The completion is sent once, after discovery recovers.'); + self::assertSame(31, $server->clusterInfoReads); + self::assertContains(['worker.retrying', 'cluster_info'], $events); + self::assertContains('worker.outage_persisting', array_column($events, 0), 'Outlasting a minute is an error.'); + self::assertNotContains('worker.fatal_failure', array_column($events, 0)); + self::assertSame([], preg_grep('~/fail$~', $server->paths), 'An outage is never reported as a task failure.'); + } + + #[DataProvider('outages')] + public function testAWorkerStartedDuringAClusterInfoOutageWaitsForIt(string $outage): void + { + $server = self::server($outage, 20, ['cooperative_cancellation' => true]); + $server->completed = true; + self::worker($server, $log = new \ArrayObject(), cooperativeCancellation: true)->run(0); + $events = $log->getArrayCopy(); + + $register = array_search('/api/worker/register', $server->paths, true); + self::assertIsInt($register, 'The Worker registered once discovery recovered.'); + self::assertSame(21, count(array_keys(array_slice($server->paths, 0, $register), '/api/cluster/info'))); + self::assertContains(['worker.outage_persisting', 'cluster_info'], $events); + self::assertNotContains('worker.fatal_failure', array_column($events, 0)); + } + + public function testAReadOnlyDiscoveryOutageIsTransientButARefusalIsNot(): void + { + self::assertTrue(TransientFailure::isReadOutage(new ServerException('x', 500))); + self::assertTrue(TransientFailure::isReadOutage(new TransportException('x', status: 500))); + self::assertTrue(TransientFailure::isReadOutage(new ServerException('x', 429))); + self::assertTrue(TransientFailure::isReadOutage(new TransportException('Connection refused', transientConnectionFailure: true))); + self::assertFalse(TransientFailure::isReadOutage(new ServerException('x', 500, null, ['retryable' => false]))); + self::assertFalse(TransientFailure::isReadOutage(new ServerException('x', 401))); + self::assertFalse(TransientFailure::isReadOutage(new ServerException('x', 403))); + self::assertFalse(TransientFailure::isReadOutage(new ServerException('x', 404))); + } +} diff --git a/tests/WorkerStalePayloadPolicyTest.php b/tests/WorkerStalePayloadPolicyTest.php new file mode 100644 index 00000000..98d8a997 --- /dev/null +++ b/tests/WorkerStalePayloadPolicyTest.php @@ -0,0 +1,214 @@ + */ + public array $paths = []; + public int $limit = WorkerStalePayloadPolicyTest::OLD_LIMIT; + public int $discoveries = 0; + public int $refusals = 0; + public int $uploads = 0; + public ?array $completion = null; + public ?array $failure = null; + public float $now = 0.0; + + public function __construct(private int $discoveryOutages, private bool $redeliver, private bool $storage) + { + } + + /** @param array $options */ + public function __invoke(RequestInterface $request, array $options): PromiseInterface + { + $path = $request->getUri()->getPath(); + $this->paths[] = $path; + if ($path === '/api/cluster/info') { + if (++$this->discoveries > 1 && $this->discoveryOutages-- > 0) { + $this->now += 5.0; + ($options['on_stats'] ?? static fn () => null)(new TransferStats($request, null, 0.0, 7)); + + return Create::rejectionFor(new ConnectException('Connection refused', $request)); + } + $answer = ['limits' => ['max_payload_bytes' => $this->limit], 'namespace' => ['external_payload_storage' => [ + 'status' => 'available', 'threshold_bytes' => $this->limit, + 'transport' => ['schema' => 'durable-workflow.v2.runtime-external-payload-transport.v1', 'version' => 1, + 'reference_schema' => RuntimePayloads::SCHEMA, 'mode' => 'authenticated_namespace_runtime', + 'upload' => ['method' => 'POST', 'path' => '/api/external-payloads/v1'], + 'fetch' => ['method' => 'GET', 'path_template' => '/api/external-payloads/v1/{referenceId}'], + 'limits' => ['max_payload_bytes' => 16_777_216, 'request_timeout_seconds' => 45]]]]]; + if (!$this->storage && $this->limit === WorkerStalePayloadPolicyTest::NEW_LIMIT) { + unset($answer['namespace']); + } + // The operator lowers the inline limit right after this Worker read the old one. + $this->limit = WorkerStalePayloadPolicyTest::NEW_LIMIT; + + return self::json(200, $answer); + } + if ($path === '/api/external-payloads/v1') { + ++$this->uploads; + $blob = (string) $request->getBody(); + + return self::json(200, ['schema' => 'durable-workflow.v2.runtime-external-payload-upload.v1', 'transport_version' => 1, + 'reference' => ['schema' => RuntimePayloads::SCHEMA, 'reference_id' => 'ep_01ARZ3NDEKTSV4RRFFQ69G5FAV', + 'codec' => 'avro', 'size_bytes' => strlen($blob), 'sha256' => hash('sha256', $blob)]]); + } + if ($path === '/api/worker/activity-tasks/activity-1/fail') { + $this->failure = json_decode((string) $request->getBody(), true, flags: JSON_THROW_ON_ERROR); + + return self::json(200, ['recorded' => true, 'outcome' => 'failed']); + } + if ($path === '/api/worker/activity-tasks/activity-1/complete') { + if ($request->getBody()->getSize() > $this->limit) { + // The Server's admission check refuses before reading the body. + ++$this->refusals; + + return self::json(413, ['reason' => 'payload_too_large', 'limit' => $this->limit, + 'message' => "Request payload exceeds the maximum allowed size of {$this->limit} bytes."]); + } + $this->completion = json_decode((string) $request->getBody(), true, flags: JSON_THROW_ON_ERROR); + + return self::json(200, ['recorded' => true, 'outcome' => 'completed']); + } + + return self::json(200, match (true) { + str_ends_with($path, '/worker/register') => ['registered' => true], + // Offered until it is completed (as after a lease expiry). + str_ends_with($path, '/activity-tasks/poll') && $this->completion === null && $this->failure === null + && ($this->redeliver || !in_array('/api/worker/activity-tasks/activity-1/complete', $this->paths, true)) => [ + 'poll_status' => 'leased', 'task' => ['task_id' => 'activity-1', 'activity_attempt_id' => 'attempt-1', + 'lease_owner' => 'worker-1', 'activity_type' => 'reports.render', 'payload_codec' => 'avro']], + str_ends_with($path, '/query-tasks/poll') && ($this->completion !== null || $this->failure !== null || count($this->paths) > 300) => [ + 'task' => null, 'poll_status' => 'stopped', 'reason' => 'worker_stopped'], + default => ['task' => null, 'poll_status' => 'empty'], + }); + } + + /** @param array $body */ + private static function json(int $status, array $body): PromiseInterface + { + return Create::promiseFor(new Response($status, ['Content-Type' => 'application/json'], json_encode($body, JSON_THROW_ON_ERROR))); + } + }; + } + + /** @return array{0: Worker, 1: \ArrayObject, 2: \ArrayObject} */ + private static function worker(object $server): array + { + $events = new \ArrayObject(); + $runs = new \ArrayObject(); + $worker = new Worker( + new Client('https://runtime.test', transport: new Psr18Transport(new GuzzleClient(['handler' => $server])), workerToken: 'w'), + 'reports', + workerId: 'worker-1', + clock: static fn (): float => $server->now, + sleeper: static function (int $microseconds) use ($server): void { + $server->now += $microseconds / 1_000_000; + }, + diagnosticListener: static function (string $event, array $context) use ($events): void { + $events[] = $event; + }, + ); + $worker->registerActivity('reports.render', static function (ActivityContext $context) use ($runs): string { + $runs[] = true; + + return str_repeat('r', 500_000); + }); + + return [$worker, $events, $runs]; + } + + public function testALoweredInlineLimitIsRediscoveredAndTheResultExternalisedAndSentOnceMore(): void + { + $server = self::server(); + [$worker, $events, $runs] = self::worker($server); + + $worker->run(0); + + self::assertCount(1, $runs, 'The activity ran once; its result was never recomputed.'); + self::assertSame(1, $server->refusals, 'One inline attempt met the lowered limit.'); + self::assertSame(2, $server->discoveries, 'The stale policy was dropped and read again.'); + self::assertSame(1, $server->uploads, 'The result was externalised under the new limit.'); + self::assertIsArray($server->completion); + self::assertSame('ep_01ARZ3NDEKTSV4RRFFQ69G5FAV', $server->completion['result']['external_payload']['reference_id'] ?? null); + self::assertSame([], preg_grep('~/fail$~', $server->paths), 'Nothing is reported as an activity failure.'); + self::assertNotContains('worker.fatal_failure', $events->getArrayCopy()); + self::assertNotContains('worker.failed', $events->getArrayCopy()); + } + + public function testRediscoveryThatMeetsAnOutageIsRetriedWithoutLosingTheResult(): void + { + $server = self::server(discoveryOutages: 4); + [$worker, $events, $runs] = self::worker($server); + + $worker->run(0); + + self::assertCount(1, $runs, 'The computed result was kept through the outage.'); + self::assertSame(1, $server->uploads); + self::assertSame('ep_01ARZ3NDEKTSV4RRFFQ69G5FAV', $server->completion['result']['external_payload']['reference_id'] ?? null); + self::assertContains('worker.retrying', $events->getArrayCopy()); + self::assertSame([], preg_grep('~/fail$~', $server->paths)); + self::assertNotContains('worker.fatal_failure', $events->getArrayCopy()); + } + + public function testAnOutageThatOutlastsTheClaimLeavesTheAttemptToTheServerAndTheWorkerRuns(): void + { + $server = self::server(discoveryOutages: 30, redeliver: true); + [$worker, $events, $runs] = self::worker($server); + + $worker->run(0); + + self::assertContains('worker.outcome_unpublished', $events->getArrayCopy()); + self::assertContains('worker.outage_persisting', $events->getArrayCopy()); + self::assertSame('ep_01ARZ3NDEKTSV4RRFFQ69G5FAV', $server->completion['result']['external_payload']['reference_id'] ?? null, + 'The re-delivered attempt completes once discovery answers.'); + self::assertSame([], preg_grep('~/fail$~', $server->paths), 'Never reported as an activity failure.'); + self::assertNotContains('worker.fatal_failure', $events->getArrayCopy()); + } + + public function testAResultTooLargeEvenUnderTheNewLimitFailsTheAttemptAndTheWorkerRuns(): void + { + $server = self::server(storage: false); + [$worker, $events, $runs] = self::worker($server); + + $worker->run(0); + + self::assertCount(1, $runs); + self::assertSame(2, $server->discoveries, 'The policy was read again before giving up.'); + self::assertNull($server->completion); + self::assertIsArray($server->failure, 'The attempt is reported as failed.'); + self::assertSame('ActivityResultTooLarge', $server->failure['failure']['type'] ?? null, 'The same in every SDK.'); + self::assertSame('ActivityResultTooLarge', $server->failure['failure']['class'] ?? null); + self::assertTrue($server->failure['failure']['non_retryable'] ?? null, 'The same result would be refused again.'); + self::assertNotContains('worker.fatal_failure', $events->getArrayCopy()); + self::assertNotContains('worker.failed', $events->getArrayCopy()); + } +}