diff --git a/CHANGELOG.md b/CHANGELOG.md index f89d35d0..90483e89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,6 +63,41 @@ project follows [Semantic Versioning](https://semver.org/). activity retry policy is unchanged, because retrying a non-idempotent activity by default could repeat its side effects. +### Changed + +- The default transport asks for gzip on every request when it owns a Guzzle + client with cURL and zlib, and decodes a gzip body a custom client left + encoded. Servers that compress (PHP Server, and native Server with its + response compression) then send several times fewer bytes. +- Worker requests send `X-Durable-Workflow-Server-Capabilities-Digest`. A + Server that supports it replaces repeated `server_capabilities` objects + (about 17 KB on native, sent on every worker answer and again inside + carried polls) with their digest, and the transport restores the held object + before any caller sees the answer. Older Servers ignore the header. An + answer naming a digest the transport does not hold is recovered without + repeating any request that may have changed state: that answer is kept as + received (a completion's answer can carry the next leased task), the + capabilities are read from `GET /api/cluster/info`, and the held pair is + forgotten. Only a task poll is sent again without the header. A digest in + an error body never replaces the HTTP error. +- A gzip response is inflated by the transport within a decoded-size limit + (`Psr18Transport` `maxDecodedResponseBytes`; by default 32 MiB, or a quarter + of `memory_limit` when that is lower). A larger body fails with + `DurableWorkflow\Exception\ResponseTooLarge`, which reaches callers as + itself. +- A `409 stale_attempt` that proves this Worker's own activity completion was + already committed (`recorded: false`, its owner and attempt, every status + `completed`) is accepted as committed on any answer: the first, after a + transient failure, or a resend by the HTTP client. +- `result()` of a completed run reads the output from the terminal status + read instead of downloading the run's whole history again. Failed, + cancelled, terminated and timed-out runs, and Servers that omit + `output_envelope`, still read the history. + +In a lab DW Standard Workflow v1 run (native Server with compression and the +digest, 1 workflow/second), response bytes per workflow went from about +385 KB to about 18 KB. + ## [3.22.0] - 2026-11-19 Signal-with-start and durable signal-external need native Server 3.15.0 or diff --git a/src/Client.php b/src/Client.php index a6f63fc8..fc49804e 100644 --- a/src/Client.php +++ b/src/Client.php @@ -17,6 +17,7 @@ use DurableWorkflow\Exception\QueryFailed; use DurableWorkflow\Exception\ScheduleAlreadyExists; use DurableWorkflow\Exception\ServerCapabilityUnavailable; +use DurableWorkflow\Exception\ResponseTooLarge; use DurableWorkflow\Exception\ServerException; use DurableWorkflow\Exception\SignalFailed; use DurableWorkflow\Exception\SignalWithStartFailed; @@ -3495,7 +3496,8 @@ private function performRequest( return $this->resolveResponsePayloads($response, $path, $worker, $body, $headers, $budget, $callerBudget, $poll, $payloadPaths); } catch (TransportException $exception) { - if ($exception instanceof FailureReportRefused) { + if ($exception instanceof FailureReportRefused || $exception instanceof ResponseTooLarge) { + // Typed transport outcomes reach the caller as themselves. throw $exception; } $details = $exception->response; @@ -3573,7 +3575,8 @@ private function carriedPollResponse(array $answer, array $request, string $path $this->requestHeaders(true), $this->boundedWorkerRequests ? new RequestBudget(5) : null, null, true, null); } catch (TransportException $exception) { - if ($exception instanceof FailureReportRefused) { + if ($exception instanceof FailureReportRefused || $exception instanceof ResponseTooLarge) { + // Typed transport outcomes reach the caller as themselves. throw $exception; } $details = $exception->response; @@ -3634,6 +3637,12 @@ private function terminalResult( string $status, WorkflowExecution $execution, ): mixed { + if ($status === 'completed' && is_array($execution->raw['output_envelope'] ?? null) + && ($execution->raw['run_id'] ?? null) === $runId) { + // The terminal read already names the run's output; reading the + // whole history again only repeats it. + return $this->completedRunOutput($workflowId, $runId, $execution); + } $history = $this->request( 'GET', '/workflows/'.$this->segment($workflowId).'/runs/'.$this->segment($runId).'/history', false, payloadPaths: self::terminalHistoryPayloadPaths(...), @@ -3673,11 +3682,7 @@ private function terminalResult( } return match ($status) { - 'completed' => $this->decodedResponsePayload( - (new RuntimePayloads($this->transport, $this->baseUri, $this->requestHeaders(false), $this->maxExternalPayloadBytes)) - ->response($execution->raw, '/workflows/'.$this->segment($workflowId).'/runs/'.$this->segment($runId), false, [['output_envelope']]), - 'output', - ), + 'completed' => $this->completedRunOutput($workflowId, $runId, $execution), 'cancelled', 'canceled' => throw new WorkflowCancelled('Workflow was cancelled.'), 'terminated' => throw new WorkflowTerminated('Workflow was terminated.'), 'timed_out', 'timeout' => throw new WorkflowTimedOut('Workflow execution timed out.'), @@ -3685,6 +3690,15 @@ private function terminalResult( }; } + private function completedRunOutput(string $workflowId, string $runId, WorkflowExecution $execution): mixed + { + return $this->decodedResponsePayload( + (new RuntimePayloads($this->transport, $this->baseUri, $this->requestHeaders(false), $this->maxExternalPayloadBytes)) + ->response($execution->raw, '/workflows/'.$this->segment($workflowId).'/runs/'.$this->segment($runId), false, [['output_envelope']]), + 'output', + ); + } + /** @param array $response * @return list> */ diff --git a/src/Exception/ResponseTooLarge.php b/src/Exception/ResponseTooLarge.php new file mode 100644 index 00000000..e4130292 --- /dev/null +++ b/src/Exception/ResponseTooLarge.php @@ -0,0 +1,15 @@ + 'response_too_large', 'limit_bytes' => $limitBytes]); + } +} diff --git a/src/Transport/Psr18Transport.php b/src/Transport/Psr18Transport.php index a83c1316..d6bd1d2a 100644 --- a/src/Transport/Psr18Transport.php +++ b/src/Transport/Psr18Transport.php @@ -5,6 +5,7 @@ namespace DurableWorkflow\Transport; use DurableWorkflow\Exception\ExternalPayloadException; +use DurableWorkflow\Exception\ResponseTooLarge; use DurableWorkflow\Exception\TransportException; use GuzzleHttp\Client as GuzzleClient; use GuzzleHttp\Psr7\FnStream; @@ -28,6 +29,13 @@ final class Psr18Transport implements BoundedPayloadTransport, BoundedPayloadUpl public const DEFAULT_REQUEST_TIMEOUT_SECONDS = 60.0; /** Worker lifecycle requests (register, heartbeat, complete, fail, deregister); long polls keep their own budget. */ public const DEFAULT_WORKER_REQUEST_TIMEOUT_SECONDS = 60.0; + /** + * The most bytes a compressed JSON response may decode to by default: + * 32 MiB, or a quarter of PHP's memory_limit when that is lower, so the + * decoded body and its JSON decoding fit under the default 128 MB limit. + */ + public const DEFAULT_MAX_DECODED_RESPONSE_BYTES = 32 * 1024 * 1024; + private readonly int $maxDecodedResponseBytes; private readonly ClientInterface $client; private readonly RequestFactoryInterface $requestFactory; @@ -42,6 +50,7 @@ final class Psr18Transport implements BoundedPayloadTransport, BoundedPayloadUpl private ?string $completionUri = null; private bool $completionArmed = false; private ?CompletionRequestHandler $completionHandler = null; + private readonly ServerCapabilitiesDigest $capabilitiesDigest; public function lastResponseStatus(): ?int { @@ -55,7 +64,12 @@ public function __construct( private readonly int $payloadTransferTimeoutSeconds = 30, ?float $requestTimeoutSeconds = null, ?float $workerRequestTimeoutSeconds = null, + ?int $maxDecodedResponseBytes = null, ) { + if ($maxDecodedResponseBytes !== null && $maxDecodedResponseBytes < 1) { + throw new \InvalidArgumentException('maxDecodedResponseBytes must be positive.'); + } + $this->maxDecodedResponseBytes = $maxDecodedResponseBytes ?? self::defaultMaxDecodedResponseBytes(); if ($payloadTransferTimeoutSeconds < 1 || $payloadTransferTimeoutSeconds > 900) { throw new \InvalidArgumentException('Supervised payload transfers require a timeout from 1 through 900 seconds.'); } @@ -71,6 +85,7 @@ public function __construct( $this->workerRequestTimeoutSeconds = $client === null ? ($workerRequestTimeoutSeconds ?? self::DEFAULT_WORKER_REQUEST_TIMEOUT_SECONDS) : null; $factory = new HttpFactory(); + $this->capabilitiesDigest = new ServerCapabilitiesDigest(); $this->defaultProcessConfiguration = $client === null && $requestFactory === null && $streamFactory === null; $this->client = $client ?? new GuzzleClient(['http_errors' => false]); $this->requestFactory = $requestFactory ?? $factory; @@ -87,6 +102,81 @@ public function send(string $method, string $uri, array $headers, ?array $body = return $this->sendJson($method, $uri, $headers, $body); } + /** The owned Guzzle client over cURL can ask for gzip; this transport inflates it. */ + private function decodesCompressedResponses(): bool + { + return $this->client instanceof GuzzleClient && extension_loaded('curl') && extension_loaded('zlib'); + } + + /** min(32 MiB, memory_limit / 4); an unlimited memory_limit keeps 32 MiB. */ + public static function defaultMaxDecodedResponseBytes(?string $memoryLimit = null): int + { + $limit = trim($memoryLimit ?? (string) ini_get('memory_limit')); + if ($limit === '' || $limit === '-1' || preg_match('/\A(\d+)\s*([kmg]?)\z/i', $limit, $match) !== 1) { + return self::DEFAULT_MAX_DECODED_RESPONSE_BYTES; + } + $bytes = (int) $match[1] * match (strtolower($match[2])) { + 'g' => 1024 ** 3, 'm' => 1024 ** 2, 'k' => 1024, default => 1, + }; + + return max(1024 * 1024, min(self::DEFAULT_MAX_DECODED_RESPONSE_BYTES, intdiv($bytes, 4))); + } + + /** + * The Server's current worker capabilities from cluster info, a read-only + * call; null when it cannot be read. + * + * @param array $headers + * @return array|null + */ + private function fetchCapabilities(string $uri, array $headers): ?array + { + $position = strpos($uri, '/api/'); + if ($position === false) { + return null; + } + try { + $info = $this->sendJson('GET', substr($uri, 0, $position).'/api/cluster/info', + array_diff_key($headers, ['Content-Type' => true]), null); + } catch (Throwable) { + return null; + } + $capabilities = is_array($info['worker_protocol'] ?? null) ? ($info['worker_protocol']['server_capabilities'] ?? null) : null; + + return is_array($capabilities) && !array_is_list($capabilities) ? $capabilities : null; + } + + /** Inflate a gzip body, refusing to produce more than the decoded-size limit. */ + private function inflate(string $body): string + { + $context = @inflate_init(ZLIB_ENCODING_GZIP); + if ($context === false) { + throw new TransportException('The server returned an invalid gzip response body.'); + } + $decoded = ''; + // Small input chunks bound how far one step can overshoot the limit. + foreach (str_split($body, 8192) as $chunk) { + $part = @inflate_add($context, $chunk, ZLIB_SYNC_FLUSH); + if ($part === false) { + throw new TransportException('The server returned an invalid gzip response body.'); + } + $decoded .= $part; + if (strlen($decoded) > $this->maxDecodedResponseBytes) { + throw new ResponseTooLarge($this->maxDecodedResponseBytes); + } + } + $tail = @inflate_add($context, '', ZLIB_FINISH); + if ($tail === false) { + throw new TransportException('The server returned an invalid gzip response body.'); + } + $decoded .= $tail; + if (strlen($decoded) > $this->maxDecodedResponseBytes) { + throw new ResponseTooLarge($this->maxDecodedResponseBytes); + } + + return $decoded; + } + public function supportsBoundedRequests(): bool { return $this->client instanceof GuzzleClient && extension_loaded('curl'); @@ -182,11 +272,45 @@ private function sendJson(string $method, string $uri, array $headers, ?array $b if ($poll) { $request = $request->withHeader('Accept', PollTransport::MEDIA_TYPE.', application/json;q=0.9'); } - $response = $this->sendRequest($request, timeoutSeconds: $timeoutSeconds ?? $controlTimeout ?? 30, - bounded: $timeoutSeconds !== null || $controlTimeout !== null, controlTimeout: $controlTimeout, control: $control); - $this->responseStatus = $response->getStatusCode(); - - return $this->decodeJsonResponse($response, $poll); + if ($this->decodesCompressedResponses() && !$request->hasHeader('Accept-Encoding')) { + // The transport inflates the body itself, within its size limit. + $request = $request->withHeader('Accept-Encoding', 'gzip'); + } + $send = function (RequestInterface $request, ?array $captured) use ($timeoutSeconds, $controlTimeout, $control, $poll): ?array { + $response = $this->sendRequest($request, timeoutSeconds: $timeoutSeconds ?? $controlTimeout ?? 30, + bounded: $timeoutSeconds !== null || $controlTimeout !== null, controlTimeout: $controlTimeout, control: $control); + $this->responseStatus = $response->getStatusCode(); + + return $this->decodeJsonResponse($response, $poll, $captured); + }; + if ($control) { + return $send($request, null); + } + // The answer is resolved against the pair this request was sent with. + $captured = $this->capabilitiesDigest->capture(); + try { + return $send($request->withHeader(ServerCapabilitiesDigest::HEADER, $this->capabilitiesDigest->header($captured)), $captured); + } catch (UnknownServerCapabilitiesDigest $unknown) { + // The answer named capabilities this worker does not hold. + // Forget the held pair; the next request asks for a tagged copy. + $this->capabilitiesDigest->forget(); + if (PollTransport::taskKind($method, $uri) !== null) { + // A task poll is safe to repeat: ask again without the header. + return $send($request->withoutHeader(ServerCapabilitiesDigest::HEADER), [null, null]); + } + if ($unknown->answer === null) { + throw $unknown; + } + // Anything else may have changed state (a completion can carry + // the next leased task), so it is never repeated. Its answer is + // kept as received; only the capabilities are fetched again, + // with a read-only cluster-info call. + $answer = $unknown->answer; + $this->capabilitiesDigest->restore($answer, + [$unknown->digest, $this->fetchCapabilities($uri, $headers) ?? $captured[1] ?? []]); + + return $answer; + } } catch (TransportException $exception) { throw $exception; } catch (JsonException $exception) { @@ -209,14 +333,26 @@ private function pollTimeoutSeconds(?array $body): int|float ? min($timeout, (float) $configured) : $timeout; } - /** @return array|list|null */ - private function decodeJsonResponse(ResponseInterface $response, bool $poll = false): ?array + /** + * @param array{0: ?string, 1: array|null}|null $captured The digest pair a worker request was sent with. + * @return array|list|null + */ + private function decodeJsonResponse(ResponseInterface $response, bool $poll = false, ?array $captured = null): ?array { $rawBody = (string) $response->getBody(); + if (str_starts_with($rawBody, "\x1f\x8b") && function_exists('inflate_init') + && strtolower(trim($response->getHeaderLine('Content-Encoding'))) === 'gzip') { + // A body left gzip-encoded: the owned client asks for gzip without + // letting cURL inflate it, so the size limit always applies. + $rawBody = $this->inflate($rawBody); + } $status = $response->getStatusCode(); $mediaType = strtolower(trim(explode(';', $response->getHeaderLine('Content-Type'), 2)[0])); if ($poll && $mediaType === PollTransport::MEDIA_TYPE) { $frame = PollTransport::decode($rawBody, $status); + if ($captured !== null) { + $this->capabilitiesDigest->restore($frame['body'], $captured); + } if ($frame['status'] >= 300) { throw TransportException::fromResponse($frame['status'], $frame['body'], $rawBody, $frame['headers']); } @@ -227,6 +363,16 @@ private function decodeJsonResponse(ResponseInterface $response, bool $poll = fa // An upstream proxy can return HTML or an empty body. Preserve // its status without interpreting it as a successful protocol reply. $decoded = json_decode($rawBody, true); + if ($captured !== null && is_array($decoded)) { + try { + $this->capabilitiesDigest->restore($decoded, $captured); + } catch (UnknownServerCapabilitiesDigest) { + // The HTTP error is the answer; an unusable digest in it + // never replaces it. Ask for full capabilities next time. + $this->capabilitiesDigest->forget(); + unset($decoded['server_capabilities']); + } + } throw TransportException::fromResponse($status, is_array($decoded) ? $decoded : null, $rawBody); } @@ -234,6 +380,14 @@ private function decodeJsonResponse(ResponseInterface $response, bool $poll = fa if ($decoded !== null && !is_array($decoded)) { throw new TransportException('The server returned a JSON value instead of an object or array.'); } + if ($captured !== null && $decoded !== null) { + $received = $decoded; + try { + $this->capabilitiesDigest->restore($decoded, $captured); + } catch (UnknownServerCapabilitiesDigest $unknown) { + throw $unknown->withAnswer($received); + } + } return $decoded; } @@ -515,6 +669,10 @@ private function sendRequest(RequestInterface $request, bool $stream = false, in $onStats = $http->getConfig('on_stats'); $options = $this->guzzleRequestOptions($stream, $timeoutSeconds, $bounded, $sink, $progress); + if (!$stream && $sink === null && $request->getHeaderLine('Accept-Encoding') === 'gzip') { + // Keep the body encoded: decodeJsonResponse() inflates it within its size limit. + $options['decode_content'] = false; + } $options['on_stats'] = static function (TransferStats $stats) use (&$connectionFailure, &$timedOut, $onStats): void { // Guzzle 8 no longer attaches cURL errno to exceptions. // Accept only DNS/connect/timeout/closed-connection errors, diff --git a/src/Transport/ServerCapabilitiesDigest.php b/src/Transport/ServerCapabilitiesDigest.php new file mode 100644 index 00000000..bb392781 --- /dev/null +++ b/src/Transport/ServerCapabilitiesDigest.php @@ -0,0 +1,103 @@ +|null */ + private ?array $capabilities = null; + + /** + * The pair a request is sent with; pass it back to restore() for its answer. + * + * @return array{0: ?string, 1: array|null} + */ + public function capture(): array + { + return [$this->digest, $this->capabilities]; + } + + /** Forget the held pair; the next request asks for full capabilities again. */ + public function forget(): void + { + $this->digest = null; + $this->capabilities = null; + } + + /** @param array{0: ?string, 1: array|null}|null $captured */ + public function header(?array $captured = null): string + { + return ($captured ?? $this->capture())[0] ?? 'none'; + } + + /** + * Restore elided copies and keep a tagged one, in place. + * + * @param array $answer + * @param array{0: ?string, 1: array|null}|null $captured The pair its request was sent with. + */ + public function restore(array &$answer, ?array $captured = null): void + { + $learned = [null, null]; + $this->restoreLevel($answer, $captured ?? $this->capture(), $learned, 1); + } + + /** + * @param array $answer + * @param array{0: ?string, 1: array|null} $captured + * @param array{0: ?string, 1: array|null} $learned A tagged copy seen in this answer. + * @param-out array{0: ?string, 1: array|null} $learned + */ + private function restoreLevel(array &$answer, array $captured, array &$learned, int $depth): void + { + if (array_is_list($answer)) { + return; + } + foreach ($answer as $key => &$value) { + if ($key === 'server_capabilities') { + if (is_string($value) && str_starts_with($value, 'sha256:')) { + $value = match (true) { + $value === $captured[0] && $captured[1] !== null => $captured[1], + $value === $learned[0] && $learned[1] !== null => $learned[1], + default => throw new UnknownServerCapabilitiesDigest($value), + }; + } elseif (is_array($value) && !array_is_list($value) && is_string($value['digest'] ?? null) + && str_starts_with($value['digest'], 'sha256:')) { + $digest = $value['digest']; + unset($value['digest']); + /** @var array $value */ + $learned = [$digest, $value]; + // Later requests send the newest digest this transport has seen. + $this->digest = $digest; + $this->capabilities = $value; + } + } elseif (is_array($value) && $depth < self::MAX_DEPTH) { + $this->restoreLevel($value, $captured, $learned, $depth + 1); + } + } + unset($value); + } +} diff --git a/src/Transport/UnknownServerCapabilitiesDigest.php b/src/Transport/UnknownServerCapabilitiesDigest.php new file mode 100644 index 00000000..3b2c08bc --- /dev/null +++ b/src/Transport/UnknownServerCapabilitiesDigest.php @@ -0,0 +1,23 @@ +|null $answer The decoded answer as received, when it was a success. */ + public function __construct(public readonly string $digest, public readonly ?array $answer = null) + { + parent::__construct('The server omitted capabilities this worker does not hold.'); + } + + /** @param array $answer */ + public function withAnswer(array $answer): self + { + return new self($this->digest, $answer); + } +} diff --git a/tests/ClientWorkflowResultTest.php b/tests/ClientWorkflowResultTest.php index 04510094..674add1e 100644 --- a/tests/ClientWorkflowResultTest.php +++ b/tests/ClientWorkflowResultTest.php @@ -35,7 +35,8 @@ public function testWaitingForStatusDoesNotDownloadUnusedInputOrOutput(): void #[DataProvider('historyFields')] public function testOnlyTheTerminalHistoryPayloadIsDownloaded(string $historyField): void { - [$client, $requests] = $this->payloadClient('completed', $historyField); + // A terminal read without the run output falls back to the history. + [$client, $requests] = $this->payloadClient('completed', $historyField, describeOutput: false); self::assertSame(['finished' => true], $client->workflowHandle('order', 'run')->resultOfSelectedRun(timeoutSeconds: 0)); self::assertSame(['/api/workflows/order/runs/run', '/api/workflows/order/runs/run/history', '/api/external-payloads/v1/ep_01ARZ3NDEKTSV4RRFFQ69G5FAV'], array_map(static fn (RequestInterface $request): string => $request->getUri()->getPath(), $requests->value)); @@ -48,12 +49,16 @@ public static function historyFields(): iterable yield 'history_events' => ['history_events']; } - public function testMissingTerminalHistoryStillUsesTheRunOutput(): void + public function testACompletedRunOutputNeedsNoHistoryRead(): void { [$client, $requests] = $this->payloadClient('completed', emptyHistory: true); self::assertSame(['finished' => true], $client->workflowHandle('order', 'run')->resultOfSelectedRun(timeoutSeconds: 0)); - self::assertCount(3, $requests->value); - self::assertStringEndsWith('/ep_01ARZ3NDEKTSV4RRFFQ69G5FAV', (string) $requests->value[2]->getUri()); + self::assertSame(['/api/workflows/order/runs/run', '/api/external-payloads/v1/ep_01ARZ3NDEKTSV4RRFFQ69G5FAV'], + array_map(static fn (RequestInterface $request): string => $request->getUri()->getPath(), $requests->value)); + [$client, $requests] = $this->payloadClient('completed', emptyHistory: true); + self::assertSame(['finished' => true], $client->workflowHandle('order', 'run')->result(timeoutSeconds: 0)); + self::assertSame(['/api/workflows/order', '/api/external-payloads/v1/ep_01ARZ3NDEKTSV4RRFFQ69G5FAV'], + array_map(static fn (RequestInterface $request): string => $request->getUri()->getPath(), $requests->value)); } public function testAnUnreadableInputDoesNotHideTheTerminalFailure(): void @@ -81,7 +86,7 @@ public function testTheSelectedTerminalOutputStillRequiresValidIntegrityMetadata } /** @return array{Client, object} */ - private function payloadClient(string $status, string $historyField = 'events', bool $emptyHistory = false, bool $corruptOutput = false): array + private function payloadClient(string $status, string $historyField = 'events', bool $emptyHistory = false, bool $corruptOutput = false, bool $describeOutput = true): array { $codec = new AvroPayloadCodec(); $blob = $codec->encode(['finished' => true]); @@ -89,7 +94,7 @@ private function payloadClient(string $status, string $historyField = 'events', $output = ['codec' => 'avro', 'external_payload' => $reference]; $unused = ['codec' => 'avro', 'external_payload' => array_replace($reference, ['reference_id' => 'ep_01ARZ3NDEKTSV4RRFFQ69G5FAW'])]; $requests = new class { public array $value = []; }; - $http = new GuzzleClient(['handler' => static function (RequestInterface $request) use ($requests, $status, $historyField, $emptyHistory, $blob, $output, $unused, $corruptOutput) { + $http = new GuzzleClient(['handler' => static function (RequestInterface $request) use ($requests, $status, $historyField, $emptyHistory, $blob, $output, $unused, $corruptOutput, $describeOutput) { $requests->value[] = $request; $path = $request->getUri()->getPath(); if (str_ends_with($path, '/ep_01ARZ3NDEKTSV4RRFFQ69G5FAW')) { @@ -109,8 +114,8 @@ private function payloadClient(string $status, string $historyField = 'events', : ['event_type' => 'WorkflowCompleted', 'payload' => ['output' => $output, 'result' => $unused]], ]]; } else { - $body = ['workflow_id' => 'order', 'run_id' => 'run', 'status' => $status, - 'input_envelope' => $unused, 'output_envelope' => $output]; + $body = ['workflow_id' => 'order', 'run_id' => 'run', 'status' => $status, 'input_envelope' => $unused] + + ($describeOutput ? ['output_envelope' => $output] : []); } return Create::promiseFor(new Response(200, [], json_encode($body, JSON_THROW_ON_ERROR))); diff --git a/tests/ServerCapabilitiesDigestTest.php b/tests/ServerCapabilitiesDigestTest.php new file mode 100644 index 00000000..60e343aa --- /dev/null +++ b/tests/ServerCapabilitiesDigestTest.php @@ -0,0 +1,399 @@ + true, 'activity_lease_renewal' => true, + 'supported_workflow_task_commands' => ['schedule_activity', 'complete_workflow']]; + + /** @param list, 2: string}> $responses */ + private function http(array $responses): ClientInterface + { + return new class ($responses) implements ClientInterface { + /** @var list */ + public array $requests = []; + + /** @param list, 2: string}> $responses */ + public function __construct(private array $responses) + { + } + + public function sendRequest(RequestInterface $request): ResponseInterface + { + $this->requests[] = $request; + [$status, $headers, $body] = array_shift($this->responses); + + return new Response($status, $headers + ['Content-Type' => 'application/json'], $body); + } + }; + } + + private static function tagged(): array + { + return self::CAPABILITIES + ['digest' => self::DIGEST]; + } + + public function testAWorkerKeepsTaggedCapabilitiesAndRestoresElidedCopies(): void + { + $http = $this->http([ + [200, [], json_encode(['task' => null, 'server_capabilities' => self::tagged()], JSON_THROW_ON_ERROR)], + [200, [], json_encode(['outcome' => 'completed', 'activity_poll' => ['leased' => true, + 'answer' => ['task' => ['id' => 'a'], 'server_capabilities' => self::DIGEST]], + 'server_capabilities' => self::DIGEST], JSON_THROW_ON_ERROR)], + [409, [], json_encode(['reason' => 'lease_expired', 'server_capabilities' => self::DIGEST], JSON_THROW_ON_ERROR)], + ]); + $transport = new Psr18Transport(client: $http); + $worker = ['X-Durable-Workflow-Protocol-Version' => '1.20']; + + $first = $transport->send('POST', 'https://runtime.test/api/worker/heartbeat', $worker, []); + self::assertSame(self::CAPABILITIES, $first['server_capabilities']); + self::assertSame('none', $http->requests[0]->getHeaderLine(ServerCapabilitiesDigest::HEADER)); + + $second = $transport->send('POST', 'https://runtime.test/api/worker/workflow-tasks/t/complete', $worker, []); + self::assertSame(self::DIGEST, $http->requests[1]->getHeaderLine(ServerCapabilitiesDigest::HEADER)); + self::assertSame(self::CAPABILITIES, $second['server_capabilities']); + self::assertSame(self::CAPABILITIES, $second['activity_poll']['answer']['server_capabilities']); + self::assertSame(['id' => 'a'], $second['activity_poll']['answer']['task']); + + try { + $transport->send('POST', 'https://runtime.test/api/worker/activity-tasks/a/complete', $worker, []); + self::fail('A 409 remains a refusal.'); + } catch (TransportException $exception) { + self::assertSame(409, $exception->status); + self::assertSame(self::CAPABILITIES, $exception->response['server_capabilities']); + } + } + + /** + * Rolling deploy: a completion is in flight under D1 while another worker + * request (as completion pipelining overlaps them) learns D2. The server + * answers the completion with the digest it was sent, D1, after committing + * it; the transport must restore the D1 object, not refuse the answer. + */ + public function testOverlappingRequestsAcrossTwoDigestsEachRestoreTheirOwnCapabilities(): void + { + $d2 = 'sha256:'.str_repeat('2', 64); + $capabilities2 = ['query_tasks' => true, 'activity_lease_renewal' => true, 'new_after_deploy' => true]; + $http = new class (self::DIGEST, self::CAPABILITIES, $d2, $capabilities2) implements ClientInterface { + public ?Psr18Transport $transport = null; + /** @var list */ + public array $sent = []; + public ?array $overlapped = null; + + public function __construct(private string $d1, private array $c1, private string $d2, private array $c2) + { + } + + public function sendRequest(RequestInterface $request): ResponseInterface + { + $path = $request->getUri()->getPath(); + $held = $request->getHeaderLine(ServerCapabilitiesDigest::HEADER); + $this->sent[] = [$path, $held]; + $body = match ($path) { + // Learn D1. + '/api/worker/heartbeat' => ['server_capabilities' => $this->c1 + ['digest' => $this->d1]], + // The deploy lands; this answer moves the transport to D2. + '/api/worker/activity-tasks/poll' => ['task' => null, 'server_capabilities' => $this->c2 + ['digest' => $this->d2]], + // Committed under D1: the server elides with the digest this request sent. + default => ['outcome' => 'completed', 'recorded' => true, 'server_capabilities' => $held], + }; + if ($path === '/api/worker/workflow-tasks/t/complete') { + // Another request runs while the completion is in flight. + $this->overlapped = $this->transport->send('POST', 'https://runtime.test/api/worker/activity-tasks/poll', + ['X-Durable-Workflow-Protocol-Version' => '1.20'], ['timeout_seconds' => 0]); + } + + return new Response(200, ['Content-Type' => 'application/json'], json_encode($body, JSON_THROW_ON_ERROR)); + } + }; + $transport = new Psr18Transport(client: $http); + $http->transport = $transport; + $worker = ['X-Durable-Workflow-Protocol-Version' => '1.20']; + + $transport->send('POST', 'https://runtime.test/api/worker/heartbeat', $worker, []); + $completion = $transport->send('POST', 'https://runtime.test/api/worker/workflow-tasks/t/complete', $worker, []); + + self::assertSame(self::CAPABILITIES, $completion['server_capabilities'], 'Resolved against the pair it was sent with.'); + self::assertTrue($completion['recorded']); + self::assertSame($capabilities2, $http->overlapped['server_capabilities']); + self::assertSame([['/api/worker/heartbeat', 'none'], ['/api/worker/workflow-tasks/t/complete', self::DIGEST], + ['/api/worker/activity-tasks/poll', self::DIGEST]], $http->sent); + + // Requests sent after the overlap carry the newest digest. + $http->overlapped = null; + $after = $transport->send('POST', 'https://runtime.test/api/worker/workflow-tasks/u/complete', $worker, []); + self::assertSame($d2, $http->sent[3][1]); + self::assertSame($capabilities2, $after['server_capabilities']); + } + + public function testControlRequestsAndFullCapabilitiesAreUnchanged(): void + { + $http = $this->http([ + [200, [], json_encode(['server_capabilities' => self::CAPABILITIES], JSON_THROW_ON_ERROR)], + [200, [], json_encode(['server_capabilities' => self::CAPABILITIES], JSON_THROW_ON_ERROR)], + ]); + $transport = new Psr18Transport(client: $http); + $worker = $transport->send('POST', 'https://runtime.test/api/worker/heartbeat', ['X-Durable-Workflow-Protocol-Version' => '1.20'], []); + self::assertSame(self::CAPABILITIES, $worker['server_capabilities']); + $control = $transport->send('GET', 'https://runtime.test/api/workflows/x', ['X-Durable-Workflow-Control-Plane-Version' => '2']); + self::assertSame(self::CAPABILITIES, $control['server_capabilities']); + self::assertSame('none', $http->requests[0]->getHeaderLine(ServerCapabilitiesDigest::HEADER)); + self::assertFalse($http->requests[1]->hasHeader(ServerCapabilitiesDigest::HEADER)); + // A custom PSR-18 client is not asked to decode compressed bodies. + self::assertFalse($http->requests[0]->hasHeader('Accept-Encoding')); + } + + public function testAnUnknownDigestIsRecoveredByAskingAgainInFull(): void + { + $http = $this->http([ + [200, [], json_encode(['task' => null, 'server_capabilities' => 'sha256:'.str_repeat('f', 64)], JSON_THROW_ON_ERROR)], + [200, [], json_encode(['task' => null, 'server_capabilities' => self::CAPABILITIES], JSON_THROW_ON_ERROR)], + [200, [], json_encode(['task' => null, 'server_capabilities' => self::tagged()], JSON_THROW_ON_ERROR)], + ]); + $transport = new Psr18Transport(client: $http); + $worker = ['X-Durable-Workflow-Protocol-Version' => '1.20']; + + $answer = $transport->send('POST', 'https://runtime.test/api/worker/workflow-tasks/poll', $worker, ['timeout_seconds' => 0]); + self::assertSame(self::CAPABILITIES, $answer['server_capabilities']); + self::assertCount(2, $http->requests); + self::assertSame('none', $http->requests[0]->getHeaderLine(ServerCapabilitiesDigest::HEADER)); + self::assertFalse($http->requests[1]->hasHeader(ServerCapabilitiesDigest::HEADER), 'Retried without the header.'); + self::assertSame((string) $http->requests[0]->getBody(), (string) $http->requests[1]->getBody()); + // The next request asks for tagged capabilities again and learns them. + $transport->send('POST', 'https://runtime.test/api/worker/heartbeat', $worker, []); + self::assertSame('none', $http->requests[2]->getHeaderLine(ServerCapabilitiesDigest::HEADER)); + } + + public function testAGarbledDigestOnAMutatingAnswerKeepsTheBodyAndReadsCapabilitiesInstead(): void + { + $http = $this->http([ + [200, [], json_encode(['outcome' => 'completed', 'recorded' => true, 'next' => 'kept', + 'server_capabilities' => 'sha256:'.str_repeat('e', 64)], JSON_THROW_ON_ERROR)], + [200, [], json_encode(['worker_protocol' => ['version' => '1.20', 'server_capabilities' => self::CAPABILITIES]], JSON_THROW_ON_ERROR)], + ]); + $answer = (new Psr18Transport(client: $http))->send('POST', 'https://runtime.test/api/worker/workflow-tasks/t/complete', + ['X-Durable-Workflow-Protocol-Version' => '1.20', 'Authorization' => 'Bearer w'], ['commands' => []]); + + self::assertSame('kept', $answer['next'], 'The answer as received is processed.'); + self::assertSame(self::CAPABILITIES, $answer['server_capabilities']); + self::assertCount(2, $http->requests, 'The completion is never repeated.'); + self::assertSame('POST', $http->requests[0]->getMethod()); + self::assertSame('GET', $http->requests[1]->getMethod()); + self::assertSame('/api/cluster/info', $http->requests[1]->getUri()->getPath()); + self::assertSame('Bearer w', $http->requests[1]->getHeaderLine('Authorization')); + } + + public function testAGarbledDigestOnAPollDoesNotStopAManagedWorker(): void + { + $polls = 0; + $http = new class ($polls) implements ClientInterface { + /** @var list */ + public array $requests = []; + + public function __construct(public int $polls) + { + } + + public function sendRequest(RequestInterface $request): ResponseInterface + { + $this->requests[] = $request; + $path = $request->getUri()->getPath(); + $garbled = 'sha256:'.str_repeat('c', 64); + $body = match (true) { + str_ends_with($path, '/worker/register') => ['registered' => true], + str_ends_with($path, '/workflow-tasks/poll') => ['task' => null, 'poll_status' => 'empty', + 'server_capabilities' => ++$this->polls === 1 && $request->hasHeader(ServerCapabilitiesDigest::HEADER) + ? $garbled : ['query_tasks' => true]], + str_ends_with($path, '/query-tasks/poll') => ['task' => null, 'poll_status' => 'stopped', 'reason' => 'worker_stopped'], + default => ['task' => null, 'poll_status' => 'empty'], + }; + + return new Response(200, ['Content-Type' => 'application/json'], json_encode($body, JSON_THROW_ON_ERROR)); + } + }; + $worker = new \DurableWorkflow\Worker(new \DurableWorkflow\Client('https://runtime.test', + transport: new Psr18Transport(client: $http), workerToken: 'w'), 'orders', workerId: 'worker-1'); + + $worker->run(0); + + $paths = array_map(static fn (RequestInterface $request): string => $request->getUri()->getPath(), $http->requests); + self::assertSame(2, count(array_keys($paths, '/api/worker/workflow-tasks/poll')), 'The garbled poll was asked again in full.'); + self::assertContains('/api/worker/query-tasks/poll', $paths, 'The Worker kept running.'); + } + + /** + * The reviewer's scenario: a workflow-task completion whose answer carries + * the next leased activity arrives with every capabilities copy garbled. + * The completion is never repeated (a repeat would lose the carried task); + * the answer is kept, capabilities are read from cluster info, the carried + * activity runs once and completes once, and the Worker keeps running. + */ + public function testAGarbledCompletionAnswerCarryingTheNextActivityIsKeptNotResent(): void + { + $now = new \DateTimeImmutable('now', new \DateTimeZone('UTC')); + $caps = ['workflow_task_completion_activity_poll' => true]; + $codec = new \DurableWorkflow\Codec\AvroPayloadCodec(); + $garbled = 'sha256:'.str_repeat('9', 64); + $answer = static fn (array $task, mixed $capabilities): array => ['task' => $task + ['lease_owner' => 'worker-1', + 'payload_codec' => 'avro', 'lease_expires_at' => $now->modify('+30 seconds')->format('Y-m-d\TH:i:s.u\Z')], + 'poll_status' => 'leased', 'protocol_version' => '1.20', 'server_capabilities' => $capabilities, + 'server_time' => $now->format('Y-m-d\TH:i:s.u\Z')]; + $http = new class ($answer, $caps, $codec, $garbled) implements ClientInterface { + /** @var list */ + public array $requests = []; + public int $workflowPolls = 0; + + public function __construct(private \Closure $answer, private array $caps, private object $codec, private string $garbled) + { + } + + public function sendRequest(RequestInterface $request): ResponseInterface + { + $this->requests[] = $request; + $path = $request->getUri()->getPath(); + $answer = $this->answer; + $body = match (true) { + str_ends_with($path, '/worker/register') => ['registered' => true], + str_ends_with($path, '/cluster/info') => ['worker_protocol' => ['version' => '1.20', 'server_capabilities' => $this->caps]], + str_ends_with($path, '/workflow-tasks/poll') && ++$this->workflowPolls === 1 => $answer(['task_id' => 'workflow-task', + 'workflow_task_attempt' => 1, 'workflow_id' => 'workflow', 'run_id' => 'run', 'workflow_type' => 'carry', + 'history_events' => []], $this->caps), + str_ends_with($path, '/workflow-tasks/workflow-task/complete') => ['completed' => true, + 'activity_poll' => ['leased' => true, 'answer' => $answer(['task_id' => 'activity-1', + 'activity_attempt_id' => 'attempt-1', 'activity_type' => 'carried.activity', + 'arguments' => $this->codec->envelope([7])], $this->garbled)], + 'server_capabilities' => $this->garbled], + str_ends_with($path, '/activity-tasks/activity-1/complete') => ['recorded' => true, 'outcome' => 'completed'], + str_ends_with($path, '/query-tasks/poll') && $this->workflowPolls >= 2 => ['task' => null, 'poll_status' => 'stopped', 'reason' => 'worker_stopped'], + default => ['task' => null, 'poll_status' => 'empty'], + }; + + return new Response(200, ['Content-Type' => 'application/json'], json_encode($body, JSON_THROW_ON_ERROR)); + } + }; + $runs = 0; + $worker = new \DurableWorkflow\Worker(new \DurableWorkflow\Client('https://runtime.test', + transport: new Psr18Transport(client: $http), workerToken: 'w'), 'orders', workerId: 'worker-1'); + $worker->registerWorkflow('carry', static fn (\DurableWorkflow\Worker\WorkflowContext $context): string => 'done'); + $worker->registerActivity('carried.activity', static function (\DurableWorkflow\Worker\ActivityContext $context, int $value) use (&$runs): int { + ++$runs; + + return $value; + }); + + $worker->run(0); + + $paths = array_map(static fn (RequestInterface $request): string => $request->getUri()->getPath(), $http->requests); + self::assertSame(1, count(array_keys($paths, '/api/worker/workflow-tasks/workflow-task/complete')), 'Never repeated.'); + self::assertContains('/api/cluster/info', $paths, 'Capabilities were read, not re-requested by repeating.'); + self::assertSame(1, $runs, 'The carried activity ran exactly once.'); + self::assertSame(1, count(array_keys($paths, '/api/worker/activity-tasks/activity-1/complete')), 'And completed once.'); + } + + public function testAnUnknownDigestInAnErrorBodyNeverReplacesTheHttpError(): void + { + $http = $this->http([[409, [], json_encode(['reason' => 'stale_attempt', 'server_capabilities' => 'sha256:'.str_repeat('d', 64)], JSON_THROW_ON_ERROR)]]); + try { + (new Psr18Transport(client: $http))->send('POST', 'https://runtime.test/api/worker/activity-tasks/a/complete', + ['X-Durable-Workflow-Protocol-Version' => '1.20'], []); + self::fail('The 409 must surface.'); + } catch (TransportException $exception) { + self::assertSame(409, $exception->status); + self::assertSame('stale_attempt', $exception->response['reason']); + self::assertArrayNotHasKey('server_capabilities', $exception->response); + } + self::assertCount(1, $http->requests, 'An error answer is not retried.'); + } + + public function testAGzipBodyThatDecodesPastTheLimitIsRefused(): void + { + $body = (string) gzencode(json_encode(['blob' => str_repeat('z', 64 * 1024)], JSON_THROW_ON_ERROR)); + self::assertLessThan(1024, strlen($body)); + $http = $this->http([[200, ['Content-Encoding' => 'gzip'], $body]]); + try { + (new Psr18Transport(client: $http, maxDecodedResponseBytes: 16 * 1024))->send('GET', 'https://runtime.test/api/workflows/x', + ['X-Durable-Workflow-Control-Plane-Version' => '2']); + self::fail('A bomb must be refused.'); + } catch (ResponseTooLarge $exception) { + self::assertSame(16 * 1024, $exception->limitBytes); + self::assertSame('response_too_large', $exception->response['reason']); + } + // Within the default limit it decodes normally. + $http = $this->http([[200, ['Content-Encoding' => 'gzip'], $body]]); + self::assertSame(64 * 1024, strlen((new Psr18Transport(client: $http))->send('GET', 'https://runtime.test/api/workflows/x', + ['X-Durable-Workflow-Control-Plane-Version' => '2'])['blob'])); + } + + public function testTheDefaultDecodeLimitFitsPhpsMemoryLimit(): void + { + self::assertSame(32 * 1024 * 1024, Psr18Transport::defaultMaxDecodedResponseBytes('128M')); + self::assertSame(16 * 1024 * 1024, Psr18Transport::defaultMaxDecodedResponseBytes('64M')); + self::assertSame(32 * 1024 * 1024, Psr18Transport::defaultMaxDecodedResponseBytes('-1')); + self::assertSame(32 * 1024 * 1024, Psr18Transport::defaultMaxDecodedResponseBytes('2G')); + self::assertSame(1024 * 1024, Psr18Transport::defaultMaxDecodedResponseBytes('2M')); + } + + public function testAnOversizedResponseReachesAClientCallerAsResponseTooLarge(): void + { + $body = (string) gzencode(json_encode(['blob' => str_repeat('z', 64 * 1024)], JSON_THROW_ON_ERROR)); + $client = new \DurableWorkflow\Client('https://runtime.test', transport: new Psr18Transport( + client: $this->http([[200, ['Content-Encoding' => 'gzip'], $body]]), maxDecodedResponseBytes: 16 * 1024)); + $this->expectException(ResponseTooLarge::class); + $client->describeWorkflow('x'); + } + + public function testAGzipBodyLeftEncodedByTheClientIsDecoded(): void + { + $body = json_encode(['ok' => true, 'blob' => str_repeat('x', 4096)], JSON_THROW_ON_ERROR); + $http = $this->http([[200, ['Content-Encoding' => 'gzip'], (string) gzencode($body)]]); + $decoded = (new Psr18Transport(client: $http))->send('GET', 'https://runtime.test/api/workflows/x', + ['X-Durable-Workflow-Control-Plane-Version' => '2']); + self::assertSame(json_decode($body, true), $decoded); + } + + public function testTheOwnedGuzzleClientAsksForGzip(): void + { + if (!extension_loaded('curl') || !extension_loaded('zlib') || !function_exists('pcntl_fork')) { + self::markTestSkipped('The owned client decodes gzip through cURL and zlib; the peer needs pcntl.'); + } + $server = stream_socket_server('tcp://127.0.0.1:0', $errno, $error); + self::assertNotFalse($server, $error); + $address = stream_socket_get_name($server, false); + $pid = pcntl_fork(); + if ($pid === 0) { + $connection = stream_socket_accept($server, 10); + $request = ''; + while (!str_contains($request, "\r\n\r\n")) { + $request .= fread($connection, 8192); + } + $accepted = str_contains(strtolower($request), "\r\naccept-encoding: gzip\r\n"); + $body = json_encode(['gzip' => $accepted, 'blob' => str_repeat('y', 4096)], JSON_THROW_ON_ERROR); + $headers = $accepted ? "Content-Encoding: gzip\r\n" : ''; + $payload = $accepted ? (string) gzencode($body) : $body; + fwrite($connection, "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n{$headers}Content-Length: ".strlen($payload)."\r\nConnection: close\r\n\r\n".$payload); + fclose($connection); + exit(0); + } + try { + $decoded = (new Psr18Transport())->send('GET', 'http://'.$address.'/api/workflows/x', ['X-Durable-Workflow-Control-Plane-Version' => '2']); + self::assertSame(['gzip' => true, 'blob' => str_repeat('y', 4096)], $decoded); + } finally { + pcntl_waitpid($pid, $status); + fclose($server); + } + } +}