From da2569d4228f5afbed6cf55cc9215b0f782b1d22 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 20 Nov 2026 00:56:51 +0000 Subject: [PATCH 1/5] Cut worker and result response bytes: gzip, capabilities digest, history-free result A lab run of DW Standard Workflow v1 against native Server moved about 385 KB of responses per workflow, enough to saturate a Cloud Dev cell's 4 Mbit/s egress at 1.5 workflows/second. - The default transport sends Accept-Encoding: gzip when it owns a Guzzle client with cURL and zlib (cURL decodes), and gunzips a body a custom client left encoded (gzip magic plus Content-Encoding: gzip). - Worker requests send X-Durable-Workflow-Server-Capabilities-Digest. A Server that supports it replaces repeated server_capabilities objects with their digest; ServerCapabilitiesDigest restores the held object in success bodies, refusals and poll frames (top level and carried answers) before any caller sees them. Older Servers ignore the header. - result() of a completed run uses the terminal status read's output_envelope instead of downloading the whole history again; other terminal states and Servers without output_envelope still read history. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 20 +++ src/Client.php | 21 ++- src/Transport/Psr18Transport.php | 38 +++++- src/Transport/ServerCapabilitiesDigest.php | 63 +++++++++ tests/ClientWorkflowResultTest.php | 21 +-- tests/ServerCapabilitiesDigestTest.php | 146 +++++++++++++++++++++ 6 files changed, 294 insertions(+), 15 deletions(-) create mode 100644 src/Transport/ServerCapabilitiesDigest.php create mode 100644 tests/ServerCapabilitiesDigestTest.php diff --git a/CHANGELOG.md b/CHANGELOG.md index f89d35d0..b9831e66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,6 +63,26 @@ project follows [Semantic Versioning](https://semver.org/). activity retry policy is unchanged, because retrying a non-idempotent activity by default could repeat its side effects. +### Changed + +- The default transport asks for gzip on every request when it owns a Guzzle + client with cURL and zlib, and decodes a gzip body a custom client left + encoded. Servers that compress (PHP Server, and native Server with its + response compression) then send several times fewer bytes. +- Worker requests send `X-Durable-Workflow-Server-Capabilities-Digest`. A + Server that supports it replaces repeated `server_capabilities` objects + (about 17 KB on native, sent on every worker answer and again inside + carried polls) with their digest, and the transport restores the held object + before any caller sees the answer. Older Servers ignore the header. +- `result()` of a completed run reads the output from the terminal status + read instead of downloading the run's whole history again. Failed, + cancelled, terminated and timed-out runs, and Servers that omit + `output_envelope`, still read the history. + +In a lab DW Standard Workflow v1 run (native Server with compression and the +digest, 1 workflow/second), response bytes per workflow went from about +385 KB to about 18 KB. + ## [3.22.0] - 2026-11-19 Signal-with-start and durable signal-external need native Server 3.15.0 or diff --git a/src/Client.php b/src/Client.php index a6f63fc8..ca8b9012 100644 --- a/src/Client.php +++ b/src/Client.php @@ -3634,6 +3634,12 @@ private function terminalResult( string $status, WorkflowExecution $execution, ): mixed { + if ($status === 'completed' && is_array($execution->raw['output_envelope'] ?? null) + && ($execution->raw['run_id'] ?? null) === $runId) { + // The terminal read already names the run's output; reading the + // whole history again only repeats it. + return $this->completedRunOutput($workflowId, $runId, $execution); + } $history = $this->request( 'GET', '/workflows/'.$this->segment($workflowId).'/runs/'.$this->segment($runId).'/history', false, payloadPaths: self::terminalHistoryPayloadPaths(...), @@ -3673,11 +3679,7 @@ private function terminalResult( } return match ($status) { - 'completed' => $this->decodedResponsePayload( - (new RuntimePayloads($this->transport, $this->baseUri, $this->requestHeaders(false), $this->maxExternalPayloadBytes)) - ->response($execution->raw, '/workflows/'.$this->segment($workflowId).'/runs/'.$this->segment($runId), false, [['output_envelope']]), - 'output', - ), + 'completed' => $this->completedRunOutput($workflowId, $runId, $execution), 'cancelled', 'canceled' => throw new WorkflowCancelled('Workflow was cancelled.'), 'terminated' => throw new WorkflowTerminated('Workflow was terminated.'), 'timed_out', 'timeout' => throw new WorkflowTimedOut('Workflow execution timed out.'), @@ -3685,6 +3687,15 @@ private function terminalResult( }; } + private function completedRunOutput(string $workflowId, string $runId, WorkflowExecution $execution): mixed + { + return $this->decodedResponsePayload( + (new RuntimePayloads($this->transport, $this->baseUri, $this->requestHeaders(false), $this->maxExternalPayloadBytes)) + ->response($execution->raw, '/workflows/'.$this->segment($workflowId).'/runs/'.$this->segment($runId), false, [['output_envelope']]), + 'output', + ); + } + /** @param array $response * @return list> */ diff --git a/src/Transport/Psr18Transport.php b/src/Transport/Psr18Transport.php index a83c1316..0c231120 100644 --- a/src/Transport/Psr18Transport.php +++ b/src/Transport/Psr18Transport.php @@ -42,6 +42,7 @@ final class Psr18Transport implements BoundedPayloadTransport, BoundedPayloadUpl private ?string $completionUri = null; private bool $completionArmed = false; private ?CompletionRequestHandler $completionHandler = null; + private readonly ServerCapabilitiesDigest $capabilitiesDigest; public function lastResponseStatus(): ?int { @@ -71,6 +72,7 @@ public function __construct( $this->workerRequestTimeoutSeconds = $client === null ? ($workerRequestTimeoutSeconds ?? self::DEFAULT_WORKER_REQUEST_TIMEOUT_SECONDS) : null; $factory = new HttpFactory(); + $this->capabilitiesDigest = new ServerCapabilitiesDigest(); $this->defaultProcessConfiguration = $client === null && $requestFactory === null && $streamFactory === null; $this->client = $client ?? new GuzzleClient(['http_errors' => false]); $this->requestFactory = $requestFactory ?? $factory; @@ -87,6 +89,12 @@ public function send(string $method, string $uri, array $headers, ?array $body = return $this->sendJson($method, $uri, $headers, $body); } + /** The owned Guzzle client over cURL decodes gzip bodies it asked for. */ + private function decodesCompressedResponses(): bool + { + return $this->client instanceof GuzzleClient && extension_loaded('curl') && extension_loaded('zlib'); + } + public function supportsBoundedRequests(): bool { return $this->client instanceof GuzzleClient && extension_loaded('curl'); @@ -182,11 +190,18 @@ private function sendJson(string $method, string $uri, array $headers, ?array $b if ($poll) { $request = $request->withHeader('Accept', PollTransport::MEDIA_TYPE.', application/json;q=0.9'); } + if (!$control) { + $request = $request->withHeader(ServerCapabilitiesDigest::HEADER, $this->capabilitiesDigest->header()); + } + if ($this->decodesCompressedResponses() && !$request->hasHeader('Accept-Encoding')) { + // cURL decodes the body; Guzzle then moves Content-Encoding aside. + $request = $request->withHeader('Accept-Encoding', 'gzip'); + } $response = $this->sendRequest($request, timeoutSeconds: $timeoutSeconds ?? $controlTimeout ?? 30, bounded: $timeoutSeconds !== null || $controlTimeout !== null, controlTimeout: $controlTimeout, control: $control); $this->responseStatus = $response->getStatusCode(); - return $this->decodeJsonResponse($response, $poll); + return $this->decodeJsonResponse($response, $poll, !$control); } catch (TransportException $exception) { throw $exception; } catch (JsonException $exception) { @@ -210,13 +225,26 @@ private function pollTimeoutSeconds(?array $body): int|float } /** @return array|list|null */ - private function decodeJsonResponse(ResponseInterface $response, bool $poll = false): ?array + private function decodeJsonResponse(ResponseInterface $response, bool $poll = false, bool $worker = false): ?array { $rawBody = (string) $response->getBody(); + if (str_starts_with($rawBody, "\x1f\x8b") && function_exists('gzdecode') + && strtolower(trim($response->getHeaderLine('Content-Encoding'))) === 'gzip') { + // A client that left the gzip body encoded. Decoding clients + // either drop the header or already return the JSON bytes. + $inflated = @gzdecode($rawBody); + if (!is_string($inflated)) { + throw new TransportException('The server returned an invalid gzip response body.'); + } + $rawBody = $inflated; + } $status = $response->getStatusCode(); $mediaType = strtolower(trim(explode(';', $response->getHeaderLine('Content-Type'), 2)[0])); if ($poll && $mediaType === PollTransport::MEDIA_TYPE) { $frame = PollTransport::decode($rawBody, $status); + if ($worker) { + $this->capabilitiesDigest->restore($frame['body']); + } if ($frame['status'] >= 300) { throw TransportException::fromResponse($frame['status'], $frame['body'], $rawBody, $frame['headers']); } @@ -227,6 +255,9 @@ private function decodeJsonResponse(ResponseInterface $response, bool $poll = fa // An upstream proxy can return HTML or an empty body. Preserve // its status without interpreting it as a successful protocol reply. $decoded = json_decode($rawBody, true); + if ($worker && is_array($decoded)) { + $this->capabilitiesDigest->restore($decoded); + } throw TransportException::fromResponse($status, is_array($decoded) ? $decoded : null, $rawBody); } @@ -234,6 +265,9 @@ private function decodeJsonResponse(ResponseInterface $response, bool $poll = fa if ($decoded !== null && !is_array($decoded)) { throw new TransportException('The server returned a JSON value instead of an object or array.'); } + if ($worker && $decoded !== null) { + $this->capabilitiesDigest->restore($decoded); + } return $decoded; } diff --git a/src/Transport/ServerCapabilitiesDigest.php b/src/Transport/ServerCapabilitiesDigest.php new file mode 100644 index 00000000..b89d4ef8 --- /dev/null +++ b/src/Transport/ServerCapabilitiesDigest.php @@ -0,0 +1,63 @@ +|null */ + private ?array $capabilities = null; + + public function header(): string + { + return $this->digest ?? 'none'; + } + + /** + * Restore elided copies and keep a tagged one, in place. + * + * @param array $answer + */ + public function restore(array &$answer, int $depth = 1): void + { + if (array_is_list($answer)) { + return; + } + foreach ($answer as $key => &$value) { + if ($key === 'server_capabilities') { + if (is_string($value) && str_starts_with($value, 'sha256:')) { + if ($value !== $this->digest || $this->capabilities === null) { + throw new TransportException('The server omitted capabilities this worker does not hold.'); + } + $value = $this->capabilities; + } elseif (is_array($value) && !array_is_list($value) && is_string($value['digest'] ?? null) + && str_starts_with($value['digest'], 'sha256:')) { + $digest = $value['digest']; + unset($value['digest']); + $this->digest = $digest; + $this->capabilities = $value; + } + } elseif (is_array($value) && $depth < self::MAX_DEPTH) { + $this->restore($value, $depth + 1); + } + } + unset($value); + } +} diff --git a/tests/ClientWorkflowResultTest.php b/tests/ClientWorkflowResultTest.php index 04510094..674add1e 100644 --- a/tests/ClientWorkflowResultTest.php +++ b/tests/ClientWorkflowResultTest.php @@ -35,7 +35,8 @@ public function testWaitingForStatusDoesNotDownloadUnusedInputOrOutput(): void #[DataProvider('historyFields')] public function testOnlyTheTerminalHistoryPayloadIsDownloaded(string $historyField): void { - [$client, $requests] = $this->payloadClient('completed', $historyField); + // A terminal read without the run output falls back to the history. + [$client, $requests] = $this->payloadClient('completed', $historyField, describeOutput: false); self::assertSame(['finished' => true], $client->workflowHandle('order', 'run')->resultOfSelectedRun(timeoutSeconds: 0)); self::assertSame(['/api/workflows/order/runs/run', '/api/workflows/order/runs/run/history', '/api/external-payloads/v1/ep_01ARZ3NDEKTSV4RRFFQ69G5FAV'], array_map(static fn (RequestInterface $request): string => $request->getUri()->getPath(), $requests->value)); @@ -48,12 +49,16 @@ public static function historyFields(): iterable yield 'history_events' => ['history_events']; } - public function testMissingTerminalHistoryStillUsesTheRunOutput(): void + public function testACompletedRunOutputNeedsNoHistoryRead(): void { [$client, $requests] = $this->payloadClient('completed', emptyHistory: true); self::assertSame(['finished' => true], $client->workflowHandle('order', 'run')->resultOfSelectedRun(timeoutSeconds: 0)); - self::assertCount(3, $requests->value); - self::assertStringEndsWith('/ep_01ARZ3NDEKTSV4RRFFQ69G5FAV', (string) $requests->value[2]->getUri()); + self::assertSame(['/api/workflows/order/runs/run', '/api/external-payloads/v1/ep_01ARZ3NDEKTSV4RRFFQ69G5FAV'], + array_map(static fn (RequestInterface $request): string => $request->getUri()->getPath(), $requests->value)); + [$client, $requests] = $this->payloadClient('completed', emptyHistory: true); + self::assertSame(['finished' => true], $client->workflowHandle('order', 'run')->result(timeoutSeconds: 0)); + self::assertSame(['/api/workflows/order', '/api/external-payloads/v1/ep_01ARZ3NDEKTSV4RRFFQ69G5FAV'], + array_map(static fn (RequestInterface $request): string => $request->getUri()->getPath(), $requests->value)); } public function testAnUnreadableInputDoesNotHideTheTerminalFailure(): void @@ -81,7 +86,7 @@ public function testTheSelectedTerminalOutputStillRequiresValidIntegrityMetadata } /** @return array{Client, object} */ - private function payloadClient(string $status, string $historyField = 'events', bool $emptyHistory = false, bool $corruptOutput = false): array + private function payloadClient(string $status, string $historyField = 'events', bool $emptyHistory = false, bool $corruptOutput = false, bool $describeOutput = true): array { $codec = new AvroPayloadCodec(); $blob = $codec->encode(['finished' => true]); @@ -89,7 +94,7 @@ private function payloadClient(string $status, string $historyField = 'events', $output = ['codec' => 'avro', 'external_payload' => $reference]; $unused = ['codec' => 'avro', 'external_payload' => array_replace($reference, ['reference_id' => 'ep_01ARZ3NDEKTSV4RRFFQ69G5FAW'])]; $requests = new class { public array $value = []; }; - $http = new GuzzleClient(['handler' => static function (RequestInterface $request) use ($requests, $status, $historyField, $emptyHistory, $blob, $output, $unused, $corruptOutput) { + $http = new GuzzleClient(['handler' => static function (RequestInterface $request) use ($requests, $status, $historyField, $emptyHistory, $blob, $output, $unused, $corruptOutput, $describeOutput) { $requests->value[] = $request; $path = $request->getUri()->getPath(); if (str_ends_with($path, '/ep_01ARZ3NDEKTSV4RRFFQ69G5FAW')) { @@ -109,8 +114,8 @@ private function payloadClient(string $status, string $historyField = 'events', : ['event_type' => 'WorkflowCompleted', 'payload' => ['output' => $output, 'result' => $unused]], ]]; } else { - $body = ['workflow_id' => 'order', 'run_id' => 'run', 'status' => $status, - 'input_envelope' => $unused, 'output_envelope' => $output]; + $body = ['workflow_id' => 'order', 'run_id' => 'run', 'status' => $status, 'input_envelope' => $unused] + + ($describeOutput ? ['output_envelope' => $output] : []); } return Create::promiseFor(new Response(200, [], json_encode($body, JSON_THROW_ON_ERROR))); diff --git a/tests/ServerCapabilitiesDigestTest.php b/tests/ServerCapabilitiesDigestTest.php new file mode 100644 index 00000000..6f34ef63 --- /dev/null +++ b/tests/ServerCapabilitiesDigestTest.php @@ -0,0 +1,146 @@ + true, 'activity_lease_renewal' => true, + 'supported_workflow_task_commands' => ['schedule_activity', 'complete_workflow']]; + + /** @param list, 2: string}> $responses */ + private function http(array $responses): ClientInterface + { + return new class ($responses) implements ClientInterface { + /** @var list */ + public array $requests = []; + + /** @param list, 2: string}> $responses */ + public function __construct(private array $responses) + { + } + + public function sendRequest(RequestInterface $request): ResponseInterface + { + $this->requests[] = $request; + [$status, $headers, $body] = array_shift($this->responses); + + return new Response($status, $headers + ['Content-Type' => 'application/json'], $body); + } + }; + } + + private static function tagged(): array + { + return self::CAPABILITIES + ['digest' => self::DIGEST]; + } + + public function testAWorkerKeepsTaggedCapabilitiesAndRestoresElidedCopies(): void + { + $http = $this->http([ + [200, [], json_encode(['task' => null, 'server_capabilities' => self::tagged()], JSON_THROW_ON_ERROR)], + [200, [], json_encode(['outcome' => 'completed', 'activity_poll' => ['leased' => true, + 'answer' => ['task' => ['id' => 'a'], 'server_capabilities' => self::DIGEST]], + 'server_capabilities' => self::DIGEST], JSON_THROW_ON_ERROR)], + [409, [], json_encode(['reason' => 'lease_expired', 'server_capabilities' => self::DIGEST], JSON_THROW_ON_ERROR)], + ]); + $transport = new Psr18Transport(client: $http); + $worker = ['X-Durable-Workflow-Protocol-Version' => '1.20']; + + $first = $transport->send('POST', 'https://runtime.test/api/worker/heartbeat', $worker, []); + self::assertSame(self::CAPABILITIES, $first['server_capabilities']); + self::assertSame('none', $http->requests[0]->getHeaderLine(ServerCapabilitiesDigest::HEADER)); + + $second = $transport->send('POST', 'https://runtime.test/api/worker/workflow-tasks/t/complete', $worker, []); + self::assertSame(self::DIGEST, $http->requests[1]->getHeaderLine(ServerCapabilitiesDigest::HEADER)); + self::assertSame(self::CAPABILITIES, $second['server_capabilities']); + self::assertSame(self::CAPABILITIES, $second['activity_poll']['answer']['server_capabilities']); + self::assertSame(['id' => 'a'], $second['activity_poll']['answer']['task']); + + try { + $transport->send('POST', 'https://runtime.test/api/worker/activity-tasks/a/complete', $worker, []); + self::fail('A 409 remains a refusal.'); + } catch (TransportException $exception) { + self::assertSame(409, $exception->status); + self::assertSame(self::CAPABILITIES, $exception->response['server_capabilities']); + } + } + + public function testControlRequestsAndFullCapabilitiesAreUnchanged(): void + { + $http = $this->http([ + [200, [], json_encode(['server_capabilities' => self::CAPABILITIES], JSON_THROW_ON_ERROR)], + [200, [], json_encode(['server_capabilities' => self::CAPABILITIES], JSON_THROW_ON_ERROR)], + ]); + $transport = new Psr18Transport(client: $http); + $worker = $transport->send('POST', 'https://runtime.test/api/worker/heartbeat', ['X-Durable-Workflow-Protocol-Version' => '1.20'], []); + self::assertSame(self::CAPABILITIES, $worker['server_capabilities']); + $control = $transport->send('GET', 'https://runtime.test/api/workflows/x', ['X-Durable-Workflow-Control-Plane-Version' => '2']); + self::assertSame(self::CAPABILITIES, $control['server_capabilities']); + self::assertSame('none', $http->requests[0]->getHeaderLine(ServerCapabilitiesDigest::HEADER)); + self::assertFalse($http->requests[1]->hasHeader(ServerCapabilitiesDigest::HEADER)); + // A custom PSR-18 client is not asked to decode compressed bodies. + self::assertFalse($http->requests[0]->hasHeader('Accept-Encoding')); + } + + public function testAnUnknownDigestIsAProtocolFailure(): void + { + $http = $this->http([[200, [], json_encode(['server_capabilities' => self::DIGEST], JSON_THROW_ON_ERROR)]]); + $this->expectException(TransportException::class); + $this->expectExceptionMessage('The server omitted capabilities this worker does not hold.'); + (new Psr18Transport(client: $http))->send('POST', 'https://runtime.test/api/worker/heartbeat', + ['X-Durable-Workflow-Protocol-Version' => '1.20'], []); + } + + public function testAGzipBodyLeftEncodedByTheClientIsDecoded(): void + { + $body = json_encode(['ok' => true, 'blob' => str_repeat('x', 4096)], JSON_THROW_ON_ERROR); + $http = $this->http([[200, ['Content-Encoding' => 'gzip'], (string) gzencode($body)]]); + $decoded = (new Psr18Transport(client: $http))->send('GET', 'https://runtime.test/api/workflows/x', + ['X-Durable-Workflow-Control-Plane-Version' => '2']); + self::assertSame(json_decode($body, true), $decoded); + } + + public function testTheOwnedGuzzleClientAsksForGzip(): void + { + if (!extension_loaded('curl') || !extension_loaded('zlib') || !function_exists('pcntl_fork')) { + self::markTestSkipped('The owned client decodes gzip through cURL and zlib; the peer needs pcntl.'); + } + $server = stream_socket_server('tcp://127.0.0.1:0', $errno, $error); + self::assertNotFalse($server, $error); + $address = stream_socket_get_name($server, false); + $pid = pcntl_fork(); + if ($pid === 0) { + $connection = stream_socket_accept($server, 10); + $request = ''; + while (!str_contains($request, "\r\n\r\n")) { + $request .= fread($connection, 8192); + } + $accepted = str_contains(strtolower($request), "\r\naccept-encoding: gzip\r\n"); + $body = json_encode(['gzip' => $accepted, 'blob' => str_repeat('y', 4096)], JSON_THROW_ON_ERROR); + $headers = $accepted ? "Content-Encoding: gzip\r\n" : ''; + $payload = $accepted ? (string) gzencode($body) : $body; + fwrite($connection, "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n{$headers}Content-Length: ".strlen($payload)."\r\nConnection: close\r\n\r\n".$payload); + fclose($connection); + exit(0); + } + try { + $decoded = (new Psr18Transport())->send('GET', 'http://'.$address.'/api/workflows/x', ['X-Durable-Workflow-Control-Plane-Version' => '2']); + self::assertSame(['gzip' => true, 'blob' => str_repeat('y', 4096)], $decoded); + } finally { + pcntl_waitpid($pid, $status); + fclose($server); + } + } +} From 84fe01dfd33fd25c28cacd2db6b6d012b5f2e83c Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 20 Nov 2026 01:49:28 +0000 Subject: [PATCH 2/5] Resolve an elided digest against the pair its request was sent with Review fix. The transport held one digest, so a completion sent under D1 whose answer arrived after another response moved it to D2 (a rolling deploy with overlapping requests, as completion pipelining allows) threw "capabilities this worker does not hold" even though the server had committed it. The server only ever replaces copies with the digest that request sent, so each worker request now captures the digest and object it is sent with, and its answer (success, refusal or poll frame) is restored against that pair. A tagged copy still updates the pair used by later requests. Test: a completion in flight under D1 overlaps a poll that learns D2; the completion restores the D1 object, the poll the D2 object, and the next request sends D2. It fails on the previous single-digest code. Co-Authored-By: Claude Opus 5.5 --- src/Transport/Psr18Transport.php | 27 ++++++---- src/Transport/ServerCapabilitiesDigest.php | 55 +++++++++++++++---- tests/ServerCapabilitiesDigestTest.php | 62 ++++++++++++++++++++++ 3 files changed, 122 insertions(+), 22 deletions(-) diff --git a/src/Transport/Psr18Transport.php b/src/Transport/Psr18Transport.php index 0c231120..c9a9a613 100644 --- a/src/Transport/Psr18Transport.php +++ b/src/Transport/Psr18Transport.php @@ -190,8 +190,10 @@ private function sendJson(string $method, string $uri, array $headers, ?array $b if ($poll) { $request = $request->withHeader('Accept', PollTransport::MEDIA_TYPE.', application/json;q=0.9'); } - if (!$control) { - $request = $request->withHeader(ServerCapabilitiesDigest::HEADER, $this->capabilitiesDigest->header()); + // The answer is resolved against the pair this request was sent with. + $captured = $control ? null : $this->capabilitiesDigest->capture(); + if ($captured !== null) { + $request = $request->withHeader(ServerCapabilitiesDigest::HEADER, $this->capabilitiesDigest->header($captured)); } if ($this->decodesCompressedResponses() && !$request->hasHeader('Accept-Encoding')) { // cURL decodes the body; Guzzle then moves Content-Encoding aside. @@ -201,7 +203,7 @@ private function sendJson(string $method, string $uri, array $headers, ?array $b bounded: $timeoutSeconds !== null || $controlTimeout !== null, controlTimeout: $controlTimeout, control: $control); $this->responseStatus = $response->getStatusCode(); - return $this->decodeJsonResponse($response, $poll, !$control); + return $this->decodeJsonResponse($response, $poll, $captured); } catch (TransportException $exception) { throw $exception; } catch (JsonException $exception) { @@ -224,8 +226,11 @@ private function pollTimeoutSeconds(?array $body): int|float ? min($timeout, (float) $configured) : $timeout; } - /** @return array|list|null */ - private function decodeJsonResponse(ResponseInterface $response, bool $poll = false, bool $worker = false): ?array + /** + * @param array{0: ?string, 1: array|null}|null $captured The digest pair a worker request was sent with. + * @return array|list|null + */ + private function decodeJsonResponse(ResponseInterface $response, bool $poll = false, ?array $captured = null): ?array { $rawBody = (string) $response->getBody(); if (str_starts_with($rawBody, "\x1f\x8b") && function_exists('gzdecode') @@ -242,8 +247,8 @@ private function decodeJsonResponse(ResponseInterface $response, bool $poll = fa $mediaType = strtolower(trim(explode(';', $response->getHeaderLine('Content-Type'), 2)[0])); if ($poll && $mediaType === PollTransport::MEDIA_TYPE) { $frame = PollTransport::decode($rawBody, $status); - if ($worker) { - $this->capabilitiesDigest->restore($frame['body']); + if ($captured !== null) { + $this->capabilitiesDigest->restore($frame['body'], $captured); } if ($frame['status'] >= 300) { throw TransportException::fromResponse($frame['status'], $frame['body'], $rawBody, $frame['headers']); @@ -255,8 +260,8 @@ private function decodeJsonResponse(ResponseInterface $response, bool $poll = fa // An upstream proxy can return HTML or an empty body. Preserve // its status without interpreting it as a successful protocol reply. $decoded = json_decode($rawBody, true); - if ($worker && is_array($decoded)) { - $this->capabilitiesDigest->restore($decoded); + if ($captured !== null && is_array($decoded)) { + $this->capabilitiesDigest->restore($decoded, $captured); } throw TransportException::fromResponse($status, is_array($decoded) ? $decoded : null, $rawBody); } @@ -265,8 +270,8 @@ private function decodeJsonResponse(ResponseInterface $response, bool $poll = fa if ($decoded !== null && !is_array($decoded)) { throw new TransportException('The server returned a JSON value instead of an object or array.'); } - if ($worker && $decoded !== null) { - $this->capabilitiesDigest->restore($decoded); + if ($captured !== null && $decoded !== null) { + $this->capabilitiesDigest->restore($decoded, $captured); } return $decoded; diff --git a/src/Transport/ServerCapabilitiesDigest.php b/src/Transport/ServerCapabilitiesDigest.php index b89d4ef8..ed0a897e 100644 --- a/src/Transport/ServerCapabilitiesDigest.php +++ b/src/Transport/ServerCapabilitiesDigest.php @@ -12,9 +12,14 @@ * A worker request sends the digest of the capabilities this transport holds * (or `none`). A server that supports the digest answers with each copy of * its current capabilities either tagged with a `digest` member, which is - * kept here, or replaced by the digest string, which is restored here before - * any caller sees the answer. Older servers ignore the header and answer in - * full, so callers always observe complete capabilities. + * kept here for later requests, or replaced by the digest string, which is + * restored here before any caller sees the answer. Older servers ignore the + * header and answer in full, so callers always observe complete capabilities. + * + * The server only ever replaces copies with the digest that request sent, so + * an answer is resolved against the pair captured when its request was sent, + * not the pair held when it arrives: overlapping requests across a change of + * capabilities (a rolling deploy) each restore their own object. */ final class ServerCapabilitiesDigest { @@ -25,17 +30,41 @@ final class ServerCapabilitiesDigest /** @var array|null */ private ?array $capabilities = null; - public function header(): string + /** + * The pair a request is sent with; pass it back to restore() for its answer. + * + * @return array{0: ?string, 1: array|null} + */ + public function capture(): array + { + return [$this->digest, $this->capabilities]; + } + + /** @param array{0: ?string, 1: array|null}|null $captured */ + public function header(?array $captured = null): string { - return $this->digest ?? 'none'; + return ($captured ?? $this->capture())[0] ?? 'none'; } /** * Restore elided copies and keep a tagged one, in place. * * @param array $answer + * @param array{0: ?string, 1: array|null}|null $captured The pair its request was sent with. + */ + public function restore(array &$answer, ?array $captured = null): void + { + $learned = [null, null]; + $this->restoreLevel($answer, $captured ?? $this->capture(), $learned, 1); + } + + /** + * @param array $answer + * @param array{0: ?string, 1: array|null} $captured + * @param array{0: ?string, 1: array|null} $learned A tagged copy seen in this answer. + * @param-out array{0: ?string, 1: array|null} $learned */ - public function restore(array &$answer, int $depth = 1): void + private function restoreLevel(array &$answer, array $captured, array &$learned, int $depth): void { if (array_is_list($answer)) { return; @@ -43,19 +72,23 @@ public function restore(array &$answer, int $depth = 1): void foreach ($answer as $key => &$value) { if ($key === 'server_capabilities') { if (is_string($value) && str_starts_with($value, 'sha256:')) { - if ($value !== $this->digest || $this->capabilities === null) { - throw new TransportException('The server omitted capabilities this worker does not hold.'); - } - $value = $this->capabilities; + $value = match (true) { + $value === $captured[0] && $captured[1] !== null => $captured[1], + $value === $learned[0] && $learned[1] !== null => $learned[1], + default => throw new TransportException('The server omitted capabilities this worker does not hold.'), + }; } elseif (is_array($value) && !array_is_list($value) && is_string($value['digest'] ?? null) && str_starts_with($value['digest'], 'sha256:')) { $digest = $value['digest']; unset($value['digest']); + /** @var array $value */ + $learned = [$digest, $value]; + // Later requests send the newest digest this transport has seen. $this->digest = $digest; $this->capabilities = $value; } } elseif (is_array($value) && $depth < self::MAX_DEPTH) { - $this->restore($value, $depth + 1); + $this->restoreLevel($value, $captured, $learned, $depth + 1); } } unset($value); diff --git a/tests/ServerCapabilitiesDigestTest.php b/tests/ServerCapabilitiesDigestTest.php index 6f34ef63..418eadcc 100644 --- a/tests/ServerCapabilitiesDigestTest.php +++ b/tests/ServerCapabilitiesDigestTest.php @@ -77,6 +77,68 @@ public function testAWorkerKeepsTaggedCapabilitiesAndRestoresElidedCopies(): voi } } + /** + * Rolling deploy: a completion is in flight under D1 while another worker + * request (as completion pipelining overlaps them) learns D2. The server + * answers the completion with the digest it was sent, D1, after committing + * it; the transport must restore the D1 object, not refuse the answer. + */ + public function testOverlappingRequestsAcrossTwoDigestsEachRestoreTheirOwnCapabilities(): void + { + $d2 = 'sha256:'.str_repeat('2', 64); + $capabilities2 = ['query_tasks' => true, 'activity_lease_renewal' => true, 'new_after_deploy' => true]; + $http = new class (self::DIGEST, self::CAPABILITIES, $d2, $capabilities2) implements ClientInterface { + public ?Psr18Transport $transport = null; + /** @var list */ + public array $sent = []; + public ?array $overlapped = null; + + public function __construct(private string $d1, private array $c1, private string $d2, private array $c2) + { + } + + public function sendRequest(RequestInterface $request): ResponseInterface + { + $path = $request->getUri()->getPath(); + $held = $request->getHeaderLine(ServerCapabilitiesDigest::HEADER); + $this->sent[] = [$path, $held]; + $body = match ($path) { + // Learn D1. + '/api/worker/heartbeat' => ['server_capabilities' => $this->c1 + ['digest' => $this->d1]], + // The deploy lands; this answer moves the transport to D2. + '/api/worker/activity-tasks/poll' => ['task' => null, 'server_capabilities' => $this->c2 + ['digest' => $this->d2]], + // Committed under D1: the server elides with the digest this request sent. + default => ['outcome' => 'completed', 'recorded' => true, 'server_capabilities' => $held], + }; + if ($path === '/api/worker/workflow-tasks/t/complete') { + // Another request runs while the completion is in flight. + $this->overlapped = $this->transport->send('POST', 'https://runtime.test/api/worker/activity-tasks/poll', + ['X-Durable-Workflow-Protocol-Version' => '1.20'], ['timeout_seconds' => 0]); + } + + return new Response(200, ['Content-Type' => 'application/json'], json_encode($body, JSON_THROW_ON_ERROR)); + } + }; + $transport = new Psr18Transport(client: $http); + $http->transport = $transport; + $worker = ['X-Durable-Workflow-Protocol-Version' => '1.20']; + + $transport->send('POST', 'https://runtime.test/api/worker/heartbeat', $worker, []); + $completion = $transport->send('POST', 'https://runtime.test/api/worker/workflow-tasks/t/complete', $worker, []); + + self::assertSame(self::CAPABILITIES, $completion['server_capabilities'], 'Resolved against the pair it was sent with.'); + self::assertTrue($completion['recorded']); + self::assertSame($capabilities2, $http->overlapped['server_capabilities']); + self::assertSame([['/api/worker/heartbeat', 'none'], ['/api/worker/workflow-tasks/t/complete', self::DIGEST], + ['/api/worker/activity-tasks/poll', self::DIGEST]], $http->sent); + + // Requests sent after the overlap carry the newest digest. + $http->overlapped = null; + $after = $transport->send('POST', 'https://runtime.test/api/worker/workflow-tasks/u/complete', $worker, []); + self::assertSame($d2, $http->sent[3][1]); + self::assertSame($capabilities2, $after['server_capabilities']); + } + public function testControlRequestsAndFullCapabilitiesAreUnchanged(): void { $http = $this->http([ From b8b8988c6620b013fb2da96e298e438fe1079d3f Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 20 Nov 2026 06:40:54 +0000 Subject: [PATCH 3/5] Recover from an unknown digest; cap decoded gzip; keep HTTP errors intact Review fixes: - An answer naming capabilities the transport does not hold no longer ends the Worker. The transport forgets its pair and sends that request once more without the digest header, which returns the full object. Worker requests are fenced, so a request the Server already committed comes back as a duplicate 409 (task_not_leased / stale_attempt) that the Worker already handles. - A digest the transport does not hold in an ERROR body is dropped; the HTTP error surfaces unchanged and is not retried. - gzip bodies are inflated by the transport (cURL no longer inflates the bodies it asked for), in 8 KiB input steps, within maxDecodedResponseBytes (default 128 MiB, configurable on Psr18Transport). A larger body fails with the typed DurableWorkflow\Exception\ResponseTooLarge. - Tests: garbled digest on a poll (retried in full; a managed Worker keeps running), on a committed completion (the duplicate 409 surfaces), in an error body, and a gzip bomb past a 16 KiB limit. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 10 +- src/Exception/ResponseTooLarge.php | 15 +++ src/Transport/Psr18Transport.php | 100 +++++++++++---- src/Transport/ServerCapabilitiesDigest.php | 9 +- .../UnknownServerCapabilitiesDigest.php | 16 +++ tests/ServerCapabilitiesDigestTest.php | 115 +++++++++++++++++- 6 files changed, 236 insertions(+), 29 deletions(-) create mode 100644 src/Exception/ResponseTooLarge.php create mode 100644 src/Transport/UnknownServerCapabilitiesDigest.php diff --git a/CHANGELOG.md b/CHANGELOG.md index b9831e66..2a8abd3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,7 +73,15 @@ project follows [Semantic Versioning](https://semver.org/). Server that supports it replaces repeated `server_capabilities` objects (about 17 KB on native, sent on every worker answer and again inside carried polls) with their digest, and the transport restores the held object - before any caller sees the answer. Older Servers ignore the header. + before any caller sees the answer. Older Servers ignore the header. An + answer naming a digest the transport does not hold is recovered: the held + pair is forgotten and that request is sent once more without the header, + which returns the full object. A Server answers a request it had already + committed as a duplicate (409). A digest in an error body never replaces + the HTTP error. +- A gzip response is inflated by the transport within a decoded-size limit + (`Psr18Transport` `maxDecodedResponseBytes`, default 128 MiB). A larger body + fails with `DurableWorkflow\Exception\ResponseTooLarge`. - `result()` of a completed run reads the output from the terminal status read instead of downloading the run's whole history again. Failed, cancelled, terminated and timed-out runs, and Servers that omit diff --git a/src/Exception/ResponseTooLarge.php b/src/Exception/ResponseTooLarge.php new file mode 100644 index 00000000..e4130292 --- /dev/null +++ b/src/Exception/ResponseTooLarge.php @@ -0,0 +1,15 @@ + 'response_too_large', 'limit_bytes' => $limitBytes]); + } +} diff --git a/src/Transport/Psr18Transport.php b/src/Transport/Psr18Transport.php index c9a9a613..4a8f08e4 100644 --- a/src/Transport/Psr18Transport.php +++ b/src/Transport/Psr18Transport.php @@ -5,6 +5,7 @@ namespace DurableWorkflow\Transport; use DurableWorkflow\Exception\ExternalPayloadException; +use DurableWorkflow\Exception\ResponseTooLarge; use DurableWorkflow\Exception\TransportException; use GuzzleHttp\Client as GuzzleClient; use GuzzleHttp\Psr7\FnStream; @@ -28,6 +29,8 @@ final class Psr18Transport implements BoundedPayloadTransport, BoundedPayloadUpl public const DEFAULT_REQUEST_TIMEOUT_SECONDS = 60.0; /** Worker lifecycle requests (register, heartbeat, complete, fail, deregister); long polls keep their own budget. */ public const DEFAULT_WORKER_REQUEST_TIMEOUT_SECONDS = 60.0; + /** The most bytes a compressed JSON response may decode to. */ + public const DEFAULT_MAX_DECODED_RESPONSE_BYTES = 128 * 1024 * 1024; private readonly ClientInterface $client; private readonly RequestFactoryInterface $requestFactory; @@ -56,7 +59,11 @@ public function __construct( private readonly int $payloadTransferTimeoutSeconds = 30, ?float $requestTimeoutSeconds = null, ?float $workerRequestTimeoutSeconds = null, + private readonly int $maxDecodedResponseBytes = self::DEFAULT_MAX_DECODED_RESPONSE_BYTES, ) { + if ($maxDecodedResponseBytes < 1) { + throw new \InvalidArgumentException('maxDecodedResponseBytes must be positive.'); + } if ($payloadTransferTimeoutSeconds < 1 || $payloadTransferTimeoutSeconds > 900) { throw new \InvalidArgumentException('Supervised payload transfers require a timeout from 1 through 900 seconds.'); } @@ -89,12 +96,43 @@ public function send(string $method, string $uri, array $headers, ?array $body = return $this->sendJson($method, $uri, $headers, $body); } - /** The owned Guzzle client over cURL decodes gzip bodies it asked for. */ + /** The owned Guzzle client over cURL can ask for gzip; this transport inflates it. */ private function decodesCompressedResponses(): bool { return $this->client instanceof GuzzleClient && extension_loaded('curl') && extension_loaded('zlib'); } + /** Inflate a gzip body, refusing to produce more than the decoded-size limit. */ + private function inflate(string $body): string + { + $context = @inflate_init(ZLIB_ENCODING_GZIP); + if ($context === false) { + throw new TransportException('The server returned an invalid gzip response body.'); + } + $decoded = ''; + // Small input chunks bound how far one step can overshoot the limit. + foreach (str_split($body, 8192) as $chunk) { + $part = @inflate_add($context, $chunk, ZLIB_SYNC_FLUSH); + if ($part === false) { + throw new TransportException('The server returned an invalid gzip response body.'); + } + $decoded .= $part; + if (strlen($decoded) > $this->maxDecodedResponseBytes) { + throw new ResponseTooLarge($this->maxDecodedResponseBytes); + } + } + $tail = @inflate_add($context, '', ZLIB_FINISH); + if ($tail === false) { + throw new TransportException('The server returned an invalid gzip response body.'); + } + $decoded .= $tail; + if (strlen($decoded) > $this->maxDecodedResponseBytes) { + throw new ResponseTooLarge($this->maxDecodedResponseBytes); + } + + return $decoded; + } + public function supportsBoundedRequests(): bool { return $this->client instanceof GuzzleClient && extension_loaded('curl'); @@ -190,20 +228,33 @@ private function sendJson(string $method, string $uri, array $headers, ?array $b if ($poll) { $request = $request->withHeader('Accept', PollTransport::MEDIA_TYPE.', application/json;q=0.9'); } - // The answer is resolved against the pair this request was sent with. - $captured = $control ? null : $this->capabilitiesDigest->capture(); - if ($captured !== null) { - $request = $request->withHeader(ServerCapabilitiesDigest::HEADER, $this->capabilitiesDigest->header($captured)); - } if ($this->decodesCompressedResponses() && !$request->hasHeader('Accept-Encoding')) { - // cURL decodes the body; Guzzle then moves Content-Encoding aside. + // The transport inflates the body itself, within its size limit. $request = $request->withHeader('Accept-Encoding', 'gzip'); } - $response = $this->sendRequest($request, timeoutSeconds: $timeoutSeconds ?? $controlTimeout ?? 30, - bounded: $timeoutSeconds !== null || $controlTimeout !== null, controlTimeout: $controlTimeout, control: $control); - $this->responseStatus = $response->getStatusCode(); - - return $this->decodeJsonResponse($response, $poll, $captured); + $send = function (RequestInterface $request, ?array $captured) use ($timeoutSeconds, $controlTimeout, $control, $poll): ?array { + $response = $this->sendRequest($request, timeoutSeconds: $timeoutSeconds ?? $controlTimeout ?? 30, + bounded: $timeoutSeconds !== null || $controlTimeout !== null, controlTimeout: $controlTimeout, control: $control); + $this->responseStatus = $response->getStatusCode(); + + return $this->decodeJsonResponse($response, $poll, $captured); + }; + if ($control) { + return $send($request, null); + } + // The answer is resolved against the pair this request was sent with. + $captured = $this->capabilitiesDigest->capture(); + try { + return $send($request->withHeader(ServerCapabilitiesDigest::HEADER, $this->capabilitiesDigest->header($captured)), $captured); + } catch (UnknownServerCapabilitiesDigest) { + // The answer named capabilities this worker does not hold. Forget + // the held pair and ask once more without the header, which gets + // the full object. Worker requests are fenced, so a request the + // Server already committed is answered as a duplicate (409). + $this->capabilitiesDigest->forget(); + + return $send($request->withoutHeader(ServerCapabilitiesDigest::HEADER), [null, null]); + } } catch (TransportException $exception) { throw $exception; } catch (JsonException $exception) { @@ -233,15 +284,11 @@ private function pollTimeoutSeconds(?array $body): int|float private function decodeJsonResponse(ResponseInterface $response, bool $poll = false, ?array $captured = null): ?array { $rawBody = (string) $response->getBody(); - if (str_starts_with($rawBody, "\x1f\x8b") && function_exists('gzdecode') + if (str_starts_with($rawBody, "\x1f\x8b") && function_exists('inflate_init') && strtolower(trim($response->getHeaderLine('Content-Encoding'))) === 'gzip') { - // A client that left the gzip body encoded. Decoding clients - // either drop the header or already return the JSON bytes. - $inflated = @gzdecode($rawBody); - if (!is_string($inflated)) { - throw new TransportException('The server returned an invalid gzip response body.'); - } - $rawBody = $inflated; + // A body left gzip-encoded: the owned client asks for gzip without + // letting cURL inflate it, so the size limit always applies. + $rawBody = $this->inflate($rawBody); } $status = $response->getStatusCode(); $mediaType = strtolower(trim(explode(';', $response->getHeaderLine('Content-Type'), 2)[0])); @@ -261,7 +308,14 @@ private function decodeJsonResponse(ResponseInterface $response, bool $poll = fa // its status without interpreting it as a successful protocol reply. $decoded = json_decode($rawBody, true); if ($captured !== null && is_array($decoded)) { - $this->capabilitiesDigest->restore($decoded, $captured); + try { + $this->capabilitiesDigest->restore($decoded, $captured); + } catch (UnknownServerCapabilitiesDigest) { + // The HTTP error is the answer; an unusable digest in it + // never replaces it. Ask for full capabilities next time. + $this->capabilitiesDigest->forget(); + unset($decoded['server_capabilities']); + } } throw TransportException::fromResponse($status, is_array($decoded) ? $decoded : null, $rawBody); } @@ -554,6 +608,10 @@ private function sendRequest(RequestInterface $request, bool $stream = false, in $onStats = $http->getConfig('on_stats'); $options = $this->guzzleRequestOptions($stream, $timeoutSeconds, $bounded, $sink, $progress); + if (!$stream && $sink === null && $request->getHeaderLine('Accept-Encoding') === 'gzip') { + // Keep the body encoded: decodeJsonResponse() inflates it within its size limit. + $options['decode_content'] = false; + } $options['on_stats'] = static function (TransferStats $stats) use (&$connectionFailure, &$timedOut, $onStats): void { // Guzzle 8 no longer attaches cURL errno to exceptions. // Accept only DNS/connect/timeout/closed-connection errors, diff --git a/src/Transport/ServerCapabilitiesDigest.php b/src/Transport/ServerCapabilitiesDigest.php index ed0a897e..7064a8a9 100644 --- a/src/Transport/ServerCapabilitiesDigest.php +++ b/src/Transport/ServerCapabilitiesDigest.php @@ -40,6 +40,13 @@ public function capture(): array return [$this->digest, $this->capabilities]; } + /** Forget the held pair; the next request asks for full capabilities again. */ + public function forget(): void + { + $this->digest = null; + $this->capabilities = null; + } + /** @param array{0: ?string, 1: array|null}|null $captured */ public function header(?array $captured = null): string { @@ -75,7 +82,7 @@ private function restoreLevel(array &$answer, array $captured, array &$learned, $value = match (true) { $value === $captured[0] && $captured[1] !== null => $captured[1], $value === $learned[0] && $learned[1] !== null => $learned[1], - default => throw new TransportException('The server omitted capabilities this worker does not hold.'), + default => throw new UnknownServerCapabilitiesDigest(), }; } elseif (is_array($value) && !array_is_list($value) && is_string($value['digest'] ?? null) && str_starts_with($value['digest'], 'sha256:')) { diff --git a/src/Transport/UnknownServerCapabilitiesDigest.php b/src/Transport/UnknownServerCapabilitiesDigest.php new file mode 100644 index 00000000..e0672df6 --- /dev/null +++ b/src/Transport/UnknownServerCapabilitiesDigest.php @@ -0,0 +1,16 @@ +requests[0]->hasHeader('Accept-Encoding')); } - public function testAnUnknownDigestIsAProtocolFailure(): void + public function testAnUnknownDigestIsRecoveredByAskingAgainInFull(): void { - $http = $this->http([[200, [], json_encode(['server_capabilities' => self::DIGEST], JSON_THROW_ON_ERROR)]]); - $this->expectException(TransportException::class); - $this->expectExceptionMessage('The server omitted capabilities this worker does not hold.'); - (new Psr18Transport(client: $http))->send('POST', 'https://runtime.test/api/worker/heartbeat', - ['X-Durable-Workflow-Protocol-Version' => '1.20'], []); + $http = $this->http([ + [200, [], json_encode(['task' => null, 'server_capabilities' => 'sha256:'.str_repeat('f', 64)], JSON_THROW_ON_ERROR)], + [200, [], json_encode(['task' => null, 'server_capabilities' => self::CAPABILITIES], JSON_THROW_ON_ERROR)], + [200, [], json_encode(['task' => null, 'server_capabilities' => self::tagged()], JSON_THROW_ON_ERROR)], + ]); + $transport = new Psr18Transport(client: $http); + $worker = ['X-Durable-Workflow-Protocol-Version' => '1.20']; + + $answer = $transport->send('POST', 'https://runtime.test/api/worker/workflow-tasks/poll', $worker, ['timeout_seconds' => 0]); + self::assertSame(self::CAPABILITIES, $answer['server_capabilities']); + self::assertCount(2, $http->requests); + self::assertSame('none', $http->requests[0]->getHeaderLine(ServerCapabilitiesDigest::HEADER)); + self::assertFalse($http->requests[1]->hasHeader(ServerCapabilitiesDigest::HEADER), 'Retried without the header.'); + self::assertSame((string) $http->requests[0]->getBody(), (string) $http->requests[1]->getBody()); + // The next request asks for tagged capabilities again and learns them. + $transport->send('POST', 'https://runtime.test/api/worker/heartbeat', $worker, []); + self::assertSame('none', $http->requests[2]->getHeaderLine(ServerCapabilitiesDigest::HEADER)); + } + + public function testACommittedCompletionRetriedAfterAnUnknownDigestSurfacesTheServersDuplicateAnswer(): void + { + $duplicate = ['reason' => 'task_not_leased', 'message' => 'The task is not leased.', 'server_capabilities' => self::CAPABILITIES]; + $http = $this->http([ + [200, [], json_encode(['outcome' => 'completed', 'server_capabilities' => 'sha256:'.str_repeat('e', 64)], JSON_THROW_ON_ERROR)], + [409, [], json_encode($duplicate, JSON_THROW_ON_ERROR)], + ]); + try { + (new Psr18Transport(client: $http))->send('POST', 'https://runtime.test/api/worker/workflow-tasks/t/complete', + ['X-Durable-Workflow-Protocol-Version' => '1.20'], ['commands' => []]); + self::fail('The duplicate answer is the Server\'s, and the Worker handles it as a 409.'); + } catch (TransportException $exception) { + self::assertSame(409, $exception->status); + self::assertSame('task_not_leased', $exception->response['reason']); + } + self::assertCount(2, $http->requests); + } + + public function testAGarbledDigestOnAPollDoesNotStopAManagedWorker(): void + { + $polls = 0; + $http = new class ($polls) implements ClientInterface { + /** @var list */ + public array $requests = []; + + public function __construct(public int $polls) + { + } + + public function sendRequest(RequestInterface $request): ResponseInterface + { + $this->requests[] = $request; + $path = $request->getUri()->getPath(); + $garbled = 'sha256:'.str_repeat('c', 64); + $body = match (true) { + str_ends_with($path, '/worker/register') => ['registered' => true], + str_ends_with($path, '/workflow-tasks/poll') => ['task' => null, 'poll_status' => 'empty', + 'server_capabilities' => ++$this->polls === 1 && $request->hasHeader(ServerCapabilitiesDigest::HEADER) + ? $garbled : ['query_tasks' => true]], + str_ends_with($path, '/query-tasks/poll') => ['task' => null, 'poll_status' => 'stopped', 'reason' => 'worker_stopped'], + default => ['task' => null, 'poll_status' => 'empty'], + }; + + return new Response(200, ['Content-Type' => 'application/json'], json_encode($body, JSON_THROW_ON_ERROR)); + } + }; + $worker = new \DurableWorkflow\Worker(new \DurableWorkflow\Client('https://runtime.test', + transport: new Psr18Transport(client: $http), workerToken: 'w'), 'orders', workerId: 'worker-1'); + + $worker->run(0); + + $paths = array_map(static fn (RequestInterface $request): string => $request->getUri()->getPath(), $http->requests); + self::assertSame(2, count(array_keys($paths, '/api/worker/workflow-tasks/poll')), 'The garbled poll was asked again in full.'); + self::assertContains('/api/worker/query-tasks/poll', $paths, 'The Worker kept running.'); + } + + public function testAnUnknownDigestInAnErrorBodyNeverReplacesTheHttpError(): void + { + $http = $this->http([[409, [], json_encode(['reason' => 'stale_attempt', 'server_capabilities' => 'sha256:'.str_repeat('d', 64)], JSON_THROW_ON_ERROR)]]); + try { + (new Psr18Transport(client: $http))->send('POST', 'https://runtime.test/api/worker/activity-tasks/a/complete', + ['X-Durable-Workflow-Protocol-Version' => '1.20'], []); + self::fail('The 409 must surface.'); + } catch (TransportException $exception) { + self::assertSame(409, $exception->status); + self::assertSame('stale_attempt', $exception->response['reason']); + self::assertArrayNotHasKey('server_capabilities', $exception->response); + } + self::assertCount(1, $http->requests, 'An error answer is not retried.'); + } + + public function testAGzipBodyThatDecodesPastTheLimitIsRefused(): void + { + $body = (string) gzencode(json_encode(['blob' => str_repeat('z', 64 * 1024)], JSON_THROW_ON_ERROR)); + self::assertLessThan(1024, strlen($body)); + $http = $this->http([[200, ['Content-Encoding' => 'gzip'], $body]]); + try { + (new Psr18Transport(client: $http, maxDecodedResponseBytes: 16 * 1024))->send('GET', 'https://runtime.test/api/workflows/x', + ['X-Durable-Workflow-Control-Plane-Version' => '2']); + self::fail('A bomb must be refused.'); + } catch (ResponseTooLarge $exception) { + self::assertSame(16 * 1024, $exception->limitBytes); + self::assertSame('response_too_large', $exception->response['reason']); + } + // Within the default limit it decodes normally. + $http = $this->http([[200, ['Content-Encoding' => 'gzip'], $body]]); + self::assertSame(64 * 1024, strlen((new Psr18Transport(client: $http))->send('GET', 'https://runtime.test/api/workflows/x', + ['X-Durable-Workflow-Control-Plane-Version' => '2'])['blob'])); } public function testAGzipBodyLeftEncodedByTheClientIsDecoded(): void From 3a5ac2e90575c0d946940255fb10d95c12562218 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 20 Nov 2026 08:36:15 +0000 Subject: [PATCH 4/5] Accept our own committed completion on any answer; safer decode default Digest re-review: - BLOCKER: a garbled 200 to an activity completion triggered the no-digest resend, the Server answered 409 stale_attempt for the completion it had already committed, and Worker::run exited. The committed-completion rule (409 stale_attempt, recorded false, our owner and attempt, every status completed) now applies to any answer: the first, after a transient failure, or after a digest resend. This block is identical to the one in #449, so whichever merges second merges cleanly. Workflow-task duplicates need nothing: Server 3.17.0 answers them 200 recorded:false already_completed, which the Worker already accepts. - End-to-end test: garbled completion answer, then the resend's 409; the Worker keeps running and the activity runs once. Verified live against durableworkflow/server:3.17.0 as well. - The default decode limit is min(32 MiB, memory_limit / 4), safe under PHP's default 128 MB, and ResponseTooLarge reaches callers unwrapped. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 10 +++- src/Client.php | 7 ++- src/Transport/Psr18Transport.php | 28 +++++++-- tests/ServerCapabilitiesDigestTest.php | 83 ++++++++++++++++++++++++++ 4 files changed, 120 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a8abd3b..fec47fa7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,8 +80,14 @@ project follows [Semantic Versioning](https://semver.org/). committed as a duplicate (409). A digest in an error body never replaces the HTTP error. - A gzip response is inflated by the transport within a decoded-size limit - (`Psr18Transport` `maxDecodedResponseBytes`, default 128 MiB). A larger body - fails with `DurableWorkflow\Exception\ResponseTooLarge`. + (`Psr18Transport` `maxDecodedResponseBytes`; by default 32 MiB, or a quarter + of `memory_limit` when that is lower). A larger body fails with + `DurableWorkflow\Exception\ResponseTooLarge`, which reaches callers as + itself. +- A `409 stale_attempt` that proves this Worker's own activity completion was + already committed (`recorded: false`, its owner and attempt, every status + `completed`) is accepted as committed on any answer: the first, after a + transient failure, or after a digest resend. - `result()` of a completed run reads the output from the terminal status read instead of downloading the run's whole history again. Failed, cancelled, terminated and timed-out runs, and Servers that omit diff --git a/src/Client.php b/src/Client.php index ca8b9012..fc49804e 100644 --- a/src/Client.php +++ b/src/Client.php @@ -17,6 +17,7 @@ use DurableWorkflow\Exception\QueryFailed; use DurableWorkflow\Exception\ScheduleAlreadyExists; use DurableWorkflow\Exception\ServerCapabilityUnavailable; +use DurableWorkflow\Exception\ResponseTooLarge; use DurableWorkflow\Exception\ServerException; use DurableWorkflow\Exception\SignalFailed; use DurableWorkflow\Exception\SignalWithStartFailed; @@ -3495,7 +3496,8 @@ private function performRequest( return $this->resolveResponsePayloads($response, $path, $worker, $body, $headers, $budget, $callerBudget, $poll, $payloadPaths); } catch (TransportException $exception) { - if ($exception instanceof FailureReportRefused) { + if ($exception instanceof FailureReportRefused || $exception instanceof ResponseTooLarge) { + // Typed transport outcomes reach the caller as themselves. throw $exception; } $details = $exception->response; @@ -3573,7 +3575,8 @@ private function carriedPollResponse(array $answer, array $request, string $path $this->requestHeaders(true), $this->boundedWorkerRequests ? new RequestBudget(5) : null, null, true, null); } catch (TransportException $exception) { - if ($exception instanceof FailureReportRefused) { + if ($exception instanceof FailureReportRefused || $exception instanceof ResponseTooLarge) { + // Typed transport outcomes reach the caller as themselves. throw $exception; } $details = $exception->response; diff --git a/src/Transport/Psr18Transport.php b/src/Transport/Psr18Transport.php index 4a8f08e4..27860650 100644 --- a/src/Transport/Psr18Transport.php +++ b/src/Transport/Psr18Transport.php @@ -29,8 +29,13 @@ final class Psr18Transport implements BoundedPayloadTransport, BoundedPayloadUpl public const DEFAULT_REQUEST_TIMEOUT_SECONDS = 60.0; /** Worker lifecycle requests (register, heartbeat, complete, fail, deregister); long polls keep their own budget. */ public const DEFAULT_WORKER_REQUEST_TIMEOUT_SECONDS = 60.0; - /** The most bytes a compressed JSON response may decode to. */ - public const DEFAULT_MAX_DECODED_RESPONSE_BYTES = 128 * 1024 * 1024; + /** + * The most bytes a compressed JSON response may decode to by default: + * 32 MiB, or a quarter of PHP's memory_limit when that is lower, so the + * decoded body and its JSON decoding fit under the default 128 MB limit. + */ + public const DEFAULT_MAX_DECODED_RESPONSE_BYTES = 32 * 1024 * 1024; + private readonly int $maxDecodedResponseBytes; private readonly ClientInterface $client; private readonly RequestFactoryInterface $requestFactory; @@ -59,11 +64,12 @@ public function __construct( private readonly int $payloadTransferTimeoutSeconds = 30, ?float $requestTimeoutSeconds = null, ?float $workerRequestTimeoutSeconds = null, - private readonly int $maxDecodedResponseBytes = self::DEFAULT_MAX_DECODED_RESPONSE_BYTES, + ?int $maxDecodedResponseBytes = null, ) { - if ($maxDecodedResponseBytes < 1) { + if ($maxDecodedResponseBytes !== null && $maxDecodedResponseBytes < 1) { throw new \InvalidArgumentException('maxDecodedResponseBytes must be positive.'); } + $this->maxDecodedResponseBytes = $maxDecodedResponseBytes ?? self::defaultMaxDecodedResponseBytes(); if ($payloadTransferTimeoutSeconds < 1 || $payloadTransferTimeoutSeconds > 900) { throw new \InvalidArgumentException('Supervised payload transfers require a timeout from 1 through 900 seconds.'); } @@ -102,6 +108,20 @@ private function decodesCompressedResponses(): bool return $this->client instanceof GuzzleClient && extension_loaded('curl') && extension_loaded('zlib'); } + /** min(32 MiB, memory_limit / 4); an unlimited memory_limit keeps 32 MiB. */ + public static function defaultMaxDecodedResponseBytes(?string $memoryLimit = null): int + { + $limit = trim($memoryLimit ?? (string) ini_get('memory_limit')); + if ($limit === '' || $limit === '-1' || preg_match('/\A(\d+)\s*([kmg]?)\z/i', $limit, $match) !== 1) { + return self::DEFAULT_MAX_DECODED_RESPONSE_BYTES; + } + $bytes = (int) $match[1] * match (strtolower($match[2])) { + 'g' => 1024 ** 3, 'm' => 1024 ** 2, 'k' => 1024, default => 1, + }; + + return max(1024 * 1024, min(self::DEFAULT_MAX_DECODED_RESPONSE_BYTES, intdiv($bytes, 4))); + } + /** Inflate a gzip body, refusing to produce more than the decoded-size limit. */ private function inflate(string $body): string { diff --git a/tests/ServerCapabilitiesDigestTest.php b/tests/ServerCapabilitiesDigestTest.php index 30e59f42..1eeb841f 100644 --- a/tests/ServerCapabilitiesDigestTest.php +++ b/tests/ServerCapabilitiesDigestTest.php @@ -234,6 +234,71 @@ public function sendRequest(RequestInterface $request): ResponseInterface self::assertContains('/api/worker/query-tasks/poll', $paths, 'The Worker kept running.'); } + /** + * End to end: a garbled 200 to an activity completion forces the no-digest + * resend; the Server answers 409 stale_attempt for the completion it already + * committed; the Worker treats that as its own committed completion, keeps + * running, and the activity runs once. + */ + public function testAGarbledCompletionAnswerResentAsADuplicateKeepsTheWorkerRunning(): void + { + $http = new class implements ClientInterface { + /** @var list */ + public array $requests = []; + public bool $leased = false; + + public function sendRequest(RequestInterface $request): ResponseInterface + { + $this->requests[] = $request; + $path = $request->getUri()->getPath(); + $status = 200; + if (str_ends_with($path, '/worker/register')) { + $body = ['registered' => true]; + } elseif (str_ends_with($path, '/activity-tasks/poll') && !$this->leased) { + $this->leased = true; + $body = ['poll_status' => 'leased', 'task' => ['task_id' => 'activity-1', 'activity_attempt_id' => 'attempt-1', + 'lease_owner' => 'worker-1', 'activity_type' => 'orders.charge', 'payload_codec' => 'avro']]; + } elseif (str_ends_with($path, '/activity-tasks/activity-1/complete')) { + if ($request->hasHeader(ServerCapabilitiesDigest::HEADER)) { + // Committed, but the answer names a digest nobody holds. + $body = ['task_id' => 'activity-1', 'outcome' => 'completed', 'recorded' => true, + 'server_capabilities' => 'sha256:'.str_repeat('b', 64)]; + } else { + $status = 409; + $body = ['task_id' => 'activity-1', 'activity_attempt_id' => 'attempt-1', 'outcome' => 'completed', + 'recorded' => false, 'reason' => 'stale_attempt', 'next_task_id' => null, 'lease_owner' => 'worker-1', + 'activity_status' => 'completed', 'attempt_status' => 'completed', 'task_status' => 'completed']; + } + } elseif (str_ends_with($path, '/query-tasks/poll') && $this->leased) { + $body = ['task' => null, 'poll_status' => 'stopped', 'reason' => 'worker_stopped']; + } else { + $body = ['task' => null, 'poll_status' => 'empty']; + } + + return new Response($status, ['Content-Type' => 'application/json'], json_encode($body, JSON_THROW_ON_ERROR)); + } + }; + $runs = 0; + $events = []; + $worker = new \DurableWorkflow\Worker(new \DurableWorkflow\Client('https://runtime.test', + transport: new Psr18Transport(client: $http), workerToken: 'w'), 'orders', workerId: 'worker-1', + diagnosticListener: static function (string $event) use (&$events): void { $events[] = $event; }); + $worker->registerActivity('orders.charge', static function (\DurableWorkflow\Worker\ActivityContext $context) use (&$runs): string { + ++$runs; + + return 'charged'; + }); + + $worker->run(0); + + $paths = array_map(static fn (RequestInterface $request): string => $request->getUri()->getPath(), $http->requests); + self::assertSame(1, $runs, 'The activity ran once.'); + self::assertSame(2, count(array_keys($paths, '/api/worker/activity-tasks/activity-1/complete')), 'Sent, then resent without the digest.'); + self::assertNotContains('/api/worker/activity-tasks/activity-1/fail', $paths); + self::assertContains('worker.completion_already_committed', $events); + self::assertContains('/api/worker/query-tasks/poll', $paths, 'The Worker kept running.'); + } + public function testAnUnknownDigestInAnErrorBodyNeverReplacesTheHttpError(): void { $http = $this->http([[409, [], json_encode(['reason' => 'stale_attempt', 'server_capabilities' => 'sha256:'.str_repeat('d', 64)], JSON_THROW_ON_ERROR)]]); @@ -268,6 +333,24 @@ public function testAGzipBodyThatDecodesPastTheLimitIsRefused(): void ['X-Durable-Workflow-Control-Plane-Version' => '2'])['blob'])); } + public function testTheDefaultDecodeLimitFitsPhpsMemoryLimit(): void + { + self::assertSame(32 * 1024 * 1024, Psr18Transport::defaultMaxDecodedResponseBytes('128M')); + self::assertSame(16 * 1024 * 1024, Psr18Transport::defaultMaxDecodedResponseBytes('64M')); + self::assertSame(32 * 1024 * 1024, Psr18Transport::defaultMaxDecodedResponseBytes('-1')); + self::assertSame(32 * 1024 * 1024, Psr18Transport::defaultMaxDecodedResponseBytes('2G')); + self::assertSame(1024 * 1024, Psr18Transport::defaultMaxDecodedResponseBytes('2M')); + } + + public function testAnOversizedResponseReachesAClientCallerAsResponseTooLarge(): void + { + $body = (string) gzencode(json_encode(['blob' => str_repeat('z', 64 * 1024)], JSON_THROW_ON_ERROR)); + $client = new \DurableWorkflow\Client('https://runtime.test', transport: new Psr18Transport( + client: $this->http([[200, ['Content-Encoding' => 'gzip'], $body]]), maxDecodedResponseBytes: 16 * 1024)); + $this->expectException(ResponseTooLarge::class); + $client->describeWorkflow('x'); + } + public function testAGzipBodyLeftEncodedByTheClientIsDecoded(): void { $body = json_encode(['ok' => true, 'blob' => str_repeat('x', 4096)], JSON_THROW_ON_ERROR); From 9e14c3fd4439d0514ab9446fa63152aa8de48dd0 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 20 Nov 2026 09:39:13 +0000 Subject: [PATCH 5/5] Never repeat a request that changed state to recover capabilities Digest re-review blocker: a workflow-task completion's answer can carry the next leased activity. When its digest was garbled the transport resent the completion; the Server answered the repeat as a duplicate without that task, and the leased activity was lost (stuck Started until its lease expired). Under a garbling proxy 9 of 15 workflows stalled. On an unknown digest in a success answer the transport now keeps that answer as received, reads the capabilities from the read-only GET /api/cluster/info (worker_protocol.server_capabilities, the same object) in their place, and forgets the held pair so the next request learns a tagged copy. Only task polls, which are safe to repeat, are still resent without the header. Errors are unchanged. The reviewer's scenario is a test over the real Psr18Transport: a garbled completion answer carrying the next activity. The completion is sent once, cluster info is read, and the activity runs and completes exactly once. It fails on the previous head. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 13 +- src/Transport/Psr18Transport.php | 57 +++++++-- src/Transport/ServerCapabilitiesDigest.php | 2 +- .../UnknownServerCapabilitiesDigest.php | 11 +- tests/ServerCapabilitiesDigestTest.php | 113 +++++++++--------- 5 files changed, 125 insertions(+), 71 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fec47fa7..90483e89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,11 +74,12 @@ project follows [Semantic Versioning](https://semver.org/). (about 17 KB on native, sent on every worker answer and again inside carried polls) with their digest, and the transport restores the held object before any caller sees the answer. Older Servers ignore the header. An - answer naming a digest the transport does not hold is recovered: the held - pair is forgotten and that request is sent once more without the header, - which returns the full object. A Server answers a request it had already - committed as a duplicate (409). A digest in an error body never replaces - the HTTP error. + answer naming a digest the transport does not hold is recovered without + repeating any request that may have changed state: that answer is kept as + received (a completion's answer can carry the next leased task), the + capabilities are read from `GET /api/cluster/info`, and the held pair is + forgotten. Only a task poll is sent again without the header. A digest in + an error body never replaces the HTTP error. - A gzip response is inflated by the transport within a decoded-size limit (`Psr18Transport` `maxDecodedResponseBytes`; by default 32 MiB, or a quarter of `memory_limit` when that is lower). A larger body fails with @@ -87,7 +88,7 @@ project follows [Semantic Versioning](https://semver.org/). - A `409 stale_attempt` that proves this Worker's own activity completion was already committed (`recorded: false`, its owner and attempt, every status `completed`) is accepted as committed on any answer: the first, after a - transient failure, or after a digest resend. + transient failure, or a resend by the HTTP client. - `result()` of a completed run reads the output from the terminal status read instead of downloading the run's whole history again. Failed, cancelled, terminated and timed-out runs, and Servers that omit diff --git a/src/Transport/Psr18Transport.php b/src/Transport/Psr18Transport.php index 27860650..d6bd1d2a 100644 --- a/src/Transport/Psr18Transport.php +++ b/src/Transport/Psr18Transport.php @@ -122,6 +122,30 @@ public static function defaultMaxDecodedResponseBytes(?string $memoryLimit = nul return max(1024 * 1024, min(self::DEFAULT_MAX_DECODED_RESPONSE_BYTES, intdiv($bytes, 4))); } + /** + * The Server's current worker capabilities from cluster info, a read-only + * call; null when it cannot be read. + * + * @param array $headers + * @return array|null + */ + private function fetchCapabilities(string $uri, array $headers): ?array + { + $position = strpos($uri, '/api/'); + if ($position === false) { + return null; + } + try { + $info = $this->sendJson('GET', substr($uri, 0, $position).'/api/cluster/info', + array_diff_key($headers, ['Content-Type' => true]), null); + } catch (Throwable) { + return null; + } + $capabilities = is_array($info['worker_protocol'] ?? null) ? ($info['worker_protocol']['server_capabilities'] ?? null) : null; + + return is_array($capabilities) && !array_is_list($capabilities) ? $capabilities : null; + } + /** Inflate a gzip body, refusing to produce more than the decoded-size limit. */ private function inflate(string $body): string { @@ -266,14 +290,26 @@ private function sendJson(string $method, string $uri, array $headers, ?array $b $captured = $this->capabilitiesDigest->capture(); try { return $send($request->withHeader(ServerCapabilitiesDigest::HEADER, $this->capabilitiesDigest->header($captured)), $captured); - } catch (UnknownServerCapabilitiesDigest) { - // The answer named capabilities this worker does not hold. Forget - // the held pair and ask once more without the header, which gets - // the full object. Worker requests are fenced, so a request the - // Server already committed is answered as a duplicate (409). + } catch (UnknownServerCapabilitiesDigest $unknown) { + // The answer named capabilities this worker does not hold. + // Forget the held pair; the next request asks for a tagged copy. $this->capabilitiesDigest->forget(); - - return $send($request->withoutHeader(ServerCapabilitiesDigest::HEADER), [null, null]); + if (PollTransport::taskKind($method, $uri) !== null) { + // A task poll is safe to repeat: ask again without the header. + return $send($request->withoutHeader(ServerCapabilitiesDigest::HEADER), [null, null]); + } + if ($unknown->answer === null) { + throw $unknown; + } + // Anything else may have changed state (a completion can carry + // the next leased task), so it is never repeated. Its answer is + // kept as received; only the capabilities are fetched again, + // with a read-only cluster-info call. + $answer = $unknown->answer; + $this->capabilitiesDigest->restore($answer, + [$unknown->digest, $this->fetchCapabilities($uri, $headers) ?? $captured[1] ?? []]); + + return $answer; } } catch (TransportException $exception) { throw $exception; @@ -345,7 +381,12 @@ private function decodeJsonResponse(ResponseInterface $response, bool $poll = fa throw new TransportException('The server returned a JSON value instead of an object or array.'); } if ($captured !== null && $decoded !== null) { - $this->capabilitiesDigest->restore($decoded, $captured); + $received = $decoded; + try { + $this->capabilitiesDigest->restore($decoded, $captured); + } catch (UnknownServerCapabilitiesDigest $unknown) { + throw $unknown->withAnswer($received); + } } return $decoded; diff --git a/src/Transport/ServerCapabilitiesDigest.php b/src/Transport/ServerCapabilitiesDigest.php index 7064a8a9..bb392781 100644 --- a/src/Transport/ServerCapabilitiesDigest.php +++ b/src/Transport/ServerCapabilitiesDigest.php @@ -82,7 +82,7 @@ private function restoreLevel(array &$answer, array $captured, array &$learned, $value = match (true) { $value === $captured[0] && $captured[1] !== null => $captured[1], $value === $learned[0] && $learned[1] !== null => $learned[1], - default => throw new UnknownServerCapabilitiesDigest(), + default => throw new UnknownServerCapabilitiesDigest($value), }; } elseif (is_array($value) && !array_is_list($value) && is_string($value['digest'] ?? null) && str_starts_with($value['digest'], 'sha256:')) { diff --git a/src/Transport/UnknownServerCapabilitiesDigest.php b/src/Transport/UnknownServerCapabilitiesDigest.php index e0672df6..3b2c08bc 100644 --- a/src/Transport/UnknownServerCapabilitiesDigest.php +++ b/src/Transport/UnknownServerCapabilitiesDigest.php @@ -6,11 +6,18 @@ use DurableWorkflow\Exception\TransportException; -/** @internal An answer named capabilities this transport does not hold; recovered by asking again in full. */ +/** @internal An answer named capabilities this transport does not hold; recovered without repeating the request. */ final class UnknownServerCapabilitiesDigest extends TransportException { - public function __construct() + /** @param array|null $answer The decoded answer as received, when it was a success. */ + public function __construct(public readonly string $digest, public readonly ?array $answer = null) { parent::__construct('The server omitted capabilities this worker does not hold.'); } + + /** @param array $answer */ + public function withAnswer(array $answer): self + { + return new self($this->digest, $answer); + } } diff --git a/tests/ServerCapabilitiesDigestTest.php b/tests/ServerCapabilitiesDigestTest.php index 1eeb841f..60e343aa 100644 --- a/tests/ServerCapabilitiesDigestTest.php +++ b/tests/ServerCapabilitiesDigestTest.php @@ -178,22 +178,23 @@ public function testAnUnknownDigestIsRecoveredByAskingAgainInFull(): void self::assertSame('none', $http->requests[2]->getHeaderLine(ServerCapabilitiesDigest::HEADER)); } - public function testACommittedCompletionRetriedAfterAnUnknownDigestSurfacesTheServersDuplicateAnswer(): void + public function testAGarbledDigestOnAMutatingAnswerKeepsTheBodyAndReadsCapabilitiesInstead(): void { - $duplicate = ['reason' => 'task_not_leased', 'message' => 'The task is not leased.', 'server_capabilities' => self::CAPABILITIES]; $http = $this->http([ - [200, [], json_encode(['outcome' => 'completed', 'server_capabilities' => 'sha256:'.str_repeat('e', 64)], JSON_THROW_ON_ERROR)], - [409, [], json_encode($duplicate, JSON_THROW_ON_ERROR)], + [200, [], json_encode(['outcome' => 'completed', 'recorded' => true, 'next' => 'kept', + 'server_capabilities' => 'sha256:'.str_repeat('e', 64)], JSON_THROW_ON_ERROR)], + [200, [], json_encode(['worker_protocol' => ['version' => '1.20', 'server_capabilities' => self::CAPABILITIES]], JSON_THROW_ON_ERROR)], ]); - try { - (new Psr18Transport(client: $http))->send('POST', 'https://runtime.test/api/worker/workflow-tasks/t/complete', - ['X-Durable-Workflow-Protocol-Version' => '1.20'], ['commands' => []]); - self::fail('The duplicate answer is the Server\'s, and the Worker handles it as a 409.'); - } catch (TransportException $exception) { - self::assertSame(409, $exception->status); - self::assertSame('task_not_leased', $exception->response['reason']); - } - self::assertCount(2, $http->requests); + $answer = (new Psr18Transport(client: $http))->send('POST', 'https://runtime.test/api/worker/workflow-tasks/t/complete', + ['X-Durable-Workflow-Protocol-Version' => '1.20', 'Authorization' => 'Bearer w'], ['commands' => []]); + + self::assertSame('kept', $answer['next'], 'The answer as received is processed.'); + self::assertSame(self::CAPABILITIES, $answer['server_capabilities']); + self::assertCount(2, $http->requests, 'The completion is never repeated.'); + self::assertSame('POST', $http->requests[0]->getMethod()); + self::assertSame('GET', $http->requests[1]->getMethod()); + self::assertSame('/api/cluster/info', $http->requests[1]->getUri()->getPath()); + self::assertSame('Bearer w', $http->requests[1]->getHeaderLine('Authorization')); } public function testAGarbledDigestOnAPollDoesNotStopAManagedWorker(): void @@ -235,68 +236,72 @@ public function sendRequest(RequestInterface $request): ResponseInterface } /** - * End to end: a garbled 200 to an activity completion forces the no-digest - * resend; the Server answers 409 stale_attempt for the completion it already - * committed; the Worker treats that as its own committed completion, keeps - * running, and the activity runs once. + * The reviewer's scenario: a workflow-task completion whose answer carries + * the next leased activity arrives with every capabilities copy garbled. + * The completion is never repeated (a repeat would lose the carried task); + * the answer is kept, capabilities are read from cluster info, the carried + * activity runs once and completes once, and the Worker keeps running. */ - public function testAGarbledCompletionAnswerResentAsADuplicateKeepsTheWorkerRunning(): void + public function testAGarbledCompletionAnswerCarryingTheNextActivityIsKeptNotResent(): void { - $http = new class implements ClientInterface { + $now = new \DateTimeImmutable('now', new \DateTimeZone('UTC')); + $caps = ['workflow_task_completion_activity_poll' => true]; + $codec = new \DurableWorkflow\Codec\AvroPayloadCodec(); + $garbled = 'sha256:'.str_repeat('9', 64); + $answer = static fn (array $task, mixed $capabilities): array => ['task' => $task + ['lease_owner' => 'worker-1', + 'payload_codec' => 'avro', 'lease_expires_at' => $now->modify('+30 seconds')->format('Y-m-d\TH:i:s.u\Z')], + 'poll_status' => 'leased', 'protocol_version' => '1.20', 'server_capabilities' => $capabilities, + 'server_time' => $now->format('Y-m-d\TH:i:s.u\Z')]; + $http = new class ($answer, $caps, $codec, $garbled) implements ClientInterface { /** @var list */ public array $requests = []; - public bool $leased = false; + public int $workflowPolls = 0; + + public function __construct(private \Closure $answer, private array $caps, private object $codec, private string $garbled) + { + } public function sendRequest(RequestInterface $request): ResponseInterface { $this->requests[] = $request; $path = $request->getUri()->getPath(); - $status = 200; - if (str_ends_with($path, '/worker/register')) { - $body = ['registered' => true]; - } elseif (str_ends_with($path, '/activity-tasks/poll') && !$this->leased) { - $this->leased = true; - $body = ['poll_status' => 'leased', 'task' => ['task_id' => 'activity-1', 'activity_attempt_id' => 'attempt-1', - 'lease_owner' => 'worker-1', 'activity_type' => 'orders.charge', 'payload_codec' => 'avro']]; - } elseif (str_ends_with($path, '/activity-tasks/activity-1/complete')) { - if ($request->hasHeader(ServerCapabilitiesDigest::HEADER)) { - // Committed, but the answer names a digest nobody holds. - $body = ['task_id' => 'activity-1', 'outcome' => 'completed', 'recorded' => true, - 'server_capabilities' => 'sha256:'.str_repeat('b', 64)]; - } else { - $status = 409; - $body = ['task_id' => 'activity-1', 'activity_attempt_id' => 'attempt-1', 'outcome' => 'completed', - 'recorded' => false, 'reason' => 'stale_attempt', 'next_task_id' => null, 'lease_owner' => 'worker-1', - 'activity_status' => 'completed', 'attempt_status' => 'completed', 'task_status' => 'completed']; - } - } elseif (str_ends_with($path, '/query-tasks/poll') && $this->leased) { - $body = ['task' => null, 'poll_status' => 'stopped', 'reason' => 'worker_stopped']; - } else { - $body = ['task' => null, 'poll_status' => 'empty']; - } + $answer = $this->answer; + $body = match (true) { + str_ends_with($path, '/worker/register') => ['registered' => true], + str_ends_with($path, '/cluster/info') => ['worker_protocol' => ['version' => '1.20', 'server_capabilities' => $this->caps]], + str_ends_with($path, '/workflow-tasks/poll') && ++$this->workflowPolls === 1 => $answer(['task_id' => 'workflow-task', + 'workflow_task_attempt' => 1, 'workflow_id' => 'workflow', 'run_id' => 'run', 'workflow_type' => 'carry', + 'history_events' => []], $this->caps), + str_ends_with($path, '/workflow-tasks/workflow-task/complete') => ['completed' => true, + 'activity_poll' => ['leased' => true, 'answer' => $answer(['task_id' => 'activity-1', + 'activity_attempt_id' => 'attempt-1', 'activity_type' => 'carried.activity', + 'arguments' => $this->codec->envelope([7])], $this->garbled)], + 'server_capabilities' => $this->garbled], + str_ends_with($path, '/activity-tasks/activity-1/complete') => ['recorded' => true, 'outcome' => 'completed'], + str_ends_with($path, '/query-tasks/poll') && $this->workflowPolls >= 2 => ['task' => null, 'poll_status' => 'stopped', 'reason' => 'worker_stopped'], + default => ['task' => null, 'poll_status' => 'empty'], + }; - return new Response($status, ['Content-Type' => 'application/json'], json_encode($body, JSON_THROW_ON_ERROR)); + return new Response(200, ['Content-Type' => 'application/json'], json_encode($body, JSON_THROW_ON_ERROR)); } }; $runs = 0; - $events = []; $worker = new \DurableWorkflow\Worker(new \DurableWorkflow\Client('https://runtime.test', - transport: new Psr18Transport(client: $http), workerToken: 'w'), 'orders', workerId: 'worker-1', - diagnosticListener: static function (string $event) use (&$events): void { $events[] = $event; }); - $worker->registerActivity('orders.charge', static function (\DurableWorkflow\Worker\ActivityContext $context) use (&$runs): string { + transport: new Psr18Transport(client: $http), workerToken: 'w'), 'orders', workerId: 'worker-1'); + $worker->registerWorkflow('carry', static fn (\DurableWorkflow\Worker\WorkflowContext $context): string => 'done'); + $worker->registerActivity('carried.activity', static function (\DurableWorkflow\Worker\ActivityContext $context, int $value) use (&$runs): int { ++$runs; - return 'charged'; + return $value; }); $worker->run(0); $paths = array_map(static fn (RequestInterface $request): string => $request->getUri()->getPath(), $http->requests); - self::assertSame(1, $runs, 'The activity ran once.'); - self::assertSame(2, count(array_keys($paths, '/api/worker/activity-tasks/activity-1/complete')), 'Sent, then resent without the digest.'); - self::assertNotContains('/api/worker/activity-tasks/activity-1/fail', $paths); - self::assertContains('worker.completion_already_committed', $events); - self::assertContains('/api/worker/query-tasks/poll', $paths, 'The Worker kept running.'); + self::assertSame(1, count(array_keys($paths, '/api/worker/workflow-tasks/workflow-task/complete')), 'Never repeated.'); + self::assertContains('/api/cluster/info', $paths, 'Capabilities were read, not re-requested by repeating.'); + self::assertSame(1, $runs, 'The carried activity ran exactly once.'); + self::assertSame(1, count(array_keys($paths, '/api/worker/activity-tasks/activity-1/complete')), 'And completed once.'); } public function testAnUnknownDigestInAnErrorBodyNeverReplacesTheHttpError(): void