From decf6e85e59227671e73a5153a339a964c45a758 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 20 Nov 2026 09:41:11 +0000 Subject: [PATCH 1/5] Never resend a non-poll request to recover server capabilities #304 resent any request once with "none" when its answer named an unknown capabilities digest. A completion's answer can carry the next leased task, and a resent completion only gets a duplicate receipt without it, so a worker could lose leased work. Only task polls (idempotent by poll request ID) are still resent. For any other request the answer is kept as received, the full capabilities are read with GET /api/cluster/info (worker_protocol.server_capabilities) and substituted for the unknown digest, and the held pair is forgotten so later requests send none. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 11 +++-- README.md | 9 ++-- src/lib.rs | 57 +++++++++++++++++++++--- src/server_capabilities_digest.rs | 24 +++++++++-- tests/server_capabilities_digest.rs | 67 ++++++++++++++++++----------- 5 files changed, 125 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 750b656..d52b5d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -90,10 +90,13 @@ used for later requests, so overlapping requests across a rolling deploy each restore their own capabilities. A digest string that matches neither the request's digest nor a tagged copy in the same answer makes the Client - forget its held digest and resend that same request once with `none`; a - resent report the Server already committed comes back as its usual - non-recorded receipt (for an activity completion, `409 stale_attempt`), which - the worker settles as before. Only an unknown digest on the resend is + forget its held digest. Only a task poll, idempotent by its poll request ID, + is resent once with `none`. Any other request (a completion, failure, + heartbeat, start or signal) is never resent: its answer is kept as received, + because a completion's answer can carry the next leased task, and the full + capabilities are read with `GET /api/cluster/info` + (`worker_protocol.server_capabilities`) and put in place of the unknown + digest. Only an unknown digest that cannot be recovered this way is `Error::Protocol` with reason `server_capabilities_digest_unknown`. Decoded response bodies are capped at 128 MiB by default (`ClientBuilder::max_response_bytes`); a larger body stops being read and diff --git a/README.md b/README.md index 1030b8f..6eff88f 100644 --- a/README.md +++ b/README.md @@ -308,10 +308,11 @@ digest and object when it sends each request and restores that request's answer before anything reads it, so readers always see the full object, even when requests overlap a rolling deploy. Clones share what they learned. An answer naming a digest that request did not send, and that the answer itself -does not carry, makes the Client forget its digest and resend that request once -asking for full capabilities. A resent report the Server already committed -gets its usual non-recorded receipt, which the worker settles. Only a second -unknown digest is `Error::Protocol` with reason +does not carry, makes the Client forget its digest. A task poll is resent once +asking for full capabilities. Any other request is never resent, since it may +already have committed: its answer is kept as received and the capabilities +are read from `GET /api/cluster/info`. An unknown digest that cannot be +recovered this way is `Error::Protocol` with reason `server_capabilities_digest_unknown`. Older Servers ignore both headers and answer in full. diff --git a/src/lib.rs b/src/lib.rs index 3f8dc31..d9fbeb8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5720,16 +5720,18 @@ impl Client { Ok(Some(restored)) => restored, Ok(None) => bytes, // An answer naming a digest this request did not send - // is recoverable once: forget the held pair and resend - // this same request asking for full capabilities. A - // resent report the Server already committed comes - // back as its usual non-recorded receipt. - Err(error) if !digest_resent => { + // is recoverable. Only a task poll, which is + // idempotent by its poll request ID, is resent asking + // for full capabilities. Any other request may have + // committed (a completion's answer can carry the next + // leased task), so its answer is kept as received and + // the capabilities are read from cluster info. + Err(error) if !digest_resent && is_task_poll(path) => { tracing::warn!( target: "durable_workflow::worker", path, error = %error, - "unknown server capabilities digest; resending without it" + "unknown server capabilities digest; resending the poll without it" ); self.capabilities_digest.forget(); request.headers_mut().insert( @@ -5740,6 +5742,40 @@ impl Client { digest_resent = true; continue; } + Err(error) if !is_task_poll(path) && path != "/cluster/info" => { + tracing::warn!( + target: "durable_workflow::worker", + path, + error = %error, + "unknown server capabilities digest; reading capabilities from cluster info" + ); + self.capabilities_digest.forget(); + let info: Value = Box::pin( + self.request_json_unresolved_with_timeout::( + reqwest::Method::GET, + "/cluster/info", + RequestProtocol::DefaultWorker, + None, + timeout, + ), + ) + .await?; + let Some(full) = info["worker_protocol"]["server_capabilities"] + .as_object() + .filter(|full| !full.contains_key("digest")) + else { + return Err(error); + }; + let restored = self.capabilities_digest.restore_with( + status.as_u16(), + &bytes, + captured, + Some(full), + )?; + // Later requests start again from `none`. + self.capabilities_digest.forget(); + restored.unwrap_or(bytes) + } Err(error) => return Err(error), } } @@ -6772,6 +6808,15 @@ fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration { .min(policy.max_backoff) } +/// A worker task poll: idempotent by its poll request ID, so the only +/// request that may be resent to recover elided capabilities. +fn is_task_poll(path: &str) -> bool { + matches!( + path, + "/worker/workflow-tasks/poll" | "/worker/activity-tasks/poll" | "/worker/query-tasks/poll" + ) +} + /// Default cap on one decoded response body. const DEFAULT_MAX_RESPONSE_BYTES: usize = 128 * 1024 * 1024; diff --git a/src/server_capabilities_digest.rs b/src/server_capabilities_digest.rs index 9852033..2fd5073 100644 --- a/src/server_capabilities_digest.rs +++ b/src/server_capabilities_digest.rs @@ -64,6 +64,19 @@ impl Digest { status: u16, bytes: &[u8], captured: &Captured, + ) -> Result>, Error> { + self.restore_with(status, bytes, captured, None) + } + + /// Like [`Digest::restore`], but a digest string that matches nothing is + /// replaced by `fallback`: full capabilities read separately (from + /// cluster info) for an answer that must not be requested again. + pub(crate) fn restore_with( + &self, + status: u16, + bytes: &[u8], + captured: &Captured, + fallback: Option<&Map>, ) -> Result>, Error> { if !contains(bytes, FIELD.as_bytes()) { return Ok(None); @@ -75,7 +88,7 @@ impl Digest { let mut learned = None; collect_tagged(&mut answer, 1, &mut learned); let mut changed = learned.is_some(); - if let Err(digest) = replace(&mut answer, 1, captured, &learned, &mut changed) { + if let Err(digest) = replace(&mut answer, 1, captured, &learned, fallback, &mut changed) { return Err(Error::Protocol(ProtocolFailure { status, reason: "server_capabilities_digest_unknown".to_owned(), @@ -141,6 +154,7 @@ fn replace( depth: usize, captured: &Captured, learned: &Option, + fallback: Option<&Map>, changed: &mut bool, ) -> Result<(), String> { for (key, value) in answer.iter_mut() { @@ -151,16 +165,18 @@ fn replace( if !digest.starts_with(PREFIX) { continue; } - let pair = [captured.as_ref(), learned.as_ref()] + let capabilities = [captured.as_ref(), learned.as_ref()] .into_iter() .flatten() .find(|pair| pair.digest == *digest) + .map(|pair| &*pair.capabilities) + .or(fallback) .ok_or_else(|| digest.clone())?; - *value = Value::Object((*pair.capabilities).clone()); + *value = Value::Object(capabilities.clone()); *changed = true; } else if depth < MAX_DEPTH { if let Value::Object(nested) = value { - replace(nested, depth + 1, captured, learned, changed)?; + replace(nested, depth + 1, captured, learned, fallback, changed)?; } } } diff --git a/tests/server_capabilities_digest.rs b/tests/server_capabilities_digest.rs index 7c2dfc3..662bdc3 100644 --- a/tests/server_capabilities_digest.rs +++ b/tests/server_capabilities_digest.rs @@ -206,21 +206,30 @@ async fn worker_answers_are_restored_from_the_digest_each_request_sent() { } #[tokio::test] -async fn a_completion_resent_after_a_garbled_answer_gets_its_committed_receipt() { +async fn a_garbled_completion_answer_is_kept_and_capabilities_come_from_cluster_info() { let d1 = "sha256:d1"; let (url, requests) = server(vec![ - json_reply(200, json!({"task": null, "server_capabilities": tagged(d1)})), + json_reply( + 200, + json!({"task": null, "server_capabilities": tagged(d1)}), + ), // The first completion discovers the inline payload policy. json_reply(200, json!({})), - // The Server committed the completion, but its answer is garbled. - json_reply(200, json!({"recorded": true, "server_capabilities": "sha256:unknown"})), - // Native Server 3.17's answer to the resent copy. + // The Server committed the completion; its answer carries a result + // that must not be lost and an unknown digest. + json_reply( + 200, + json!({"recorded": true, "next_task": {"task_id": "next-1"}, + "server_capabilities": "sha256:unknown"}), + ), + // Read-only capability recovery. + json_reply( + 200, + json!({"worker_protocol": {"server_capabilities": tagged("sha256:d2")}}), + ), json_reply( - 409, - json!({"task_id":"task-1","activity_attempt_id":"attempt-1","outcome":"completed", - "recorded":false,"reason":"stale_attempt","lease_owner":"worker", - "activity_status":"completed","attempt_status":"completed","task_status":"completed", - "server_capabilities": tagged(d1)}), + 200, + json!({"task": null, "server_capabilities": tagged("sha256:d2")}), ), ]) .await; @@ -229,30 +238,38 @@ async fn a_completion_resent_after_a_garbled_answer_gets_its_committed_receipt() .poll_workflow_task_response("worker", "queue", Duration::from_secs(1)) .await .unwrap(); - let Err(Error::ActivityTaskRejected(receipt)) = client + let answer = client .complete_activity_task("task-1", "attempt-1", "worker", json!("done"), "avro") .await - else { - panic!("the resent completion must return the Server's receipt"); - }; - // The worker settles this non-recorded receipt like any stale attempt. - assert_eq!(receipt.status, 409); - assert_eq!(receipt.reason, "stale_attempt"); - assert_eq!(receipt.body["server_capabilities"], capabilities()); + .expect("the completion's own answer is used"); + assert_eq!(answer["recorded"], true); + assert_eq!(answer["next_task"]["task_id"], "next-1"); + assert_eq!(answer["server_capabilities"], capabilities()); + client + .poll_workflow_task_response("worker", "queue", Duration::from_secs(1)) + .await + .unwrap(); let heads = requests.await.unwrap(); + let completes = heads + .iter() + .filter(|head| head.starts_with("post /api/worker/activity-tasks/task-1/complete")) + .count(); + assert_eq!(completes, 1, "a completion is never resent"); + assert!( + heads[3].starts_with("get /api/cluster/info"), + "{}", + heads[3] + ); let sent: Vec<_> = heads .iter() .map(|head| header(head, "x-durable-workflow-server-capabilities-digest")) .collect(); - assert!( - heads[1].starts_with("get /api/cluster/info"), - "{}", - heads[1] + // The held pair is forgotten: cluster info and the next poll send none. + assert_eq!( + sent, + [Some("none"), None, Some(d1), Some("none"), Some("none")] ); - assert!(heads[2].starts_with("post /api/worker/activity-tasks/task-1/complete")); - assert!(heads[3].starts_with("post /api/worker/activity-tasks/task-1/complete")); - assert_eq!(sent, [Some("none"), None, Some(d1), Some("none")]); } #[tokio::test] From 14a6118f6e6a38da6a8d8d97a67f0aa72ba0c267 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 20 Nov 2026 10:17:35 +0000 Subject: [PATCH 2/5] Never fail a received answer on a cluster-info outage; retry discovery Review fixes: - After a garbled digest the one cluster-info read used `?`, so a failed read threw away an answer already received (including its next task) and stopped the worker. The answer is now used without the capabilities (the unknown copies are dropped), the held digest is forgotten, and later requests learn them again with `none`. - Any cluster-info failure used to stop a running worker. Startup discovery (query-task authority, Nexus cooperation) and the inline payload policy read before a report now retry a connection failure, timeout, 5xx or 429 with the shared jittered backoff and the 60 s / 5 min ERROR escalation until they succeed or the worker stops. Refusals still return; Clients outside a running worker make one attempt. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 15 +++- README.md | 11 ++- src/lib.rs | 119 ++++++++++++++++++++++----- src/runtime_uploads.rs | 11 ++- src/server_capabilities_digest.rs | 31 +++++-- src/tests/worker_transient_outage.rs | 117 ++++++++++++++++++++++++++ tests/server_capabilities_digest.rs | 79 ++++++++++++++++++ 7 files changed, 346 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d52b5d9..65aeb2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ ## Unreleased +- A failed cluster-info read no longer stops a running worker. Startup + discovery (query-task authority, Nexus cooperation) and the inline-payload + policy read before a report retry a connection failure, timeout, 5xx or 429 + with the shared jittered backoff, uniform in `[0, min(4 s, 0.25 s * + 2^(n-1))]`, logging a `tracing` ERROR once the outage has lasted 60 s and + every 5 minutes after, until it succeeds or the worker stops. An explicit + `retryable: false`, 401, 403 and other refusals are still returned. + `Client::cluster_info` and Clients outside a running worker make one attempt. + - **BEHAVIOUR CHANGE: `Worker::recover_transient_outages` now defaults to `true`.** `run` and `run_until` keep running through transient Server outages instead of returning the error once `WorkerRetryPolicy.max_retries` @@ -96,8 +105,10 @@ because a completion's answer can carry the next leased task, and the full capabilities are read with `GET /api/cluster/info` (`worker_protocol.server_capabilities`) and put in place of the unknown - digest. Only an unknown digest that cannot be recovered this way is - `Error::Protocol` with reason `server_capabilities_digest_unknown`. + digest. If that read fails, the answer is still used without the field and + later requests learn the capabilities again; the request never fails. A + poll whose resend names an unknown digest again is `Error::Protocol` with + reason `server_capabilities_digest_unknown`. Decoded response bodies are capped at 128 MiB by default (`ClientBuilder::max_response_bytes`); a larger body stops being read and fails with the new `Error::ResponseBodyTooLarge`, so a small gzip answer diff --git a/README.md b/README.md index 6eff88f..47732c9 100644 --- a/README.md +++ b/README.md @@ -311,9 +311,10 @@ answer naming a digest that request did not send, and that the answer itself does not carry, makes the Client forget its digest. A task poll is resent once asking for full capabilities. Any other request is never resent, since it may already have committed: its answer is kept as received and the capabilities -are read from `GET /api/cluster/info`. An unknown digest that cannot be -recovered this way is `Error::Protocol` with reason -`server_capabilities_digest_unknown`. Older Servers ignore both headers and +are read from `GET /api/cluster/info`. If that read fails, the answer is still +used without the field and later requests learn the capabilities again. A +poll whose resend names an unknown digest again is `Error::Protocol` with +reason `server_capabilities_digest_unknown`. Older Servers ignore both headers and answer in full. A decoded response body is capped at 128 MiB by default. Set @@ -1170,6 +1171,10 @@ task, attempt and owner with every status `completed` (or `attempt_status` else. When the lease runs out first, the worker logs a `tracing` error and the Server settles the attempt. +A running worker's cluster-info reads (startup discovery and the payload +policy read before a report) retry a connection failure, timeout, 5xx or 429 +the same way. + An explicit `retryable: false`, 401, 403 and other protocol refusals, codec and handler failures stop the worker: it emits a `tracing` error event and returns the error. `.recover_transient_outages(false)` restores bounded retries, diff --git a/src/lib.rs b/src/lib.rs index d9fbeb8..f650525 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3566,6 +3566,44 @@ impl Client { .await } + /// Run a read-only cluster-info discovery. For a running worker, a + /// transient failure is retried with the shared jittered backoff (an + /// outage logged as ERROR after 60 s, then every 5 minutes) until it + /// succeeds or the worker stops; elsewhere it is one attempt. + pub(crate) async fn retry_cluster_info(&self, mut read: F) -> Result + where + F: FnMut() -> Fut, + Fut: Future>, + { + let Some(admission) = self.worker_storage_admission.as_ref() else { + return read().await; + }; + let mut failures = 0_u32; + let mut outage = OutageLog::default(); + loop { + let error = match read().await { + Ok(value) => return Ok(value), + Err(error) => error, + }; + if !cluster_info_failure_is_transient(&error) || admission.stop.load(Ordering::SeqCst) { + return Err(error); + } + failures = failures.saturating_add(1); + outage.report("", "", &error); + let delay = worker_outage_delay(failures); + tracing::warn!( + target: "durable_workflow::worker", + error = %error, + retry_in_ms = delay.as_millis() as u64, + "cluster info unavailable; retrying" + ); + tokio::select! { + _ = tokio::time::sleep(delay) => {} + _ = wait_for_worker_stop(&admission.stop) => return Err(error), + } + } + } + pub async fn cluster_info(&self) -> Result { let discovery: Value = self .request_json( @@ -5750,7 +5788,9 @@ impl Client { "unknown server capabilities digest; reading capabilities from cluster info" ); self.capabilities_digest.forget(); - let info: Value = Box::pin( + // One read-only attempt. It never fails this + // request: the answer is already received. + let info = Box::pin( self.request_json_unresolved_with_timeout::( reqwest::Method::GET, "/cluster/info", @@ -5759,18 +5799,33 @@ impl Client { timeout, ), ) - .await?; - let Some(full) = info["worker_protocol"]["server_capabilities"] - .as_object() - .filter(|full| !full.contains_key("digest")) - else { - return Err(error); + .await; + let full = info.as_ref().ok().and_then(|info| { + info["worker_protocol"]["server_capabilities"] + .as_object() + .filter(|full| !full.contains_key("digest")) + .cloned() + }); + if full.is_none() { + tracing::warn!( + target: "durable_workflow::worker", + path, + "cluster info unavailable; using the answer without its capabilities, which later requests learn again" + ); + } + // Without capabilities the unknown copies are + // dropped from the answer, which is still used. + let fallback = match &full { + Some(full) => { + server_capabilities_digest::Fallback::Capabilities(full) + } + None => server_capabilities_digest::Fallback::Drop, }; let restored = self.capabilities_digest.restore_with( status.as_u16(), &bytes, captured, - Some(full), + Some(fallback), )?; // Later requests start again from `none`. self.capabilities_digest.forget(); @@ -6808,6 +6863,23 @@ fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration { .min(policy.max_backoff) } +/// A failed read-only cluster-info discovery a running worker waits out: a +/// connection or timeout failure, any 5xx or a 429, unless the Server says +/// `retryable: false`. +fn cluster_info_failure_is_transient(error: &Error) -> bool { + let (status, body) = match error { + Error::Timeout | Error::Transport(_) => return true, + Error::Http { status, body } => (status.as_u16(), body.as_str()), + Error::Protocol(failure) => (failure.status, ""), + _ => return false, + }; + let refused = serde_json::from_str::(body) + .ok() + .and_then(|body| body.get("retryable").and_then(Value::as_bool)) + == Some(false); + (status >= 500 || status == 429) && !refused +} + /// A worker task poll: idempotent by its poll request ID, so the only /// request that may be resent to recover elided capabilities. fn is_task_poll(path: &str) -> bool { @@ -9407,12 +9479,14 @@ impl Worker { if self.nexus_cooperative_cancellation_enabled { let info: Value = self .client - .request_json( - reqwest::Method::GET, - "/cluster/info", - RequestProtocol::Worker("1.20"), - Option::<&Value>::None, - ) + .retry_cluster_info(|| { + self.client.request_json( + reqwest::Method::GET, + "/cluster/info", + RequestProtocol::Worker("1.20"), + Option::<&Value>::None, + ) + }) .await?; if !nexus::installed_cooperation(&info["worker_protocol"]["server_capabilities"]) { return Err(Error::CooperativeCancellationUnavailable( @@ -9429,13 +9503,16 @@ impl Worker { // Discovery uses the Worker credential, including worker-only Clients. let discovery: Value = self .client - .request_json_unresolved_with_timeout::( - reqwest::Method::GET, - "/cluster/info", - RequestProtocol::Worker("1.20"), - None, - Duration::from_secs(5), - ) + .retry_cluster_info(|| { + self.client + .request_json_unresolved_with_timeout::( + reqwest::Method::GET, + "/cluster/info", + RequestProtocol::Worker("1.20"), + None, + Duration::from_secs(5), + ) + }) .await?; if discovery["worker_protocol"]["server_capabilities"]["query_task_authority"] != true { return Err(Error::WorkerLoop( diff --git a/src/runtime_uploads.rs b/src/runtime_uploads.rs index ca7f95c..b0909b7 100644 --- a/src/runtime_uploads.rs +++ b/src/runtime_uploads.rs @@ -295,11 +295,14 @@ impl Client { } // Discovery uses the control-plane header with this operation's credential // role, so a worker never needs the application's client token. - let request = self - .runtime_payload_request(reqwest::Method::GET, "/cluster/info", protocol, true)? - .build()?; let info = self - .runtime_payload_json(request, protocol, 2 * 1024 * 1024, None) + .retry_cluster_info(|| async { + let request = self + .runtime_payload_request(reqwest::Method::GET, "/cluster/info", protocol, true)? + .build()?; + self.runtime_payload_json(request, protocol, 2 * 1024 * 1024, None) + .await + }) .await?; self.observe_failure_request_limit(&info); let policy = Policy::from_info(&info)?; diff --git a/src/server_capabilities_digest.rs b/src/server_capabilities_digest.rs index 2fd5073..100326b 100644 --- a/src/server_capabilities_digest.rs +++ b/src/server_capabilities_digest.rs @@ -32,6 +32,15 @@ pub(crate) struct Pair { /// The pair a request was sent with (`None` before any tagged copy). pub(crate) type Captured = Option; +/// What replaces a digest string that matches nothing. +#[derive(Clone, Copy, Debug)] +pub(crate) enum Fallback<'a> { + /// Full capabilities read separately (from cluster info). + Capabilities(&'a Map), + /// Remove the field: the answer is used without its capabilities. + Drop, +} + /// The newest pair this Client learned, shared by its clones. #[derive(Clone, Debug, Default)] pub(crate) struct Digest(Arc>>); @@ -76,7 +85,7 @@ impl Digest { status: u16, bytes: &[u8], captured: &Captured, - fallback: Option<&Map>, + fallback: Option>, ) -> Result>, Error> { if !contains(bytes, FIELD.as_bytes()) { return Ok(None); @@ -154,9 +163,10 @@ fn replace( depth: usize, captured: &Captured, learned: &Option, - fallback: Option<&Map>, + fallback: Option>, changed: &mut bool, ) -> Result<(), String> { + let mut dropped = false; for (key, value) in answer.iter_mut() { if key == FIELD { let Value::String(digest) = value else { @@ -165,14 +175,18 @@ fn replace( if !digest.starts_with(PREFIX) { continue; } - let capabilities = [captured.as_ref(), learned.as_ref()] + let known = [captured.as_ref(), learned.as_ref()] .into_iter() .flatten() .find(|pair| pair.digest == *digest) - .map(|pair| &*pair.capabilities) - .or(fallback) - .ok_or_else(|| digest.clone())?; - *value = Value::Object(capabilities.clone()); + .map(|pair| &*pair.capabilities); + match (known, fallback) { + (Some(capabilities), _) | (None, Some(Fallback::Capabilities(capabilities))) => { + *value = Value::Object(capabilities.clone()); + } + (None, Some(Fallback::Drop)) => dropped = true, + (None, None) => return Err(digest.clone()), + } *changed = true; } else if depth < MAX_DEPTH { if let Value::Object(nested) = value { @@ -180,6 +194,9 @@ fn replace( } } } + if dropped { + answer.remove(FIELD); + } Ok(()) } diff --git a/src/tests/worker_transient_outage.rs b/src/tests/worker_transient_outage.rs index 665cf32..72e9b97 100644 --- a/src/tests/worker_transient_outage.rs +++ b/src/tests/worker_transient_outage.rs @@ -416,3 +416,120 @@ fn a_continuing_outage_escalates_to_error_at_one_minute_then_every_five() { assert_eq!(outage.log.failed(at(759)), None); assert_eq!(outage.log.failed(at(760)), Some(Duration::from_secs(60))); } + +const CLUSTER_INFO: &str = "/api/cluster/info"; + +fn cluster_info_down_twice( + path: &str, + body: &str, + number: usize, +) -> Option<(&'static str, String)> { + match (path, number) { + (CLUSTER_INFO, 1) => Some(("500 Internal Server Error", "upstream exploded".into())), + // Request 2 is dropped without a reply (see `refused_second`). + (CLUSTER_INFO, _) => Some(( + "200 OK", + json!({"worker_protocol":{"server_capabilities":{"query_task_authority":true}}}) + .to_string(), + )), + ("/api/worker/register", _) => { + let request: Value = serde_json::from_str(body).unwrap(); + Some(( + "200 OK", + json!({"registered":true,"worker_id":request["worker_id"], + "heartbeat_interval_seconds":3600,"protocol_version":"1.20", + "server_capabilities":{"query_task_authority":true}}) + .to_string(), + )) + } + _ => None, + } +} + +fn refused_second(path: &str, number: usize) -> bool { + path == CLUSTER_INFO && number == 2 +} + +#[tokio::test] +async fn startup_cluster_info_failures_are_waited_out() { + let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior { + request_override: Some(cluster_info_down_twice), + drop_request: Some(refused_second), + ..MockWorkerBehavior::default() + }); + let mut worker = worker(&server).require_query_task_authority(true); + worker.register_query("counter", "current", |_ctx, _args| async move { + Ok(Value::Null) + }); + let polling = async { + while server.request_count(WORKFLOW_POLL) == 0 { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }; + tokio::time::timeout(Duration::from_secs(10), worker.run_until(polling)) + .await + .expect("startup must get past the outage") + .expect("a cluster-info outage must not stop the worker"); + assert_eq!(server.request_count(CLUSTER_INFO), 3); + assert_eq!(server.request_count("/api/worker/register"), 1); +} + +fn completion_discovery_down_twice( + path: &str, + _body: &str, + number: usize, +) -> Option<(&'static str, String)> { + match (path, number) { + (ACTIVITY_POLL, 1) => Some(("200 OK", activity_delivery(Duration::from_secs(30)))), + (ACTIVITY_POLL, _) => Some(("200 OK", json!({"task": null}).to_string())), + (CLUSTER_INFO, 1) => Some(("500 Internal Server Error", "upstream exploded".into())), + (CLUSTER_INFO, _) => Some(("200 OK", json!({}).to_string())), + (COMPLETE, _) => Some(("200 OK", json!({"recorded":true}).to_string())), + _ => None, + } +} + +#[tokio::test] +async fn run_time_cluster_info_failures_are_waited_out() { + let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior { + request_override: Some(completion_discovery_down_twice), + drop_request: Some(refused_second), + concurrent_requests: true, + ..MockWorkerBehavior::default() + }); + let worker = activity_worker(&server) + .max_concurrent_activity_tasks(1) + .poll_timeout(Duration::from_millis(10)); + let completed = async { + while server.request_count(COMPLETE) == 0 { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }; + tokio::time::timeout(Duration::from_secs(10), worker.run_until(completed)) + .await + .expect("the completion must get past the outage") + .expect("a cluster-info outage must not stop the worker"); + assert_eq!(server.request_count(CLUSTER_INFO), 3); + assert_eq!(server.request_count(COMPLETE), 1); +} + +#[test] +fn cluster_info_failures_other_than_refusals_are_transient() { + let http = |status: u16, body: &str| Error::Http { + status: reqwest::StatusCode::from_u16(status).unwrap(), + body: body.to_owned(), + }; + assert!(cluster_info_failure_is_transient(&http( + 500, + "upstream exploded" + ))); + assert!(cluster_info_failure_is_transient(&http(503, ""))); + assert!(cluster_info_failure_is_transient(&http(429, "{}"))); + assert!(cluster_info_failure_is_transient(&Error::Timeout)); + assert!(!cluster_info_failure_is_transient(&http( + 500, + r#"{"retryable":false}"# + ))); + assert!(!cluster_info_failure_is_transient(&http(401, ""))); + assert!(!cluster_info_failure_is_transient(&http(404, ""))); +} diff --git a/tests/server_capabilities_digest.rs b/tests/server_capabilities_digest.rs index 662bdc3..1841218 100644 --- a/tests/server_capabilities_digest.rs +++ b/tests/server_capabilities_digest.rs @@ -68,6 +68,11 @@ async fn server(replies: Vec) -> (String, JoinHandle>) { bytes.extend_from_slice(&buf[..count]); } heads.push(head); + if reply.status == 0 { + // Close without any reply: the client sees a transport error. + drop(stream); + continue; + } let mut response = format!("HTTP/1.1 {} Reply\r\n", reply.status); for (name, value) in &reply.headers { response.push_str(&format!("{name}: {value}\r\n")); @@ -297,3 +302,77 @@ async fn decoded_response_bodies_are_capped() { ); requests.await.unwrap(); } + +#[tokio::test] +async fn a_garbled_answer_survives_unavailable_cluster_info() { + for (case, failed_read) in [ + ( + "500", + json_reply(500, json!({"message": "upstream exploded"})), + ), + ( + "connection dropped", + Reply { + status: 0, + headers: vec![], + body: vec![], + }, + ), + ] { + let d1 = "sha256:d1"; + let (url, requests) = server(vec![ + json_reply( + 200, + json!({"task": null, "server_capabilities": tagged(d1)}), + ), + json_reply(200, json!({})), + json_reply( + 200, + json!({"recorded": true, "next_task": {"task_id": "next-1"}, + "server_capabilities": "sha256:unknown"}), + ), + failed_read, + // Later requests ask with `none` and learn the capabilities again. + json_reply( + 200, + json!({"task": null, "server_capabilities": tagged("sha256:d2")}), + ), + ]) + .await; + let client = client(&url); + client + .poll_workflow_task_response("worker", "queue", Duration::from_secs(1)) + .await + .unwrap(); + let answer = client + .complete_activity_task("task-1", "attempt-1", "worker", json!("done"), "avro") + .await + .unwrap_or_else(|error| panic!("{case}: the received answer must be used: {error:?}")); + assert_eq!(answer["recorded"], true, "{case}"); + assert_eq!(answer["next_task"]["task_id"], "next-1", "{case}"); + assert!( + answer.get("server_capabilities").is_none(), + "{case}: dropped, not garbled" + ); + let next = client + .poll_workflow_task_response("worker", "queue", Duration::from_secs(1)) + .await + .unwrap(); + assert_eq!( + next.server_capabilities, + Some(capabilities()), + "{case}: learned again" + ); + + let heads = requests.await.unwrap(); + let sent: Vec<_> = heads + .iter() + .map(|head| header(head, "x-durable-workflow-server-capabilities-digest")) + .collect(); + assert_eq!( + sent, + [Some("none"), None, Some(d1), Some("none"), Some("none")], + "{case}" + ); + } +} From e5cc74ac015e541ef6f9f26b78314d758817a0f9 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 20 Nov 2026 11:14:33 +0000 Subject: [PATCH 3/5] Keep reports moving on a stale payload policy; re-encode after 413 Review fixes: - When the 60 s payload-policy cache expired during a cluster-info outage, every report waited for cluster info. An expired policy is now refreshed with one attempt; on an outage the last-known policy is used and discovery is tried again 10 s later. Only a worker with no policy yet waits under the outage rules. - An activity report sent inline under a stale policy, after the Server lowered its inline limit, was refused with 413 payload_too_large and lost. The worker now forgets the policy, rediscovers it (outage rules apply), encodes the same report again, externalizing what the new limit requires, and resends it once. A second refusal keeps today's handling. Workflow-task completions keep their typed size refusals. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 8 ++ README.md | 6 +- src/lib.rs | 48 +++++++++- src/runtime_uploads.rs | 76 ++++++++++++---- src/tests/worker_transient_outage.rs | 127 +++++++++++++++++++++++++++ 5 files changed, 247 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65aeb2f..9320409 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,14 @@ every 5 minutes after, until it succeeds or the worker stops. An explicit `retryable: false`, 401, 403 and other refusals are still returned. `Client::cluster_info` and Clients outside a running worker make one attempt. + When the cached payload policy (refreshed every 60 s) expires during such an + outage, reports keep using the last-known policy and discovery is tried + again 10 s later; only a worker that has no policy yet waits. +- An activity report the Server refuses as too large (413 `payload_too_large`) + because it lowered its inline limit since the policy was cached is no longer + lost. The worker forgets the policy, rediscovers it, encodes the same report + again (externalizing payloads the new limit requires) and resends it once. + A second refusal is handled as a genuinely too-large report. - **BEHAVIOUR CHANGE: `Worker::recover_transient_outages` now defaults to `true`.** `run` and `run_until` keep running through transient Server diff --git a/README.md b/README.md index 47732c9..042b0fb 100644 --- a/README.md +++ b/README.md @@ -1173,7 +1173,11 @@ Server settles the attempt. A running worker's cluster-info reads (startup discovery and the payload policy read before a report) retry a connection failure, timeout, 5xx or 429 -the same way. +the same way. Once a payload policy is known, an outage does not hold reports +back: the last-known policy is used and discovery is tried again 10 s later. +If the Server refuses a report as too large because it lowered its inline +limit, the worker rediscovers the policy, encodes the report again and resends +it once. An explicit `retryable: false`, 401, 403 and other protocol refusals, codec and handler failures stop the worker: it emits a `tracing` error event and diff --git a/src/lib.rs b/src/lib.rs index f650525..ae1810f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5659,6 +5659,10 @@ impl Client { request = request.bearer_auth(token); } + // A task report before externalization, kept so it can be encoded + // again (once) if the Server refuses it as too large for a policy + // that changed since it was cached. + let mut unexternalized: Option = None; if let Some(body) = body { let mut body = serde_json::to_value(body)?; self.check_activity_lease_request(path, Some(&body))?; @@ -5678,6 +5682,9 @@ impl Client { reqwest::Method::POST | reqwest::Method::PUT | reqwest::Method::PATCH ) { self.check_activity_lease_request(path, Some(&body))?; + if report.is_none() && is_task_report(path) { + unexternalized = Some(body.clone()); + } self.externalize_runtime_payloads(&mut body, path, protocol) .await?; self.check_activity_lease_request(path, Some(&body))?; @@ -5852,6 +5859,28 @@ impl Client { if let Some(refusal) = workflow_task_history_limit_exceeded(status, &body, path) { return Err(Error::WorkflowTaskHistoryLimitExceeded(refusal)); } + // A report sent inline under a cached payload policy the + // Server has since lowered is refused as too large, and nothing + // was recorded. Rediscover the policy, encode the same report + // again (externalizing what the new limit requires) and resend + // it once. A second refusal is a genuinely too-large report. + if let Some(original) = unexternalized + .take() + .filter(|_| report_payload_too_large(status, &body)) + { + tracing::warn!( + target: "durable_workflow::worker", + path, + "report refused as too large; rediscovering the payload policy and resending it once" + ); + self.forget_runtime_upload_policy(protocol); + let mut fresh = original; + self.externalize_runtime_payloads(&mut fresh, path, protocol) + .await?; + self.check_activity_lease_request(path, Some(&fresh))?; + *request.body_mut() = Some(serde_json::to_vec(&fresh)?.into()); + continue; + } let error = Error::Http { status, body }; deferred_workflow_completion |= worker_deferred_workflow_completion_retry_after( &error, @@ -6866,7 +6895,7 @@ fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration { /// A failed read-only cluster-info discovery a running worker waits out: a /// connection or timeout failure, any 5xx or a 429, unless the Server says /// `retryable: false`. -fn cluster_info_failure_is_transient(error: &Error) -> bool { +pub(crate) fn cluster_info_failure_is_transient(error: &Error) -> bool { let (status, body) = match error { Error::Timeout | Error::Transport(_) => return true, Error::Http { status, body } => (status.as_u16(), body.as_str()), @@ -6880,6 +6909,23 @@ fn cluster_info_failure_is_transient(error: &Error) -> bool { (status >= 500 || status == 429) && !refused } +/// An activity report (completion or failure) whose payloads may be encoded +/// again after a too-large refusal. Workflow-task completions keep their own +/// typed size refusals unchanged. +fn is_task_report(path: &str) -> bool { + path.starts_with("/worker/activity-tasks/") + && (path.ends_with("/complete") || path.ends_with("/fail")) +} + +/// The Server refused a report as larger than its current limit. +fn report_payload_too_large(status: reqwest::StatusCode, body: &str) -> bool { + status == reqwest::StatusCode::PAYLOAD_TOO_LARGE + || serde_json::from_str::(body) + .ok() + .and_then(|body| body.get("reason")?.as_str().map(str::to_owned)) + .is_some_and(|reason| reason == "payload_too_large") +} + /// A worker task poll: idempotent by its poll request ID, so the only /// request that may be resent to recover elided capabilities. fn is_task_poll(path: &str) -> bool { diff --git a/src/runtime_uploads.rs b/src/runtime_uploads.rs index b0909b7..68b6a16 100644 --- a/src/runtime_uploads.rs +++ b/src/runtime_uploads.rs @@ -17,6 +17,19 @@ pub(super) struct Policy { completion_context: bool, } +/// How long a discovered payload policy is used before it is refreshed. +const POLICY_TTL: Duration = Duration::from_secs(60); +/// After a failed refresh, how long the last-known policy is kept before +/// discovery is tried again. +const POLICY_OUTAGE_RETRY: Duration = Duration::from_secs(10); + +fn policy_role(protocol: RequestProtocol) -> usize { + usize::from(matches!( + protocol, + RequestProtocol::DefaultWorker | RequestProtocol::Worker(_) + )) +} + fn unsupported(message: &str) -> Error { Error::Codec(format!("external_payload_unsupported: {message}")) } @@ -279,31 +292,54 @@ impl Client { } async fn runtime_upload_policy(&self, protocol: RequestProtocol) -> Result { - let role = usize::from(matches!( - protocol, - RequestProtocol::DefaultWorker | RequestProtocol::Worker(_) - )); + let role = policy_role(protocol); let cached = self .runtime_upload_policy .lock() .map_err(|_| unsupported("discovery cache poisoned"))?[role] .clone(); - if let Some((checked, policy)) = cached { - if checked.elapsed() < Duration::from_secs(60) { - return Ok(policy); + if let Some((checked, policy)) = &cached { + if checked.elapsed() < POLICY_TTL { + return Ok(policy.clone()); } } // Discovery uses the control-plane header with this operation's credential // role, so a worker never needs the application's client token. - let info = self - .retry_cluster_info(|| async { - let request = self - .runtime_payload_request(reqwest::Method::GET, "/cluster/info", protocol, true)? - .build()?; - self.runtime_payload_json(request, protocol, 2 * 1024 * 1024, None) - .await - }) - .await?; + let read = || async { + let request = self + .runtime_payload_request(reqwest::Method::GET, "/cluster/info", protocol, true)? + .build()?; + self.runtime_payload_json(request, protocol, 2 * 1024 * 1024, None) + .await + }; + let info = match cached { + // An expired policy is refreshed with one attempt. Through a + // cluster-info outage the last-known policy keeps reports moving, + // and discovery is tried again after a short pause, not before + // every request. + Some((_, stale)) => match read().await { + Ok(info) => info, + Err(error) if crate::cluster_info_failure_is_transient(&error) => { + tracing::warn!( + target: "durable_workflow::worker", + error = %error, + "cluster info unavailable; keeping the last-known payload policy" + ); + let retry_at = Instant::now() + .checked_sub(POLICY_TTL.saturating_sub(POLICY_OUTAGE_RETRY)) + .unwrap_or_else(Instant::now); + self.runtime_upload_policy + .lock() + .map_err(|_| unsupported("discovery cache poisoned"))?[role] = + Some((retry_at, stale.clone())); + return Ok(stale); + } + Err(error) => return Err(error), + }, + // Without any policy, a report cannot be encoded: wait out the + // outage under the running worker's rules. + None => self.retry_cluster_info(read).await?, + }; self.observe_failure_request_limit(&info); let policy = Policy::from_info(&info)?; self.runtime_upload_policy @@ -313,6 +349,14 @@ impl Client { Ok(policy) } + /// Forget the cached payload policy, so the next report rediscovers it + /// (after the Server refused a report as too large). + pub(super) fn forget_runtime_upload_policy(&self, protocol: RequestProtocol) { + if let Ok(mut cache) = self.runtime_upload_policy.lock() { + cache[policy_role(protocol)] = None; + } + } + fn runtime_payload_request( &self, method: reqwest::Method, diff --git a/src/tests/worker_transient_outage.rs b/src/tests/worker_transient_outage.rs index 72e9b97..68844ab 100644 --- a/src/tests/worker_transient_outage.rs +++ b/src/tests/worker_transient_outage.rs @@ -533,3 +533,130 @@ fn cluster_info_failures_other_than_refusals_are_transient() { assert!(!cluster_info_failure_is_transient(&http(401, ""))); assert!(!cluster_info_failure_is_transient(&http(404, ""))); } + +const UPLOAD: &str = "/api/external-payloads/v1"; + +fn payload_policy(threshold: usize) -> String { + let mut policy = super::runtime_uploads::policy(); + policy["limits"]["max_payload_bytes"] = json!(4 * 1024 * 1024); + let storage = &mut policy["namespace"]["external_payload_storage"]; + storage["threshold_bytes"] = json!(threshold); + storage["transport"]["limits"]["max_payload_bytes"] = json!(8 * 1024 * 1024); + policy.to_string() +} + +fn uploaded(blob: &str) -> String { + json!({"schema":"durable-workflow.v2.runtime-external-payload-upload.v1", + "transport_version":1, "reference":super::runtime_uploads::reference(blob)}) + .to_string() +} + +fn sized_worker(server: &MockWorkerServer) -> Worker { + let client = Client::builder(server.base_url()).build().expect("client"); + let mut worker = Worker::new(client, "rust-workers") + .worker_id("retry-worker") + .retry_policy(WorkerRetryPolicy { + max_retries: 5, + initial_backoff: Duration::from_millis(2), + max_backoff: Duration::from_millis(5), + }); + let calls = Arc::new(AtomicUsize::new(0)); + worker.register_activity("RetryActivity", move |_ctx, _args| { + let call = calls.fetch_add(1, Ordering::SeqCst); + async move { + Ok(if call == 0 { + json!("small") + } else { + json!("x".repeat(500_000)) + }) + } + }); + worker +} + +fn lowered_limit(path: &str, body: &str, number: usize) -> Option<(&'static str, String)> { + match (path, number) { + (ACTIVITY_POLL, 1 | 2) => Some(("200 OK", activity_delivery(Duration::from_secs(30)))), + (ACTIVITY_POLL, _) => Some(("200 OK", json!({"task": null}).to_string())), + // Cached at 2 MB; the Server then lowers its inline limit to 100 KB. + (CLUSTER_INFO, 1) => Some(("200 OK", payload_policy(2 * 1024 * 1024))), + (CLUSTER_INFO, _) => Some(("200 OK", payload_policy(100 * 1024))), + (UPLOAD, _) => Some(("201 Created", uploaded(body))), + // The large result sent inline under the stale policy is refused. + (COMPLETE, 2) => Some(( + "413 Payload Too Large", + json!({"reason":"payload_too_large","message":"request exceeds 100 KB"}).to_string(), + )), + (COMPLETE, _) => Some(("200 OK", json!({"recorded":true}).to_string())), + _ => None, + } +} + +#[tokio::test] +async fn a_report_refused_after_a_lowered_limit_is_externalized_and_resent_once() { + let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior { + request_override: Some(lowered_limit), + ..MockWorkerBehavior::default() + }); + let worker = sized_worker(&server); + for _ in 0..2 { + let outcome = worker.poll_activity_once().await; + assert!( + matches!(outcome, Ok(ManagedPollOutcome::Handled)), + "{outcome:?}" + ); + } + let completes = server.request_bodies(COMPLETE); + assert_eq!(completes.len(), 3, "small, refused large, re-encoded large"); + assert!( + completes[1]["result"].to_string().len() > 500_000, + "first sent inline" + ); + assert_eq!( + completes[2]["result"]["external_payload"]["schema"], + crate::runtime_payloads::SCHEMA, + "resent with the payload externalized: {}", + completes[2]["result"] + ); + assert_eq!(server.request_count(UPLOAD), 1); + assert_eq!(server.request_count(CLUSTER_INFO), 2, "rediscovered once"); +} + +fn policy_then_outage(path: &str, _body: &str, number: usize) -> Option<(&'static str, String)> { + match (path, number) { + (ACTIVITY_POLL, 1..=3) => Some(("200 OK", activity_delivery(Duration::from_secs(30)))), + (ACTIVITY_POLL, _) => Some(("200 OK", json!({"task": null}).to_string())), + (CLUSTER_INFO, 1) => Some(("200 OK", payload_policy(2 * 1024 * 1024))), + (CLUSTER_INFO, _) => Some(("500 Internal Server Error", "upstream exploded".into())), + (COMPLETE, _) => Some(("200 OK", json!({"recorded":true}).to_string())), + _ => None, + } +} + +#[tokio::test] +async fn an_expired_policy_keeps_reports_moving_through_a_cluster_info_outage() { + let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior { + request_override: Some(policy_then_outage), + ..MockWorkerBehavior::default() + }); + let worker = activity_worker(&server); + worker.poll_activity_once().await.unwrap(); + assert_eq!(server.request_count(CLUSTER_INFO), 1); + // Age the cached policy past its 60 s lifetime. + { + let mut cache = worker.client.runtime_upload_policy.lock().unwrap(); + for entry in cache.iter_mut().flatten() { + entry.0 = std::time::Instant::now() - Duration::from_secs(61); + } + } + let started = Instant::now(); + worker.poll_activity_once().await.unwrap(); + worker.poll_activity_once().await.unwrap(); + assert!( + started.elapsed() < Duration::from_secs(2), + "reports did not wait for the outage" + ); + assert_eq!(server.request_count(COMPLETE), 3, "every report was sent"); + // One refresh attempt, then the last-known policy for the next 10 s. + assert_eq!(server.request_count(CLUSTER_INFO), 2); +} From 9caaf738bd00956f6c5c4a8ebeb8d049fc09103a Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 20 Nov 2026 11:27:42 +0000 Subject: [PATCH 4/5] Fail a still-too-large activity result instead of stopping the worker A completion still refused as too large after the payload policy was rediscovered, or one that could not be externalized (above the upload limit, or no payload storage), returned an error from the settlement, and the poll lane treated it as fatal: the worker exited. Such an attempt is now reported as a non-retryable ActivityResultTooLarge failure (failure.type and failure.class), as the PHP SDK does, and the worker keeps running. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 5 +- README.md | 3 +- src/cooperative_cancellation.rs | 33 ++++--- src/lib.rs | 125 +++++++++++++++++++++++---- src/tests/worker_transient_outage.rs | 70 +++++++++++++++ 5 files changed, 199 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9320409..614e8ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,10 @@ because it lowered its inline limit since the policy was cached is no longer lost. The worker forgets the policy, rediscovers it, encodes the same report again (externalizing payloads the new limit requires) and resends it once. - A second refusal is handled as a genuinely too-large report. + A result that is still too large (refused again, above the upload limit, or + with no payload storage) no longer stops the worker: the attempt is failed + as a non-retryable `ActivityResultTooLarge` (`failure.type` and + `failure.class`), as in the PHP SDK, and the worker keeps running. - **BEHAVIOUR CHANGE: `Worker::recover_transient_outages` now defaults to `true`.** `run` and `run_until` keep running through transient Server diff --git a/README.md b/README.md index 042b0fb..b2c15be 100644 --- a/README.md +++ b/README.md @@ -1177,7 +1177,8 @@ the same way. Once a payload policy is known, an outage does not hold reports back: the last-known policy is used and discovery is tried again 10 s later. If the Server refuses a report as too large because it lowered its inline limit, the worker rediscovers the policy, encodes the report again and resends -it once. +it once. A result that is still too large, or cannot be externalized, fails the +attempt as a non-retryable `ActivityResultTooLarge`; the worker keeps running. An explicit `retryable: false`, 401, 403 and other protocol refusals, codec and handler failures stop the worker: it emits a `tracing` error event and diff --git a/src/cooperative_cancellation.rs b/src/cooperative_cancellation.rs index 39ac558..c3effbf 100644 --- a/src/cooperative_cancellation.rs +++ b/src/cooperative_cancellation.rs @@ -1892,30 +1892,27 @@ impl Worker { } return Ok(ManagedPollOutcome::Handled); } - let task_outcome = if result.is_ok() { + let mut task_outcome = if result.is_ok() { "completed" } else { "failed" }; let settled = match result { Ok(value) => { - self.settle_activity_report( - crate::ActivityReport::Completed, - &guard.task_id, - &guard.attempt_id, - &guard.owner, - lease_deadline, - || { - self.client.complete_activity_task( - &guard.task_id, - &guard.attempt_id, - &guard.owner, - value.clone(), - &task.payload_codec, - ) - }, - ) - .await? + let (settlement, too_large) = self + .settle_activity_completion( + &guard.task_id, + &guard.attempt_id, + &guard.owner, + lease_deadline, + &value, + &task.payload_codec, + ) + .await?; + if too_large { + task_outcome = "failed"; + } + settlement } Err(error) if worker_storage_admission_body(&error).is_some() => return Err(error), Err(error) => { diff --git a/src/lib.rs b/src/lib.rs index ae1810f..a0cf6e9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6926,6 +6926,29 @@ fn report_payload_too_large(status: reqwest::StatusCode, body: &str) -> bool { .is_some_and(|reason| reason == "payload_too_large") } +/// `failure.type` and `failure.class` of an attempt whose result could not be +/// delivered because it is too large. +pub(crate) const ACTIVITY_RESULT_TOO_LARGE: &str = "ActivityResultTooLarge"; + +/// An activity completion that failed only because its result is too large: +/// the Server refused it as too large (after the one re-encode), or it could +/// not be externalized (too large to upload, or no payload storage). +fn activity_result_too_large(error: &Error) -> bool { + match error { + Error::ActivityTaskRejected(rejection) => { + rejection.status == 413 || rejection.reason == "payload_too_large" + } + Error::Codec(message) => [ + "external_payload_oversized:", + "external_payload_unavailable:", + "payload_too_large:", + ] + .iter() + .any(|prefix| message.starts_with(prefix)), + _ => false, + } +} + /// A worker task poll: idempotent by its poll request ID, so the only /// request that may be resent to recover elided capabilities. fn is_task_poll(path: &str) -> bool { @@ -10414,23 +10437,20 @@ impl Worker { Err(error) if activity_hydration::stopped(&error) => ActivitySettlement::Recorded, Err(error) if worker_storage_admission_body(&error).is_some() => return Err(error), Ok(value) => { - self.settle_activity_report( - ActivityReport::Completed, - &task_id, - &attempt_id, - &lease_owner, - lease_deadline, - || { - self.client.complete_activity_task( - &task_id, - &attempt_id, - &lease_owner, - value.clone(), - &codec, - ) - }, - ) - .await? + let (settlement, too_large) = self + .settle_activity_completion( + &task_id, + &attempt_id, + &lease_owner, + lease_deadline, + &value, + &codec, + ) + .await?; + if too_large { + task_outcome = "failed"; + } + settlement } Err(error) => { let failure = ActivityError::from_error(&error, "RustActivityFailure"); @@ -10556,6 +10576,77 @@ impl Worker { } } + /// Complete an activity. When its result cannot be delivered because it is + /// too large (still refused after the payload policy was rediscovered, or + /// too large to externalize), report the attempt as a non-retryable + /// `ActivityResultTooLarge` failure instead, so the worker keeps running. + /// Returns the settlement and whether it was that failure. + #[allow(clippy::too_many_arguments)] + pub(crate) async fn settle_activity_completion( + &self, + task_id: &str, + attempt_id: &str, + owner: &str, + lease_deadline: Option, + value: &T, + codec: &str, + ) -> Result<(ActivitySettlement, bool)> { + let completion = self + .settle_activity_report( + ActivityReport::Completed, + task_id, + attempt_id, + owner, + lease_deadline, + || { + self.client.complete_activity_task( + task_id, + attempt_id, + owner, + value.clone(), + codec, + ) + }, + ) + .await; + let error = match completion { + Ok(settlement) => return Ok((settlement, false)), + Err(error) if activity_result_too_large(&error) => error, + Err(error) => return Err(error), + }; + tracing::error!( + target: "durable_workflow::worker", + worker_id = %self.worker_id, + task_id, + error = %error, + "activity result is too large to deliver; reporting a non-retryable ActivityResultTooLarge failure" + ); + let failure = json!({ + "type": ACTIVITY_RESULT_TOO_LARGE, + "class": ACTIVITY_RESULT_TOO_LARGE, + "non_retryable": true, + "message": format!("activity result is too large to deliver: {error}"), + }); + let settlement = self + .settle_activity_report( + ActivityReport::Failed, + task_id, + attempt_id, + owner, + lease_deadline, + || { + self.client.fail_activity_task_with_failure( + task_id, + attempt_id, + owner, + failure.clone(), + ) + }, + ) + .await?; + Ok((settlement, true)) + } + async fn poll_query_once(&self) -> Result { let mut poll_metrics = self.metrics.worker(false, "query", &self.task_queue); let poll_request_id = unique_request_id("rust-query-poll"); diff --git a/src/tests/worker_transient_outage.rs b/src/tests/worker_transient_outage.rs index 68844ab..4f7adfa 100644 --- a/src/tests/worker_transient_outage.rs +++ b/src/tests/worker_transient_outage.rs @@ -660,3 +660,73 @@ async fn an_expired_policy_keeps_reports_moving_through_a_cluster_info_outage() // One refresh attempt, then the last-known policy for the next 10 s. assert_eq!(server.request_count(CLUSTER_INFO), 2); } + +const FAIL: &str = "/api/worker/activity-tasks/retry-task/fail"; + +fn too_large_refusal() -> (&'static str, String) { + ( + "413 Payload Too Large", + json!({"reason":"payload_too_large","message":"request exceeds the limit"}).to_string(), + ) +} + +fn still_too_large(path: &str, body: &str, number: usize) -> Option<(&'static str, String)> { + match (path, number) { + (ACTIVITY_POLL, 1 | 2) => Some(("200 OK", activity_delivery(Duration::from_secs(30)))), + (ACTIVITY_POLL, _) => Some(("200 OK", json!({"task": null}).to_string())), + (CLUSTER_INFO, 1) => Some(("200 OK", payload_policy(2 * 1024 * 1024))), + (CLUSTER_INFO, _) => Some(("200 OK", payload_policy(100 * 1024))), + (UPLOAD, _) => Some(("201 Created", uploaded(body))), + // Refused inline, and refused again after it was re-encoded. + (COMPLETE, 2 | 3) => Some(too_large_refusal()), + (COMPLETE, _) | (FAIL, _) => Some(("200 OK", json!({"recorded":true}).to_string())), + _ => None, + } +} + +fn no_storage_after_rediscovery( + path: &str, + body: &str, + number: usize, +) -> Option<(&'static str, String)> { + match (path, number) { + (CLUSTER_INFO, 2..) => { + let mut policy: Value = serde_json::from_str(&payload_policy(100 * 1024)).unwrap(); + policy["namespace"]["external_payload_storage"]["status"] = json!("unavailable"); + Some(("200 OK", policy.to_string())) + } + (COMPLETE, 2) => Some(too_large_refusal()), + _ => still_too_large(path, body, number), + } +} + +#[tokio::test] +async fn a_result_still_too_large_after_rediscovery_fails_the_attempt_not_the_worker() { + for (case, handler) in [ + ("refused again", still_too_large as RequestOverride), + ("no storage", no_storage_after_rediscovery), + ] { + let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior { + request_override: Some(handler), + ..MockWorkerBehavior::default() + }); + let worker = sized_worker(&server); + for _ in 0..2 { + let outcome = worker.poll_activity_once().await; + assert!( + matches!(outcome, Ok(ManagedPollOutcome::Handled)), + "{case}: the worker keeps running: {outcome:?}" + ); + } + let failures = server.request_bodies(FAIL); + assert_eq!(failures.len(), 1, "{case}"); + let failure = &failures[0]["failure"]; + assert_eq!(failure["type"], "ActivityResultTooLarge", "{case}"); + assert_eq!(failure["class"], "ActivityResultTooLarge", "{case}"); + assert_eq!(failure["non_retryable"], true, "{case}"); + assert!( + failure["message"].as_str().unwrap().contains("too large"), + "{case}" + ); + } +} From 5a42199acfa98115f1dde3419999b36f1cd1bf80 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sat, 21 Nov 2026 03:43:28 +0000 Subject: [PATCH 5/5] Require a typed size refusal before re-encoding an activity report --- src/lib.rs | 6 +++--- src/tests/worker_transient_outage.rs | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index a0cf6e9..2436a6f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6917,10 +6917,10 @@ fn is_task_report(path: &str) -> bool { && (path.ends_with("/complete") || path.ends_with("/fail")) } -/// The Server refused a report as larger than its current limit. +/// The Server refused the report's size before reading its body. fn report_payload_too_large(status: reqwest::StatusCode, body: &str) -> bool { status == reqwest::StatusCode::PAYLOAD_TOO_LARGE - || serde_json::from_str::(body) + && serde_json::from_str::(body) .ok() .and_then(|body| body.get("reason")?.as_str().map(str::to_owned)) .is_some_and(|reason| reason == "payload_too_large") @@ -6936,7 +6936,7 @@ pub(crate) const ACTIVITY_RESULT_TOO_LARGE: &str = "ActivityResultTooLarge"; fn activity_result_too_large(error: &Error) -> bool { match error { Error::ActivityTaskRejected(rejection) => { - rejection.status == 413 || rejection.reason == "payload_too_large" + rejection.status == 413 && rejection.reason == "payload_too_large" } Error::Codec(message) => [ "external_payload_oversized:", diff --git a/src/tests/worker_transient_outage.rs b/src/tests/worker_transient_outage.rs index 4f7adfa..bfe29f9 100644 --- a/src/tests/worker_transient_outage.rs +++ b/src/tests/worker_transient_outage.rs @@ -670,6 +670,23 @@ fn too_large_refusal() -> (&'static str, String) { ) } +#[test] +fn only_a_typed_size_refusal_allows_reencoding_a_report() { + let (_, typed_body) = too_large_refusal(); + assert!(crate::report_payload_too_large( + reqwest::StatusCode::PAYLOAD_TOO_LARGE, + &typed_body, + )); + assert!(!crate::report_payload_too_large( + reqwest::StatusCode::PAYLOAD_TOO_LARGE, + "Proxy rejected the request", + )); + assert!(!crate::report_payload_too_large( + reqwest::StatusCode::INTERNAL_SERVER_ERROR, + &typed_body, + )); +} + fn still_too_large(path: &str, body: &str, number: usize) -> Option<(&'static str, String)> { match (path, number) { (ACTIVITY_POLL, 1 | 2) => Some(("200 OK", activity_delivery(Duration::from_secs(30)))),