From 5be854b60d5cca5963f9bbf0f75d28423213b416 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 20 Nov 2026 00:54:06 +0000 Subject: [PATCH 1/6] Native: compress JSON responses like PHP's CompressResponse Native sent every answer uncompressed and advertised response_compression: []. PHP Server gzip- or deflate-encodes JSON answers of 1024 bytes or more when Accept-Encoding allows it (DW_COMPRESSION_ENABLED, default true). On a Dev cell behind a Cloudflare tunnel the origin answers are the cell's egress, which is shaped to 4 Mbit/s. The Cloudflare edge always asks the origin for "br, gzip" and decodes for clients that did not ask, so this reduces tunnel egress without any SDK change. Lab measurement (DW Standard Workflow v1, 1 wf/s, 4 workers): 384.8 KB -> 106.0 KB of origin response bytes per workflow. - New response_compression middleware, outermost on the router: PHP's encoding preference (gzip over deflate at equal q, *, q=0), JSON content types only, 1024-byte minimum, existing Content-Encoding left alone, Vary: Accept-Encoding. Only in-memory (exact-sized) bodies are encoded; streamed answers pass through. - DW_COMPRESSION_ENABLED (and its legacy name) is honored instead of ignored. - cluster info advertises ["gzip", "deflate"] when enabled, as PHP does; the native deviation is removed from the parity fixture. - History-page gzip reuses the same RFC 1952 encoder. Co-Authored-By: Claude Opus 5.5 --- docs/rust-server-port.md | 7 +- rust/src/config_audit.rs | 4 +- rust/src/runtime/cluster_info.rs | 6 +- rust/src/runtime/history_pages.rs | 12 +- rust/src/runtime/http.rs | 2 + rust/src/runtime/mod.rs | 1 + rust/src/runtime/response_compression.rs | 253 ++++++++++++++++++ rust/tests/execution/activity_results.rs | 2 +- .../ServerParity/cluster-capabilities.json | 4 - ...rParityClusterCapabilitiesContractTest.mjs | 2 +- 10 files changed, 269 insertions(+), 24 deletions(-) create mode 100644 rust/src/runtime/response_compression.rs diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index b3dbb5ebe..8de8d60ae 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -338,8 +338,11 @@ contract fixture before its correction. Capabilities are true only where native serves the feature. `workflow_activity_result_inspection` is now advertised: the `cluster-capabilities` fixture qualifies the activity result read against - published 2.11.7. Native advertises no manifest it does not publish and no - response compression. `native_unqualified` names each member it still + published 2.11.7. Native advertises no manifest it does not publish. + Like PHP's `CompressResponse`, native gzip- or deflate-encodes JSON answers + of 1024 bytes or more when `Accept-Encoding` allows it, and advertises + `response_compression: ["gzip", "deflate"]`, unless + `DW_COMPRESSION_ENABLED` is false. `native_unqualified` names each member it still leaves out: `topology`, the diagnostics view, the external payload transaction limit and the conformance manifests. Since server#1005 (3.1.0) native serves `supported_sdk_versions` and `client_compatibility` from one diff --git a/rust/src/config_audit.rs b/rust/src/config_audit.rs index dfaaafa23..c64684649 100644 --- a/rust/src/config_audit.rs +++ b/rust/src/config_audit.rs @@ -581,7 +581,7 @@ pub const SETTINGS: &[Setting] = &[ Setting { name: "DW_BOOTSTRAP_RETRIES", legacy: Some("WORKFLOW_SERVER_BOOTSTRAP_RETRIES"), disposition: Ignored }, Setting { name: "DW_CALLER_LABEL_HEADER", legacy: Some("WORKFLOW_SERVER_CALLER_LABEL_HEADER"), disposition: Ignored }, Setting { name: "DW_CALLER_TYPE_HEADER", legacy: Some("WORKFLOW_SERVER_CALLER_TYPE_HEADER"), disposition: Ignored }, - Setting { name: "DW_COMPRESSION_ENABLED", legacy: Some("WORKFLOW_SERVER_COMPRESSION_ENABLED"), disposition: Ignored }, + Setting { name: "DW_COMPRESSION_ENABLED", legacy: Some("WORKFLOW_SERVER_COMPRESSION_ENABLED"), disposition: Honored }, Setting { name: "DW_DEFAULT_NAMESPACE", legacy: Some("WORKFLOW_SERVER_DEFAULT_NAMESPACE"), disposition: Refused(AnyCase("default"), DEFAULT_NAMESPACE) }, Setting { name: "DW_DUE_TIMER_RECOVERY_SCAN_LIMIT", legacy: Some("WORKFLOW_SERVER_DUE_TIMER_RECOVERY_SCAN_LIMIT"), disposition: Ignored }, Setting { name: "DW_ENV_AUDIT_STRICT", legacy: None, disposition: Honored }, @@ -995,7 +995,7 @@ mod tests { &[("DW_V2_PIN_TO_RECORDED_FINGERPRINT", "no")], &[("DW_SERIALIZER", "")], &[("DW_STORAGE_CONNECTION", "pgsql"), ("DB_CONNECTION", "pgsql")], - // Ignored and reported settings take any value. + // Ignored and reported settings, and honored booleans, take any value. &[("DW_MAX_PENDING_ACTIVITIES", "10"), ("DW_COMPRESSION_ENABLED", "false")], ]; for values in accepted { diff --git a/rust/src/runtime/cluster_info.rs b/rust/src/runtime/cluster_info.rs index b8efc3520..a94fa73af 100644 --- a/rust/src/runtime/cluster_info.rs +++ b/rust/src/runtime/cluster_info.rs @@ -134,8 +134,8 @@ pub(super) fn capabilities() -> Value { json!({"schema": "durable_workflow.protocol.Value", "fingerprint": super::history_export::VALUE_FINGERPRINT, "framing": "single_object", "magic_hex": "c301"}), ); - // Native sends uncompressed responses. - map.insert("response_compression".into(), json!([])); + // PHP's CompressResponse encodings, unless DW_COMPRESSION_ENABLED is false. + map.insert("response_compression".into(), json!(super::response_compression::advertised())); Value::Object(map) } @@ -244,7 +244,7 @@ mod tests { assert_eq!(caps["child_workflow_start_turn"], json!(true)); assert_eq!(caps["condition_wait_idempotent_reopen"], json!(true)); assert_eq!(caps["condition_wait_occurrence_deadline"], json!(true)); - assert_eq!(caps["response_compression"], json!([])); + assert_eq!(caps["response_compression"], json!(["gzip", "deflate"])); assert_eq!(caps["timer_runtime_contract"], json!(false)); assert_eq!(caps["principal_attribution_contract"], json!(true)); } diff --git a/rust/src/runtime/history_pages.rs b/rust/src/runtime/history_pages.rs index 53ca3f54c..99c4e9279 100644 --- a/rust/src/runtime/history_pages.rs +++ b/rust/src/runtime/history_pages.rs @@ -113,7 +113,7 @@ pub(super) fn compress(payload: &mut Value, encoding: Option<&'static str>) -> R } let json = serde_json::to_vec(events)?; let compressed = match encoding { - "gzip" => gzip(&json), + "gzip" => super::response_compression::gzip(&json), _ => miniz_oxide::deflate::compress_to_vec(&json, LEVEL), }; payload["history_events"] = json!([]); @@ -122,16 +122,6 @@ pub(super) fn compress(payload: &mut Value, encoding: Option<&'static str>) -> R Ok(()) } -/// `gzencode`: an RFC 1952 member around one raw DEFLATE stream. -fn gzip(data: &[u8]) -> Vec { - const CRC32: crc::Crc = crc::Crc::::new(&crc::CRC_32_ISO_HDLC); - // Magic, deflate method, no flags or time, default level, Unix. - let mut member = vec![0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 0, 3]; - member.extend(miniz_oxide::deflate::compress_to_vec(data, LEVEL)); - member.extend(CRC32.checksum(data).to_le_bytes()); - member.extend((data.len() as u32).to_le_bytes()); - member -} #[cfg(test)] mod tests { diff --git a/rust/src/runtime/http.rs b/rust/src/runtime/http.rs index da763b270..2f8d7d414 100644 --- a/rust/src/runtime/http.rs +++ b/rust/src/runtime/http.rs @@ -425,6 +425,8 @@ pub fn router(runtime: Runtime) -> Router { .layer(middleware::from_fn_with_state(runtime.clone(), storage_admission_guard)) .layer(middleware::from_fn_with_state(runtime.clone(), guard)) .layer(middleware::from_fn(super::payload_limits::enforce)) + // PHP's CompressResponse wraps every answer, refusals included. + .layer(middleware::from_fn(super::response_compression::compress)) .with_state(runtime) } diff --git a/rust/src/runtime/mod.rs b/rust/src/runtime/mod.rs index 5575946a1..8ea36a32a 100644 --- a/rust/src/runtime/mod.rs +++ b/rust/src/runtime/mod.rs @@ -99,6 +99,7 @@ mod command_payloads; mod completion_rules; mod deployments; mod priority_fairness; +mod response_compression; mod route_fallback; mod redrive; mod redrive_plan; diff --git a/rust/src/runtime/response_compression.rs b/rust/src/runtime/response_compression.rs new file mode 100644 index 000000000..368d71b8d --- /dev/null +++ b/rust/src/runtime/response_compression.rs @@ -0,0 +1,253 @@ +//! PHP's `CompressResponse` middleware: a JSON answer of 1024 bytes or more +//! is sent gzip- or deflate-encoded when the request's `Accept-Encoding` +//! allows it, unless `DW_COMPRESSION_ENABLED` is false (default true). +//! +//! As in PHP, gzip wins over deflate at equal weight, `*` stands for both, +//! `q=0` refuses an encoding, an answer that already has a +//! `Content-Encoding` is left alone, and `Vary: Accept-Encoding` is added. +//! Only answers whose whole body is already in memory (an exact size hint) +//! are encoded; streamed bodies pass through unchanged. +use axum::{ + body::{Body, HttpBody as _}, + extract::Request, + http::{HeaderMap, HeaderValue, header}, + middleware::Next, + response::Response, +}; +use std::sync::OnceLock; + +/// `CompressResponse::MIN_COMPRESS_BYTES`. +const MIN_COMPRESS_BYTES: u64 = 1024; +/// The largest in-memory answer that is encoded; bigger ones pass through. +const MAX_COMPRESS_BYTES: u64 = 64 * 1024 * 1024; +/// zlib's default level, which PHP's `gzencode` and `gzdeflate` use. +const LEVEL: u8 = 6; + +/// `config('server.compression.enabled')`: `filter_var(DW_COMPRESSION_ENABLED +/// ?? WORKFLOW_SERVER_COMPRESSION_ENABLED ?? true, FILTER_VALIDATE_BOOLEAN, +/// FILTER_NULL_ON_FAILURE) ?? true`, read once. +pub(super) fn enabled() -> bool { + static ENABLED: OnceLock = OnceLock::new(); + *ENABLED.get_or_init(|| configured(|name| std::env::var(name).ok())) +} + +fn configured(lookup: impl Fn(&str) -> Option) -> bool { + crate::config_audit::effective(lookup, "DW_COMPRESSION_ENABLED", Some("WORKFLOW_SERVER_COMPRESSION_ENABLED")) + .is_none_or(|value| value.filter_bool().unwrap_or(true)) +} + +/// The encodings advertised as `response_compression`. +pub(super) fn advertised() -> &'static [&'static str] { + if enabled() { &["gzip", "deflate"] } else { &[] } +} + +pub(super) async fn compress(request: Request, next: Next) -> Response { + let encoding = if enabled() { preferred(request.headers()) } else { None }; + let response = next.run(request).await; + match encoding { + Some(encoding) => encode(response, encoding).await, + None => response, + } +} + +/// `CompressResponse::preferredEncoding` over every `Accept-Encoding` value. +fn preferred(headers: &HeaderMap) -> Option<&'static str> { + let mut accept = String::new(); + for value in headers.get_all(header::ACCEPT_ENCODING) { + if let Ok(value) = value.to_str() { + if !accept.is_empty() { + accept.push(','); + } + accept.push_str(value); + } + } + let (mut gzip, mut deflate, mut wildcard) = (None::, None::, None::); + for part in accept.to_ascii_lowercase().split(',') { + let mut segments = part.split(';').map(str::trim); + let name = segments.next().unwrap_or(""); + if name.is_empty() { + continue; + } + let mut weight = 1.0; + for segment in segments { + if let Some(value) = segment.strip_prefix("q=") { + weight = value.trim().parse::().ok().filter(|q| q.is_finite()).map_or(0.0, |q| q.clamp(0.0, 1.0)); + } + } + match name { + "*" => wildcard = Some(weight), + "gzip" => gzip = Some(gzip.map_or(weight, |q| q.max(weight))), + "deflate" => deflate = Some(deflate.map_or(weight, |q| q.max(weight))), + _ => {} + } + } + let gzip = gzip.or(wildcard).unwrap_or(0.0); + let deflate = deflate.or(wildcard).unwrap_or(0.0); + if gzip > 0.0 && gzip >= deflate { + Some("gzip") + } else if deflate > 0.0 { + Some("deflate") + } else { + None + } +} + +fn json(headers: &HeaderMap) -> bool { + headers + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.contains("application/json")) +} + +async fn encode(response: Response, encoding: &'static str) -> Response { + let size = response.body().size_hint().exact(); + if response.headers().contains_key(header::CONTENT_ENCODING) + || !json(response.headers()) + || !size.is_some_and(|size| (MIN_COMPRESS_BYTES..=MAX_COMPRESS_BYTES).contains(&size)) + { + return response; + } + let (mut parts, body) = response.into_parts(); + let Ok(bytes) = axum::body::to_bytes(body, MAX_COMPRESS_BYTES as usize).await else { + // An exact-sized in-memory body cannot fail to collect. + return Response::from_parts(parts, Body::empty()); + }; + let encoded = match encoding { + "gzip" => gzip(&bytes), + _ => miniz_oxide::deflate::compress_to_vec(&bytes, LEVEL), + }; + parts.headers.remove(header::CONTENT_LENGTH); + parts.headers.insert(header::CONTENT_ENCODING, HeaderValue::from_static(encoding)); + let vary = parts + .headers + .get_all(header::VARY) + .iter() + .filter_map(|value| value.to_str().ok()) + .any(|value| value.split(',').any(|name| name.trim().eq_ignore_ascii_case("accept-encoding"))); + if !vary { + parts.headers.append(header::VARY, HeaderValue::from_static("Accept-Encoding")); + } + Response::from_parts(parts, Body::from(encoded)) +} + +/// `gzencode`: an RFC 1952 member around one raw DEFLATE stream. +pub(super) fn gzip(data: &[u8]) -> Vec { + const CRC32: crc::Crc = crc::Crc::::new(&crc::CRC_32_ISO_HDLC); + // Magic, deflate method, no flags or time, default level, Unix. + let mut member = vec![0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 0, 3]; + member.extend(miniz_oxide::deflate::compress_to_vec(data, LEVEL)); + member.extend(CRC32.checksum(data).to_le_bytes()); + member.extend((data.len() as u32).to_le_bytes()); + member +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::{Router, http::StatusCode, middleware, response::IntoResponse, routing::get}; + use tower::ServiceExt as _; + + fn accept(value: &str) -> HeaderMap { + let mut headers = HeaderMap::new(); + headers.insert(header::ACCEPT_ENCODING, HeaderValue::from_str(value).unwrap()); + headers + } + + #[test] + fn preferred_encoding_follows_php() { + assert_eq!(preferred(&HeaderMap::new()), None); + assert_eq!(preferred(&accept("gzip")), Some("gzip")); + assert_eq!(preferred(&accept("br, gzip")), Some("gzip")); + assert_eq!(preferred(&accept("deflate")), Some("deflate")); + assert_eq!(preferred(&accept("deflate, gzip")), Some("gzip")); + assert_eq!(preferred(&accept("gzip;q=0.5, deflate")), Some("deflate")); + assert_eq!(preferred(&accept("gzip;q=0")), None); + assert_eq!(preferred(&accept("GZIP")), Some("gzip")); + assert_eq!(preferred(&accept("*")), Some("gzip")); + assert_eq!(preferred(&accept("*;q=0")), None); + assert_eq!(preferred(&accept("gzip;q=0, *")), Some("deflate")); + assert_eq!(preferred(&accept("br, zstd")), None); + assert_eq!(preferred(&accept("identity")), None); + assert_eq!(preferred(&accept("gzip;q=x")), None); + } + + #[test] + fn setting_defaults_to_enabled_and_reads_the_legacy_name() { + let env = |pairs: &'static [(&'static str, &'static str)]| { + move |name: &str| pairs.iter().find(|(key, _)| *key == name).map(|(_, value)| (*value).to_owned()) + }; + assert!(configured(env(&[]))); + assert!(configured(env(&[("DW_COMPRESSION_ENABLED", "true")]))); + assert!(!configured(env(&[("DW_COMPRESSION_ENABLED", "false")]))); + assert!(!configured(env(&[("DW_COMPRESSION_ENABLED", "0")]))); + assert!(!configured(env(&[("DW_COMPRESSION_ENABLED", "")]))); + assert!(configured(env(&[("DW_COMPRESSION_ENABLED", "maybe")]))); + assert!(!configured(env(&[("WORKFLOW_SERVER_COMPRESSION_ENABLED", "off")]))); + assert!(configured(env(&[("DW_COMPRESSION_ENABLED", "on"), ("WORKFLOW_SERVER_COMPRESSION_ENABLED", "off")]))); + } + + fn app() -> Router { + Router::new() + .route("/big", get(|| async { axum::Json(serde_json::json!({"blob": "x".repeat(4096)})) })) + .route("/small", get(|| async { axum::Json(serde_json::json!({"ok": true})) })) + .route("/text", get(|| async { "y".repeat(4096) })) + .route( + "/encoded", + get(|| async { + ([(header::CONTENT_TYPE, "application/json"), (header::CONTENT_ENCODING, "identity")], "z".repeat(4096)) + .into_response() + }), + ) + .route( + "/streamed", + get(|| async { + let chunks = futures_util::stream::iter(vec![Ok::<_, std::io::Error>("w".repeat(4096))]); + ([(header::CONTENT_TYPE, "application/json")], Body::from_stream(chunks)).into_response() + }), + ) + .layer(middleware::from_fn(compress)) + } + + async fn get_with(path: &str, accept_encoding: Option<&str>) -> (StatusCode, HeaderMap, Vec) { + let mut request = axum::http::Request::builder().uri(path); + if let Some(value) = accept_encoding { + request = request.header(header::ACCEPT_ENCODING, value); + } + let response = app().oneshot(request.body(Body::empty()).unwrap()).await.unwrap(); + let (parts, body) = response.into_parts(); + (parts.status, parts.headers, axum::body::to_bytes(body, usize::MAX).await.unwrap().to_vec()) + } + + #[tokio::test] + async fn large_json_answers_are_encoded_and_decode_to_the_same_bytes() { + let (_, plain_headers, plain) = get_with("/big", None).await; + assert!(!plain_headers.contains_key(header::CONTENT_ENCODING)); + let (status, headers, gzipped) = get_with("/big", Some("br, gzip")).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(headers[header::CONTENT_ENCODING], "gzip"); + assert_eq!(headers[header::VARY], "Accept-Encoding"); + assert!(gzipped.len() < plain.len() / 10); + assert_eq!(&gzipped[..2], &[0x1f, 0x8b]); + let crc = u32::from_le_bytes(gzipped[gzipped.len() - 8..gzipped.len() - 4].try_into().unwrap()); + let length = u32::from_le_bytes(gzipped[gzipped.len() - 4..].try_into().unwrap()); + let inflated = miniz_oxide::inflate::decompress_to_vec(&gzipped[10..gzipped.len() - 8]).unwrap(); + assert_eq!(inflated, plain); + assert_eq!(length as usize, plain.len()); + assert_eq!(crc, crc::Crc::::new(&crc::CRC_32_ISO_HDLC).checksum(&plain)); + + let (_, headers, deflated) = get_with("/big", Some("deflate")).await; + assert_eq!(headers[header::CONTENT_ENCODING], "deflate"); + assert_eq!(miniz_oxide::inflate::decompress_to_vec(&deflated).unwrap(), plain); + } + + #[tokio::test] + async fn small_non_json_encoded_and_streamed_answers_pass_through() { + for path in ["/small", "/text", "/encoded", "/streamed"] { + let (_, plain_headers, plain) = get_with(path, None).await; + let (_, headers, body) = get_with(path, Some("gzip")).await; + assert_eq!(body, plain, "{path}"); + assert_eq!(headers.get(header::CONTENT_ENCODING), plain_headers.get(header::CONTENT_ENCODING), "{path}"); + assert!(!headers.contains_key(header::VARY), "{path}"); + } + } +} diff --git a/rust/tests/execution/activity_results.rs b/rust/tests/execution/activity_results.rs index 86a50a276..231479919 100644 --- a/rust/tests/execution/activity_results.rs +++ b/rust/tests/execution/activity_results.rs @@ -125,7 +125,7 @@ async fn activity_result_read_preserves_encoded_values_and_closed_parent_history // capability Waterline waits on is advertised. let info = request(&router(reopened.clone()), "GET", "/api/cluster/info", Value::Null).await.1; assert_eq!(info["capabilities"]["workflow_activity_result_inspection"], json!(true), "{info}"); - assert_eq!(info["capabilities"]["response_compression"], json!([]), "{info}"); + assert_eq!(info["capabilities"]["response_compression"], json!(["gzip", "deflate"]), "{info}"); assert_eq!(info["structural_limits"]["pending_signal_count"], json!(5000), "{info}"); assert_eq!(info["discovery"]["task_repair_path"], json!("/api/system/repair"), "{info}"); assert!(info["native_unqualified"]["topology"].is_string(), "{info}"); diff --git a/tests/Fixtures/ServerParity/cluster-capabilities.json b/tests/Fixtures/ServerParity/cluster-capabilities.json index 018866b0c..71b43836c 100644 --- a/tests/Fixtures/ServerParity/cluster-capabilities.json +++ b/tests/Fixtures/ServerParity/cluster-capabilities.json @@ -295,10 +295,6 @@ "value": false, "reason": "Native does not publish this manifest document, so it does not advertise the contract." }, - "capabilities.response_compression": { - "value": [], - "reason": "Native sends uncompressed responses." - }, "structural_limits.external_payload_transaction_bytes": { "value": "{absent}", "reason": "Native reads no DW_V2_LIMIT_EXTERNAL_PAYLOAD_TRANSACTION_BYTES." diff --git a/tests/Unit/ServerParityClusterCapabilitiesContractTest.mjs b/tests/Unit/ServerParityClusterCapabilitiesContractTest.mjs index b3e33a591..2d5893bb4 100644 --- a/tests/Unit/ServerParityClusterCapabilitiesContractTest.mjs +++ b/tests/Unit/ServerParityClusterCapabilitiesContractTest.mjs @@ -54,7 +54,7 @@ for (const [name, mutate] of [ ['native without build retirement', state => { delete state.capabilities.worker_build_retirement; }], ['native declaring build retirement unavailable', state => { state.capabilities.worker_build_retirement = false; }], ['native copying a manifest flag it does not publish', state => { state.capabilities.timer_runtime_contract = true; }], - ['native advertising response compression', state => { state.capabilities.response_compression = ['gzip']; }], + ['native advertising no response compression', state => { state.capabilities.response_compression = []; }], ['native without its unqualified list', state => { state.native_unqualified = []; }], ['an activity result read that differs', state => { state.activity_result.available.response.available = false; }], ]) test(`the contract rejects ${name}`, () => { From feea2571ed34c719b3c6979e99f08f5e937ea07e Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 20 Nov 2026 01:42:54 +0000 Subject: [PATCH 2/6] Compress large answers off the async workers; zlib-wrapped deflate Review fixes: - Bodies above 256 KB are encoded on a blocking thread (spawn_blocking). At roughly 30 MB/s for level 6, an 8 MB body would otherwise hold an async worker for about 300 ms on a 2-thread runtime. - HTTP's deflate coding is the zlib format (RFC 1950), not raw DEFLATE; deflate answers are now zlib-wrapped. - Tests: a large answer encodes on the blocking path and round-trips for gzip and deflate, the zlib header is checked, and an ignored benchmark reports encode cost by size. Co-Authored-By: Claude Opus 5.5 --- rust/src/runtime/response_compression.rs | 67 +++++++++++++++++++++--- 1 file changed, 61 insertions(+), 6 deletions(-) diff --git a/rust/src/runtime/response_compression.rs b/rust/src/runtime/response_compression.rs index 368d71b8d..07dc5bfbd 100644 --- a/rust/src/runtime/response_compression.rs +++ b/rust/src/runtime/response_compression.rs @@ -5,8 +5,11 @@ //! As in PHP, gzip wins over deflate at equal weight, `*` stands for both, //! `q=0` refuses an encoding, an answer that already has a //! `Content-Encoding` is left alone, and `Vary: Accept-Encoding` is added. -//! Only answers whose whole body is already in memory (an exact size hint) -//! are encoded; streamed bodies pass through unchanged. +//! `deflate` is the zlib format (RFC 1950) that HTTP's `deflate` coding names, +//! not raw DEFLATE. Only answers whose whole body is already in memory (an +//! exact size hint) are encoded; streamed bodies pass through unchanged. +//! Bodies above [`INLINE_MAX_BYTES`] are encoded on a blocking thread, so a +//! large answer never stalls the runtime's async workers. use axum::{ body::{Body, HttpBody as _}, extract::Request, @@ -20,6 +23,9 @@ use std::sync::OnceLock; const MIN_COMPRESS_BYTES: u64 = 1024; /// The largest in-memory answer that is encoded; bigger ones pass through. const MAX_COMPRESS_BYTES: u64 = 64 * 1024 * 1024; +/// The largest body encoded on an async worker (about 8 ms at ~30 MB/s, level +/// 6); bigger bodies move to a blocking thread. +const INLINE_MAX_BYTES: usize = 256 * 1024; /// zlib's default level, which PHP's `gzencode` and `gzdeflate` use. const LEVEL: u8 = 6; @@ -112,9 +118,15 @@ async fn encode(response: Response, encoding: &'static str) -> Response { // An exact-sized in-memory body cannot fail to collect. return Response::from_parts(parts, Body::empty()); }; - let encoded = match encoding { - "gzip" => gzip(&bytes), - _ => miniz_oxide::deflate::compress_to_vec(&bytes, LEVEL), + let encoded = if bytes.len() <= INLINE_MAX_BYTES { + encode_bytes(&bytes, encoding) + } else { + let source = bytes.clone(); + match tokio::task::spawn_blocking(move || encode_bytes(&source, encoding)).await { + Ok(encoded) => encoded, + // A failed encoder thread leaves the answer as it was. + Err(_) => return Response::from_parts(parts, Body::from(bytes)), + } }; parts.headers.remove(header::CONTENT_LENGTH); parts.headers.insert(header::CONTENT_ENCODING, HeaderValue::from_static(encoding)); @@ -130,6 +142,13 @@ async fn encode(response: Response, encoding: &'static str) -> Response { Response::from_parts(parts, Body::from(encoded)) } +fn encode_bytes(bytes: &[u8], encoding: &str) -> Vec { + match encoding { + "gzip" => gzip(bytes), + _ => miniz_oxide::deflate::compress_to_vec_zlib(bytes, LEVEL), + } +} + /// `gzencode`: an RFC 1952 member around one raw DEFLATE stream. pub(super) fn gzip(data: &[u8]) -> Vec { const CRC32: crc::Crc = crc::Crc::::new(&crc::CRC_32_ISO_HDLC); @@ -190,6 +209,11 @@ mod tests { Router::new() .route("/big", get(|| async { axum::Json(serde_json::json!({"blob": "x".repeat(4096)})) })) .route("/small", get(|| async { axum::Json(serde_json::json!({"ok": true})) })) + .route("/large", get(|| async { + // Above INLINE_MAX_BYTES: encoded on a blocking thread. + let events: Vec = (0..40_000).map(|n| format!("event-{n:08}-{}", "y".repeat(16))).collect(); + axum::Json(serde_json::json!({"events": events})) + })) .route("/text", get(|| async { "y".repeat(4096) })) .route( "/encoded", @@ -237,7 +261,38 @@ mod tests { let (_, headers, deflated) = get_with("/big", Some("deflate")).await; assert_eq!(headers[header::CONTENT_ENCODING], "deflate"); - assert_eq!(miniz_oxide::inflate::decompress_to_vec(&deflated).unwrap(), plain); + // HTTP `deflate` is the zlib format: a CMF/FLG header, then DEFLATE. + assert_eq!(deflated[0] & 0x0f, 8); + assert_eq!((u16::from(deflated[0]) << 8 | u16::from(deflated[1])) % 31, 0); + assert_eq!(miniz_oxide::inflate::decompress_to_vec_zlib(&deflated).unwrap(), plain); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_large_answer_is_encoded_off_the_async_workers_and_decodes_to_the_same_bytes() { + let (_, _, plain) = get_with("/large", None).await; + assert!(plain.len() > INLINE_MAX_BYTES, "{}", plain.len()); + let (_, headers, gzipped) = get_with("/large", Some("gzip")).await; + assert_eq!(headers[header::CONTENT_ENCODING], "gzip"); + let inflated = miniz_oxide::inflate::decompress_to_vec(&gzipped[10..gzipped.len() - 8]).unwrap(); + assert_eq!(inflated, plain); + let (_, headers, deflated) = get_with("/large", Some("deflate")).await; + assert_eq!(headers[header::CONTENT_ENCODING], "deflate"); + assert_eq!(miniz_oxide::inflate::decompress_to_vec_zlib(&deflated).unwrap(), plain); + } + + /// Level-6 cost by answer size, which sizes INLINE_MAX_BYTES: + /// `cargo test --release --lib response_compression::tests::encode_cost -- --ignored --nocapture`. + #[test] + #[ignore = "benchmark"] + fn encode_cost() { + for size in [17 * 1024, 256 * 1024, 8 * 1024 * 1024] { + let body = serde_json::to_vec(&(0..size / 24).map(|n| format!("event-{n:08}-{}", n % 97)).collect::>()).unwrap(); + let started = std::time::Instant::now(); + let encoded = gzip(&body); + let elapsed = started.elapsed(); + println!("{} -> {} bytes in {:?} ({:.0} MB/s)", body.len(), encoded.len(), elapsed, + body.len() as f64 / elapsed.as_secs_f64() / 1e6); + } } #[tokio::test] From efb0c0062393f4b8f4f4d6ec927d8f349697b44c Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 20 Nov 2026 01:56:33 +0000 Subject: [PATCH 3/6] Record the measured inline encode cost encode_cost (release, lab): 14 KB in 0.6 ms, 217 KB in 10 ms, 7 MB in 196 ms (22-36 MB/s). Co-Authored-By: Claude Opus 5.5 --- rust/src/runtime/response_compression.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/src/runtime/response_compression.rs b/rust/src/runtime/response_compression.rs index 07dc5bfbd..22ee8c990 100644 --- a/rust/src/runtime/response_compression.rs +++ b/rust/src/runtime/response_compression.rs @@ -23,8 +23,8 @@ use std::sync::OnceLock; const MIN_COMPRESS_BYTES: u64 = 1024; /// The largest in-memory answer that is encoded; bigger ones pass through. const MAX_COMPRESS_BYTES: u64 = 64 * 1024 * 1024; -/// The largest body encoded on an async worker (about 8 ms at ~30 MB/s, level -/// 6); bigger bodies move to a blocking thread. +/// The largest body encoded on an async worker (measured about 10 ms at level +/// 6, 22-36 MB/s; see `encode_cost`); bigger bodies move to a blocking thread. const INLINE_MAX_BYTES: usize = 256 * 1024; /// zlib's default level, which PHP's `gzencode` and `gzdeflate` use. const LEVEL: u8 = 6; From 766c15207e5f089b99323eea98ef6d92ea32fe09 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 20 Nov 2026 00:54:21 +0000 Subject: [PATCH 4/6] Native: opt-in digest in place of repeated worker server_capabilities Every worker-plane JSON answer carries the ~17 KB server_capabilities object, and a completion that carries a poll carries a second copy in its answer. For DW Standard Workflow v1 that is about 18 copies (~320 KB) of the ~385 KB of responses per workflow, and it dominated a Dev cell's 4 Mbit/s egress. A worker that sends X-Durable-Workflow-Server-Capabilities-Digest opts in. When the header names the current digest, each copy of the current capabilities (top level, or inside a carried poll's answer) is replaced by the digest string; otherwise each copy is sent in full with a digest member for the worker to keep. Requests without the header are unchanged, so existing SDKs see the same bytes. Contract: docs/contracts/server-capabilities-digest.md. Co-Authored-By: Claude Opus 5.5 --- docs/contracts/server-capabilities-digest.md | 49 ++++ rust/src/runtime/capabilities_digest.rs | 222 +++++++++++++++++++ rust/src/runtime/http.rs | 9 +- rust/src/runtime/mod.rs | 1 + 4 files changed, 279 insertions(+), 2 deletions(-) create mode 100644 docs/contracts/server-capabilities-digest.md create mode 100644 rust/src/runtime/capabilities_digest.rs diff --git a/docs/contracts/server-capabilities-digest.md b/docs/contracts/server-capabilities-digest.md new file mode 100644 index 000000000..7ad6c938c --- /dev/null +++ b/docs/contracts/server-capabilities-digest.md @@ -0,0 +1,49 @@ +# Server capabilities digest (opt-in) + +Every worker-plane JSON answer carries `server_capabilities` (PHP's +`WorkerProtocol::json`, server#1001). A completion that carries a poll +(`activity_poll` / `workflow_poll`) carries a second copy inside its `answer`. +On native 3.15/3.16 the object is about 17 KB, so it is most of a worker's +response bytes: in the lab, DW Standard Workflow v1 (one start, one activity, +one completion) moved about 385 KB of responses per workflow, and about 18 of +those copies (about 320 KB) were the same capabilities. + +## Request + +A worker opts in by sending + +``` +X-Durable-Workflow-Server-Capabilities-Digest: |none +``` + +on worker-plane requests. `none` means it holds no capabilities yet. A request +without the header gets exactly the bytes it got before this change. + +## Answer + +For an opted-in request, the server compares the header with the digest of its +current capabilities. + +- **Match:** each copy of the current capabilities in the answer (the top level, + or up to two objects deep, as in a carried poll's `answer`) is replaced by the + digest string. The worker restores its held object before using the answer. +- **No match:** each copy is sent in full with one added member, `digest`, whose + value is the digest of the capabilities without that member. The worker keeps + the object (without `digest`) and sends the digest on its next request. +- Every opted-in answer also carries the current digest in the same response + header. + +The digest is `sha256:` followed by the lowercase hex SHA-256 of the +capabilities' compact JSON. A `server_capabilities` value that differs from the +current capabilities is never replaced. The capabilities can change while the +server runs (the external payload transport manifest is read on each call); +the next answer then has a different digest and is sent in full. + +A worker that receives a digest string it does not hold must treat the answer +as a protocol failure. Older servers ignore the header and always answer in +full, so a worker that sends it works against both. + +## Measured effect + +Lab, native build of main with this change and SDK support, 1 wf/s, 4 workers, +1 s task polls, origin response bytes per workflow (see the PR for the table). diff --git a/rust/src/runtime/capabilities_digest.rs b/rust/src/runtime/capabilities_digest.rs new file mode 100644 index 000000000..7d59c9c2f --- /dev/null +++ b/rust/src/runtime/capabilities_digest.rs @@ -0,0 +1,222 @@ +//! Worker answers without repeated server capabilities (opt-in). +//! +//! Every worker-plane JSON answer carries `server_capabilities` (PHP's +//! `WorkerProtocol::json`), and a completion that carries a poll carries a +//! second copy in its `answer`. The object is about 17 KB, so on a small +//! deployment it is most of a worker's response bytes. A worker that sends +//! `X-Durable-Workflow-Server-Capabilities-Digest` opts in: +//! +//! - the header names the digest of the capabilities the worker holds, or +//! `none`; +//! - when it names the current digest, each copy of the current capabilities +//! in the answer (top level, or one or two objects deep, as in a carried +//! poll's `answer`) is replaced by the digest string; +//! - otherwise each copy is sent in full with a `digest` member added, which +//! the worker keeps for its next request; +//! - every opted-in answer also has the current digest in the same header. +//! +//! A request without the header gets exactly the bytes it got before. The +//! digest is `sha256:` and the lowercase hex SHA-256 of the capabilities' +//! compact JSON. A copy that differs from the current capabilities is never +//! replaced. +use axum::{ + body::{Body, HttpBody as _}, + extract::Request, + http::{HeaderValue, header}, + middleware::Next, + response::Response, +}; +use serde_json::{json, value::RawValue}; +use sha2::{Digest, Sha256}; + +pub(super) const HEADER: &str = "x-durable-workflow-server-capabilities-digest"; +/// The largest answer that is rewritten; bigger ones pass through. +const MAX_BYTES: u64 = 64 * 1024 * 1024; +/// Top level, a carried poll, and its `answer`. +const MAX_DEPTH: usize = 3; + +pub(super) fn digest(text: &str) -> String { + let hash = Sha256::digest(text.as_bytes()); + let mut digest = String::with_capacity(7 + 64); + digest.push_str("sha256:"); + for byte in hash { + digest.push_str(&format!("{byte:02x}")); + } + digest +} + +pub(super) async fn elide(request: Request, next: Next) -> Response { + let path = request.uri().path(); + let worker = path == "/api/worker" || path.starts_with("/api/worker/"); + let held = worker + .then(|| request.headers().get(HEADER).and_then(|value| value.to_str().ok()).map(|value| value.trim().to_owned())) + .flatten(); + let response = next.run(request).await; + match held { + Some(held) => rewrite(response, &held).await, + None => response, + } +} + +async fn rewrite(response: Response, held: &str) -> Response { + let current = super::store::capabilities().to_string(); + let current_digest = digest(¤t); + let json = response + .headers() + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.starts_with("application/json")); + let size = response.body().size_hint().exact(); + let (mut parts, body) = response.into_parts(); + if let Ok(value) = HeaderValue::from_str(¤t_digest) { + parts.headers.insert(HEADER, value); + } + if !json || parts.headers.contains_key(header::CONTENT_ENCODING) || !size.is_some_and(|size| size <= MAX_BYTES) { + return Response::from_parts(parts, body); + } + let Ok(bytes) = axum::body::to_bytes(body, MAX_BYTES as usize).await else { + // An exact-sized in-memory body cannot fail to collect. + return Response::from_parts(parts, Body::empty()); + }; + let replacement = if held == current_digest { + json!(current_digest).to_string() + } else { + // `{...}` plus the digest member before the closing brace. + let mut tagged = current[..current.len() - 1].to_owned(); + if tagged.len() > 1 { + tagged.push(','); + } + tagged.push_str(&format!("\"digest\":{}}}", json!(current_digest))); + tagged + }; + match std::str::from_utf8(&bytes).ok().and_then(|text| replace(text, ¤t, &replacement, 1)) { + Some(rewritten) => { + parts.headers.remove(header::CONTENT_LENGTH); + Response::from_parts(parts, Body::from(rewritten)) + } + None => Response::from_parts(parts, Body::from(bytes)), + } +} + +/// The object with each `server_capabilities` member whose raw value is +/// `current` replaced, or `None` when nothing changed or it is not an object. +/// Members are rewritten from their raw text, so only whitespace between +/// them changes. +fn replace(text: &str, current: &str, replacement: &str, depth: usize) -> Option { + if !text.trim_start().starts_with('{') { + return None; + } + let super::http::ObjectMembers(members) = serde_json::from_str(text).ok()?; + let mut changed = false; + let rendered: Vec = members + .iter() + .map(|(key, raw): &(String, &RawValue)| { + let raw = raw.get(); + let value = if key == "server_capabilities" && raw == current { + changed = true; + replacement.to_owned() + } else if depth < MAX_DEPTH { + match replace(raw, current, replacement, depth + 1) { + Some(nested) => { + changed = true; + nested + } + None => raw.to_owned(), + } + } else { + raw.to_owned() + }; + format!("{}:{value}", json!(key)) + }) + .collect(); + changed.then(|| format!("{{{}}}", rendered.join(","))) +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::{Router, middleware, routing::post}; + use serde_json::Value; + use tower::ServiceExt as _; + + fn caps() -> Value { + super::super::store::capabilities() + } + + fn app() -> Router { + let carried = || async { + axum::Json(json!({"outcome": "completed", "server_capabilities": caps(), + "activity_poll": {"leased": true, "answer": {"task": {"id": 1}, "poll_status": "leased", "server_capabilities": caps()}}, + "protocol_version": "1.20"})) + }; + Router::new() + .route("/api/worker/activity-tasks/poll", post(|| async { axum::Json(json!({"task": null, "server_capabilities": caps()})) })) + .route("/api/worker/workflow-tasks/t/complete", post(carried)) + .route("/api/worker/other", post(|| async { axum::Json(json!({"server_capabilities": {"different": true}})) })) + .route("/api/workflows", post(|| async { axum::Json(json!({"server_capabilities": caps()})) })) + .layer(middleware::from_fn(elide)) + } + + async fn call(path: &str, held: Option<&str>) -> (Option, Value, Vec) { + let mut request = axum::http::Request::builder().method("POST").uri(path); + if let Some(held) = held { + request = request.header(HEADER, held); + } + let response = app().oneshot(request.body(Body::empty()).unwrap()).await.unwrap(); + let header = response.headers().get(HEADER).map(|value| value.to_str().unwrap().to_owned()); + let bytes = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap().to_vec(); + (header, serde_json::from_slice(&bytes).unwrap(), bytes) + } + + #[tokio::test] + async fn answers_without_the_header_are_unchanged() { + for path in ["/api/worker/activity-tasks/poll", "/api/worker/workflow-tasks/t/complete"] { + let (header, body, _) = call(path, None).await; + assert_eq!(header, None); + assert_eq!(body["server_capabilities"], caps()); + } + let (_, body, _) = call("/api/worker/workflow-tasks/t/complete", None).await; + assert_eq!(body["activity_poll"]["answer"]["server_capabilities"], caps()); + } + + #[tokio::test] + async fn a_worker_without_the_digest_gets_tagged_capabilities() { + let current = digest(&caps().to_string()); + for held in ["none", "sha256:0000"] { + let (header, body, _) = call("/api/worker/workflow-tasks/t/complete", Some(held)).await; + assert_eq!(header.as_deref(), Some(current.as_str())); + let mut tagged = caps(); + tagged["digest"] = json!(current); + assert_eq!(body["server_capabilities"], tagged); + assert_eq!(body["activity_poll"]["answer"]["server_capabilities"], tagged); + // The digest covers the capabilities without the digest member. + let mut untagged = body["server_capabilities"].clone(); + untagged.as_object_mut().unwrap().remove("digest"); + assert_eq!(digest(&untagged.to_string()), current); + } + } + + #[tokio::test] + async fn a_worker_with_the_current_digest_gets_the_digest_in_place_of_each_copy() { + let current = digest(&caps().to_string()); + let (header, body, bytes) = call("/api/worker/workflow-tasks/t/complete", Some(¤t)).await; + assert_eq!(header.as_deref(), Some(current.as_str())); + assert_eq!(body["server_capabilities"], json!(current)); + assert_eq!(body["activity_poll"]["answer"]["server_capabilities"], json!(current)); + assert_eq!(body["activity_poll"]["answer"]["task"], json!({"id": 1})); + assert_eq!(body["outcome"], json!("completed")); + assert!(bytes.len() < 1024, "{}", bytes.len()); + let (_, poll, _) = call("/api/worker/activity-tasks/poll", Some(¤t)).await; + assert_eq!(poll, json!({"task": null, "server_capabilities": current})); + } + + #[tokio::test] + async fn other_values_and_the_control_plane_are_never_replaced() { + let current = digest(&caps().to_string()); + let (_, body, _) = call("/api/worker/other", Some(¤t)).await; + assert_eq!(body["server_capabilities"], json!({"different": true})); + let (header, body, _) = call("/api/workflows", Some(¤t)).await; + assert_eq!(header, None); + assert_eq!(body["server_capabilities"], caps()); + } +} diff --git a/rust/src/runtime/http.rs b/rust/src/runtime/http.rs index 2f8d7d414..88fa0d450 100644 --- a/rust/src/runtime/http.rs +++ b/rust/src/runtime/http.rs @@ -425,7 +425,12 @@ pub fn router(runtime: Runtime) -> Router { .layer(middleware::from_fn_with_state(runtime.clone(), storage_admission_guard)) .layer(middleware::from_fn_with_state(runtime.clone(), guard)) .layer(middleware::from_fn(super::payload_limits::enforce)) - // PHP's CompressResponse wraps every answer, refusals included. + // After the worker envelope is complete (server#1001), and inside + // compression: the digest layer reads the plain JSON body, and + // compression must encode what it returns. + .layer(middleware::from_fn(super::capabilities_digest::elide)) + // PHP's CompressResponse wraps every answer, refusals included. It + // must stay outermost. .layer(middleware::from_fn(super::response_compression::compress)) .with_state(runtime) } @@ -841,7 +846,7 @@ fn complete_worker_envelope(bytes: &[u8]) -> Option> { } /// A JSON object's members in order, each value kept as its raw text. -struct ObjectMembers<'a>(Vec<(String, &'a serde_json::value::RawValue)>); +pub(super) struct ObjectMembers<'a>(pub(super) Vec<(String, &'a serde_json::value::RawValue)>); impl<'de: 'a, 'a> serde::Deserialize<'de> for ObjectMembers<'a> { fn deserialize>(deserializer: D) -> std::result::Result { diff --git a/rust/src/runtime/mod.rs b/rust/src/runtime/mod.rs index 8ea36a32a..dd7cf1e76 100644 --- a/rust/src/runtime/mod.rs +++ b/rust/src/runtime/mod.rs @@ -40,6 +40,7 @@ mod bridge_adapters; mod bundle_verifier; mod cancellation; mod cancellation_cascade; +mod capabilities_digest; mod cancellation_scopes; mod json_order; mod scope_checkpoint; From 93be253628e2fc74e3ce1b35dbe5a9cfe69630f6 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 20 Nov 2026 01:44:34 +0000 Subject: [PATCH 5/6] Digest: per-request resolution rule, compression-outermost test, docs Review fixes: - The contract states that the server replaces copies only with the digest that request sent, so a worker resolves a digest string against the digest and object it captured when sending that request. A worker that kept only its latest digest could reject a committed answer during a rolling deploy. Version names and PR links are removed from the contract prose. - A router-level test sends Accept-Encoding: gzip and the digest header and asserts the digest is applied inside a gzip-encoded body (compression is the outer layer). - The header is documented in server-reference.md and rust-server-port.md. Co-Authored-By: Claude Opus 5.5 --- docs/contracts/server-capabilities-digest.md | 72 +++++++++++++------- docs/rust-server-port.md | 7 +- docs/server-reference.md | 12 ++++ rust/tests/execution.rs | 49 +++++++++++++ 4 files changed, 113 insertions(+), 27 deletions(-) diff --git a/docs/contracts/server-capabilities-digest.md b/docs/contracts/server-capabilities-digest.md index 7ad6c938c..81b888a42 100644 --- a/docs/contracts/server-capabilities-digest.md +++ b/docs/contracts/server-capabilities-digest.md @@ -1,12 +1,11 @@ # Server capabilities digest (opt-in) -Every worker-plane JSON answer carries `server_capabilities` (PHP's -`WorkerProtocol::json`, server#1001). A completion that carries a poll -(`activity_poll` / `workflow_poll`) carries a second copy inside its `answer`. -On native 3.15/3.16 the object is about 17 KB, so it is most of a worker's -response bytes: in the lab, DW Standard Workflow v1 (one start, one activity, -one completion) moved about 385 KB of responses per workflow, and about 18 of -those copies (about 320 KB) were the same capabilities. +Every worker-plane JSON answer carries `server_capabilities`, as PHP's +`WorkerProtocol::json` requires. A completion that carries a poll +(`activity_poll` or `workflow_poll`) carries a second copy inside its `answer`. +The object is about 17 KB, so it is most of a worker's response bytes. In a lab +run of the standard one-activity workflow it was about 306 KB of the 385 KB of +responses per workflow. ## Request @@ -16,34 +15,55 @@ A worker opts in by sending X-Durable-Workflow-Server-Capabilities-Digest: |none ``` -on worker-plane requests. `none` means it holds no capabilities yet. A request -without the header gets exactly the bytes it got before this change. +on worker-plane requests. The value is the digest of the capabilities the +worker holds, or `none` if it holds none yet. A request without the header gets +exactly the same bytes as before. ## Answer -For an opted-in request, the server compares the header with the digest of its -current capabilities. +For an opted-in request, the server compares the request's header with the +digest of its current capabilities. -- **Match:** each copy of the current capabilities in the answer (the top level, - or up to two objects deep, as in a carried poll's `answer`) is replaced by the - digest string. The worker restores its held object before using the answer. -- **No match:** each copy is sent in full with one added member, `digest`, whose - value is the digest of the capabilities without that member. The worker keeps - the object (without `digest`) and sends the digest on its next request. +- **Match:** each copy of the current capabilities in the answer is replaced by + the digest string. That covers the top level and up to two objects deep, as in + a carried poll's `answer`. +- **No match:** each copy is sent in full with one added member, `digest`. Its + value is the digest of the capabilities without that member. - Every opted-in answer also carries the current digest in the same response header. The digest is `sha256:` followed by the lowercase hex SHA-256 of the capabilities' compact JSON. A `server_capabilities` value that differs from the -current capabilities is never replaced. The capabilities can change while the -server runs (the external payload transport manifest is read on each call); -the next answer then has a different digest and is sent in full. +current capabilities is never replaced. -A worker that receives a digest string it does not hold must treat the answer -as a protocol failure. Older servers ignore the header and always answer in -full, so a worker that sends it works against both. +## Resolving a digest string: per request -## Measured effect +The server replaces copies only with the digest **that request sent**. A digest +string in an answer therefore always names the capabilities the worker held when +it sent that request, never a value it learned later. -Lab, native build of main with this change and SDK support, 1 wf/s, 4 workers, -1 s task polls, origin response bytes per workflow (see the PR for the table). +Requests can overlap. A worker can have a completion in flight under digest +`D1` while another answer moves it to `D2`; this happens during a rolling +deploy, or when the capabilities change at run time. A worker must handle this: + +- Capture the digest and the capabilities object when it sends each request. +- Resolve a digest string in that request's answer against the captured pair, + not against whatever it holds when the answer arrives. +- Learn a newer digest only from a tagged full copy, and use it for requests + sent afterwards. + +A worker that keeps only its latest digest would reject a correct `D1` answer +that arrives after it switched to `D2`. If that answer came after the server +had already committed the request, the worker would lose a committed outcome. + +A digest string that matches neither the digest captured for that request nor a +tagged copy in the same answer is a protocol failure. + +## Compatibility and layering + +Servers that do not support the header ignore it and always answer in full, so +a worker that sends it works against both. + +On the server, the digest layer runs after the worker envelope is complete and +inside response compression. Compression then encodes the smaller body. The +digest layer never rewrites an already-encoded body. diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index 8de8d60ae..136659683 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -342,7 +342,12 @@ contract fixture before its correction. Like PHP's `CompressResponse`, native gzip- or deflate-encodes JSON answers of 1024 bytes or more when `Accept-Encoding` allows it, and advertises `response_compression: ["gzip", "deflate"]`, unless - `DW_COMPRESSION_ENABLED` is false. `native_unqualified` names each member it still + `DW_COMPRESSION_ENABLED` is false. A worker can opt out of receiving + `server_capabilities` again in every answer: it sends + `X-Durable-Workflow-Server-Capabilities-Digest` and gets the digest string in + place of copies it already holds + ([contract](contracts/server-capabilities-digest.md)). That layer runs inside + compression, so the smaller body is still encoded. `native_unqualified` names each member it still leaves out: `topology`, the diagnostics view, the external payload transaction limit and the conformance manifests. Since server#1005 (3.1.0) native serves `supported_sdk_versions` and `client_compatibility` from one diff --git a/docs/server-reference.md b/docs/server-reference.md index 79b0577c6..c55624f7f 100644 --- a/docs/server-reference.md +++ b/docs/server-reference.md @@ -1462,6 +1462,18 @@ worker capability fields. Adding new workflow-task commands or optional capability booleans is additive; removing or renaming a command or capability requires a protocol version bump. +Native Server: a worker can send +`X-Durable-Workflow-Server-Capabilities-Digest: |none` on +worker-plane requests to stop receiving the same `server_capabilities` object +in every answer. When the header names the current digest, each copy of the +current capabilities (top level, or inside a carried poll's `answer`) is +replaced by the digest string. Otherwise each copy is sent in full with an added +`digest` member. Every opted-in answer also carries the current digest in the +same response header. A digest string always names the digest that request +sent, so a worker resolves it against the capabilities it captured when sending +that request. Requests without the header are unchanged. The full contract is in +`docs/contracts/server-capabilities-digest.md`. + Workflow task polling returns a leased task plus `workflow_task_attempt`. Clients must echo both `workflow_task_attempt` and `lease_owner` on workflow-task `heartbeat`, `complete`, and `fail` calls. Workflow-task completion supports diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index e6722c760..114f57fca 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -5441,3 +5441,52 @@ async fn sdk_capability_manifest(request: Request, next: axum::middleware: }; next.run(Request::from_parts(parts, Body::from(bytes))).await } + +/// Compression wraps the capabilities digest layer: a worker that sends both +/// `Accept-Encoding: gzip` and the digest gets the digest in place of the +/// capabilities, and that smaller body gzip-encoded. +#[tokio::test] +async fn the_capabilities_digest_is_applied_inside_gzip_compression() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + register(&app, "worker", json!(["echo"]), json!(["echo"])).await; + let poll = |digest: String| { + let app = app.clone(); + async move { + let request = Request::builder() + .method("POST") + .uri("/api/worker/activity-tasks/poll") + .header("authorization", "Bearer test-token") + .header("content-type", "application/json") + .header("x-namespace", "default") + .header("x-durable-workflow-protocol-version", "1.19") + .header("accept-encoding", "gzip") + .header("x-durable-workflow-server-capabilities-digest", digest) + .body(Body::from(json!({"worker_id":"worker","task_queue":"test","timeout_seconds":0}).to_string())) + .unwrap(); + let response = app.oneshot(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let headers = response.headers().clone(); + let encoded = to_bytes(response.into_body(), 4 * 1024 * 1024).await.unwrap().to_vec(); + (headers, encoded) + } + }; + let gunzip = |encoded: &[u8]| -> Value { + assert_eq!(&encoded[..2], &[0x1f, 0x8b], "gzip member"); + serde_json::from_slice(&miniz_oxide::inflate::decompress_to_vec(&encoded[10..encoded.len() - 8]).unwrap()).unwrap() + }; + + let (headers, first) = poll("none".into()).await; + assert_eq!(headers["content-encoding"], "gzip"); + let digest = headers["x-durable-workflow-server-capabilities-digest"].to_str().unwrap().to_owned(); + let tagged = gunzip(&first); + assert_eq!(tagged["server_capabilities"]["digest"], json!(digest), "{tagged}"); + assert_eq!(tagged["server_capabilities"]["activity_lease_renewal"], json!(true)); + + let (headers, second) = poll(digest.clone()).await; + assert_eq!(headers["content-encoding"], "gzip"); + let elided = gunzip(&second); + assert_eq!(elided["server_capabilities"], json!(digest), "{elided}"); + assert_eq!(elided["poll_status"], json!("empty")); + assert!(second.len() * 4 < first.len(), "{} vs {}", second.len(), first.len()); +} From 87d41b5888bd49ce9706bffe0d5ed836fd083f1b Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 20 Nov 2026 02:18:40 +0000 Subject: [PATCH 6/6] Router test: assert digest and gzip on one leased answer The elided empty poll is under the 1024-byte compression threshold, so the second answer is now a leased workflow task: still over the threshold with its capabilities elided, so both layers apply to it. Co-Authored-By: Claude Opus 5.5 --- rust/tests/execution.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index 114f57fca..cf3f04551 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -5450,12 +5450,12 @@ async fn the_capabilities_digest_is_applied_inside_gzip_compression() { let database = TestDatabase::new().await; let app = router(database.open().await.unwrap()); register(&app, "worker", json!(["echo"]), json!(["echo"])).await; - let poll = |digest: String| { + let poll = |kind: &'static str, digest: String| { let app = app.clone(); async move { let request = Request::builder() .method("POST") - .uri("/api/worker/activity-tasks/poll") + .uri(format!("/api/worker/{kind}-tasks/poll")) .header("authorization", "Bearer test-token") .header("content-type", "application/json") .header("x-namespace", "default") @@ -5476,17 +5476,22 @@ async fn the_capabilities_digest_is_applied_inside_gzip_compression() { serde_json::from_slice(&miniz_oxide::inflate::decompress_to_vec(&encoded[10..encoded.len() - 8]).unwrap()).unwrap() }; - let (headers, first) = poll("none".into()).await; + let (headers, first) = poll("activity", "none".into()).await; assert_eq!(headers["content-encoding"], "gzip"); let digest = headers["x-durable-workflow-server-capabilities-digest"].to_str().unwrap().to_owned(); let tagged = gunzip(&first); assert_eq!(tagged["server_capabilities"]["digest"], json!(digest), "{tagged}"); assert_eq!(tagged["server_capabilities"]["activity_lease_renewal"], json!(true)); - let (headers, second) = poll(digest.clone()).await; + // A leased workflow task stays over the 1024-byte compression threshold + // once its capabilities are elided, so both layers apply to one answer. + start(&app, "digest-and-gzip").await; + let (headers, second) = poll("workflow", digest.clone()).await; assert_eq!(headers["content-encoding"], "gzip"); let elided = gunzip(&second); assert_eq!(elided["server_capabilities"], json!(digest), "{elided}"); - assert_eq!(elided["poll_status"], json!("empty")); - assert!(second.len() * 4 < first.len(), "{} vs {}", second.len(), first.len()); + assert_eq!(elided["poll_status"], json!("leased"), "{elided}"); + assert_eq!(elided["task"]["workflow_id"], json!("digest-and-gzip")); + assert!(!String::from_utf8_lossy(&miniz_oxide::inflate::decompress_to_vec(&second[10..second.len() - 8]).unwrap()) + .contains("supported_workflow_task_commands")); }