diff --git a/Cargo.lock b/Cargo.lock index 03ab7f4c..92be819c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4148,7 +4148,7 @@ dependencies = [ [[package]] name = "tw-api" -version = "0.98.0" +version = "0.98.1" dependencies = [ "serde", "serde_json", @@ -4159,7 +4159,7 @@ dependencies = [ [[package]] name = "tw-bedrock" -version = "0.98.0" +version = "0.98.1" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -4180,7 +4180,7 @@ dependencies = [ [[package]] name = "tw-breaker" -version = "0.98.0" +version = "0.98.1" dependencies = [ "serde", "serde_json", @@ -4188,7 +4188,7 @@ dependencies = [ [[package]] name = "tw-config" -version = "0.98.0" +version = "0.98.1" dependencies = [ "blake3", "icu_locale_core", @@ -4216,7 +4216,7 @@ dependencies = [ [[package]] name = "tw-control" -version = "0.98.0" +version = "0.98.1" dependencies = [ "axum", "base64 0.22.1", @@ -4262,7 +4262,7 @@ dependencies = [ [[package]] name = "tw-dialect" -version = "0.98.0" +version = "0.98.1" dependencies = [ "memchr", "serde", @@ -4271,7 +4271,7 @@ dependencies = [ [[package]] name = "tw-engine" -version = "0.98.0" +version = "0.98.1" dependencies = [ "chrono", "serde", @@ -4285,7 +4285,7 @@ dependencies = [ [[package]] name = "tw-gateway" -version = "0.98.0" +version = "0.98.1" dependencies = [ "arc-swap", "async-stream", @@ -4340,7 +4340,7 @@ dependencies = [ [[package]] name = "tw-guard" -version = "0.98.0" +version = "0.98.1" dependencies = [ "aho-corasick", "base64 0.22.1", @@ -4361,7 +4361,7 @@ dependencies = [ [[package]] name = "tw-handover" -version = "0.98.0" +version = "0.98.1" dependencies = [ "libc", "serde", @@ -4374,7 +4374,7 @@ dependencies = [ [[package]] name = "tw-link" -version = "0.98.0" +version = "0.98.1" dependencies = [ "serde", "serde_json", @@ -4388,7 +4388,7 @@ dependencies = [ [[package]] name = "tw-observe" -version = "0.98.0" +version = "0.98.1" dependencies = [ "tokio", "tracing", @@ -4397,7 +4397,7 @@ dependencies = [ [[package]] name = "tw-pack" -version = "0.98.0" +version = "0.98.1" dependencies = [ "age", "blake3", @@ -4421,7 +4421,7 @@ dependencies = [ [[package]] name = "tw-plugin" -version = "0.98.0" +version = "0.98.1" dependencies = [ "libc", "rand 0.10.2", @@ -4437,7 +4437,7 @@ dependencies = [ [[package]] name = "tw-pricing" -version = "0.98.0" +version = "0.98.1" dependencies = [ "arc-swap", "flate2", @@ -4450,14 +4450,14 @@ dependencies = [ [[package]] name = "tw-secret" -version = "0.98.0" +version = "0.98.1" dependencies = [ "thiserror 2.0.20", ] [[package]] name = "tw-store" -version = "0.98.0" +version = "0.98.1" dependencies = [ "blake3", "bytes", @@ -4477,7 +4477,7 @@ dependencies = [ [[package]] name = "tw-sync" -version = "0.98.0" +version = "0.98.1" dependencies = [ "age", "axum", @@ -4505,7 +4505,7 @@ dependencies = [ [[package]] name = "tw-types" -version = "0.98.0" +version = "0.98.1" dependencies = [ "serde", "serde_json", @@ -4514,7 +4514,7 @@ dependencies = [ [[package]] name = "tw-watch" -version = "0.98.0" +version = "0.98.1" dependencies = [ "libc", "notify", @@ -4525,7 +4525,7 @@ dependencies = [ [[package]] name = "tw-yaml" -version = "0.98.0" +version = "0.98.1" dependencies = [ "saphyr-parser", "serde_yaml_ng", @@ -4535,7 +4535,7 @@ dependencies = [ [[package]] name = "twcore" -version = "0.98.0" +version = "0.98.1" dependencies = [ "anyhow", "axum", diff --git a/Cargo.toml b/Cargo.toml index 13067cf5..1b55d1ca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,7 +61,7 @@ categories = ["network-programming", "web-programming::http-server"] # 二进制走 CalVer,crate 走 SemVer —— 两者是两回事,卖给不同的人。 # 这里是 crate 的版本。 -version = "0.98.0" +version = "0.98.1" [workspace.dependencies] # ── 内部 crate ─────────────────────────────────────────── diff --git a/crates/tw-api/msg-codes.txt b/crates/tw-api/msg-codes.txt index a8e64872..e347a970 100644 --- a/crates/tw-api/msg-codes.txt +++ b/crates/tw-api/msg-codes.txt @@ -429,6 +429,7 @@ gw.probe.timeout gw.request.aborted gw.request.bad_encoding gw.request.body_declared_too_large +gw.request.body_idle gw.request.body_over_limit gw.request.body_unreadable gw.request.decoded_body_over_limit diff --git a/crates/tw-gateway/src/resources.rs b/crates/tw-gateway/src/resources.rs index d9ec5234..602c778c 100644 --- a/crates/tw-gateway/src/resources.rs +++ b/crates/tw-gateway/src/resources.rs @@ -1,8 +1,9 @@ //! 进程级的自动资源调度。配置重载、多个运行时共用同一份预算。 //! //! CPU 重任务先异步等名额,再交给阻塞线程;等候者不会占满 Tokio 的线程。 -//! 正文按字节占预算,并为解析、复制留三倍余量。预算不足时先退避重试,避免已经 -//! 持有一半正文的请求互相等内存。名额跟着 Bytes 的最后一份引用归还。 +//! 正文按字节占预算,并为解析、复制留三倍余量。读取前按顺序等预算,避免小请求 +//! 插队让大请求一直重试。未知长度的正文同一时刻只读一份,避免拿着半份正文互等。 +//! 名额跟着 Bytes 的最后一份引用归还。 //! //! 单个合法的大正文可以独占预算:自动预算不改变 256 MiB 的单请求上限。这是 //! 正文缓冲的预算,不是对整个进程 RSS 的硬限制(插件、JSON 对象等另有开销)。 @@ -11,7 +12,7 @@ use std::sync::{Arc, Mutex, OnceLock}; use std::time::Duration; use bytes::Bytes; -use tokio::sync::{OwnedSemaphorePermit, Semaphore}; +use tokio::sync::{Mutex as AsyncMutex, MutexGuard, Notify, OwnedSemaphorePermit, Semaphore}; #[cfg(any(target_os = "linux", test))] mod cgroup; @@ -20,6 +21,9 @@ const MIB: usize = 1024 * 1024; const BODY_HEADROOM: usize = 3; const HEAVY_BODY: usize = 1024 * 1024; +/// 等预算或准入顺序的上限;一直被占用时仍向客户端返回可重试的 429。 +pub(crate) const BODY_WAIT: Duration = Duration::from_secs(30); + /// 突发负载过去后回收阻塞线程;不改变常驻 I/O worker。 pub const CPU_KEEP_ALIVE: Duration = Duration::from_secs(10); @@ -131,6 +135,8 @@ pub(crate) fn body_budget() -> &'static Arc { pub(crate) struct Budget { limit: usize, used: Mutex, + intake: AsyncMutex<()>, + released: Notify, } impl Budget { @@ -138,9 +144,29 @@ impl Budget { Arc::new(Self { limit, used: Mutex::new(0), + intake: AsyncMutex::new(()), + released: Notify::new(), }) } + /// Tokio 的 mutex 按调用顺序放行。拿到名额之前不读正文,也不占正文预算。 + pub(crate) async fn admit(&self) -> Result, crate::GatewayError> { + tokio::time::timeout(BODY_WAIT, self.intake.lock()) + .await + .map_err(|_| self.busy()) + } + + fn busy(&self) -> crate::GatewayError { + let used = *self.used.lock().unwrap_or_else(|e| e.into_inner()); + tracing::warn!( + code = "gw.request.memory_busy", + budget_bytes = self.limit, + used_bytes = used, + "request body budget wait timed out" + ); + memory_busy() + } + pub(crate) fn reserve(self: &Arc) -> Reservation { Reservation { budget: self.clone(), @@ -155,6 +181,29 @@ pub(crate) struct Reservation { } impl Reservation { + /// 只能由持有 intake 名额的读取者调用;其他请求不能在等候期间插队占预算。 + pub(crate) async fn wait(&mut self, bytes: usize) -> Result<(), crate::GatewayError> { + let budget = self.budget.clone(); + let deadline = tokio::time::Instant::now() + BODY_WAIT; + loop { + let released = budget.released.notified(); + tokio::pin!(released); + // 先登记,再检查:释放可能发生在检查和 await 之间。 + released.as_mut().enable(); + if self.grow(bytes) { + return Ok(()); + } + tokio::time::timeout_at(deadline, released) + .await + .map_err(|_| budget.busy())?; + } + } + + /// 解压需要更多空间时,先释放 CPU 名额,再等到可以独占预算后重试一次。 + pub(crate) async fn exclusive(&mut self) -> Result<(), crate::GatewayError> { + self.wait(self.budget.limit).await + } + /// 在追加正文或解压输出之前调用。失败不会改变已经占用的预算。 pub(crate) fn grow(&mut self, bytes: usize) -> bool { let charge = bytes.saturating_mul(BODY_HEADROOM).min(self.budget.limit); @@ -174,12 +223,25 @@ impl Reservation { _reservation: self, }) } + + pub(crate) fn shrink(&mut self, bytes: usize) { + let charge = bytes.saturating_mul(BODY_HEADROOM).min(self.budget.limit); + if charge < self.charged { + let mut used = self.budget.used.lock().unwrap_or_else(|e| e.into_inner()); + *used -= self.charged - charge; + self.charged = charge; + drop(used); + self.budget.released.notify_waiters(); + } + } } impl Drop for Reservation { fn drop(&mut self) { let mut used = self.budget.used.lock().unwrap_or_else(|e| e.into_inner()); *used -= self.charged; + drop(used); + self.budget.released.notify_waiters(); } } diff --git a/crates/tw-gateway/src/server/intake.rs b/crates/tw-gateway/src/server/intake.rs index 03da9957..9e8b6aa0 100644 --- a/crates/tw-gateway/src/server/intake.rs +++ b/crates/tw-gateway/src/server/intake.rs @@ -10,7 +10,7 @@ //! 没压缩时一个样;`Content-Encoding` 头一并摘掉,上游不会看到它。上限按**解开之后**的大小算: //! 一小段压缩数据能解出几百兆,声明的长度和压缩的长度都不作数。 -use axum::body::Body; +use axum::body::{Body, HttpBody}; use axum::http::{HeaderMap, header}; use bytes::{Bytes, BytesMut}; use http_body_util::BodyExt; @@ -56,9 +56,25 @@ async fn read_with_budget( } // 认不得的编码在读体之前就拒:不用先收完几百兆再说不认识 let coding = coding_of(headers)?; + // 已知上界的未压缩正文先留足预算,再并行读取。看 body 自己的上界,不信 + // 客户端随手写的 Content-Length;HTTP 传输层负责兑现它的 size_hint。 + let bounded = coding.is_none().then(|| body.size_hint().upper()).flatten(); + let mut admission = Some(budget.admit().await?); let mut reservation = budget.reserve(); + if let Some(size) = bounded { + reservation.wait(size.min(max as u64) as usize).await?; + drop(admission.take()); + } let mut raw = BytesMut::new(); - while let Some(frame) = body.frame().await { + while let Some(frame) = tokio::time::timeout(crate::resources::BODY_WAIT, body.frame()) + .await + .map_err(|_| { + GatewayError::request(msg!( + "gw.request.body_idle" => + "The request body stopped arriving for 30 seconds. Send the request again." + )) + })? + { let frame = frame.map_err(|e| { GatewayError::request(msg!( "gw.request.body_unreadable", detail = e => @@ -75,33 +91,23 @@ async fn read_with_budget( "The request body is over the {max}-byte limit." ))); } - // 先记账再拷贝。用满就让客户端退避,不拿着半份正文等别的请求释放。 - if !reservation.grow(next) { + // 只有一个未知长度的读取者会拿着半份正文等预算;其他未完成读取的请求 + // 都已经按可信上界留足了空间,完成它们不需要再等预算。 + if admission.is_some() { + reservation.wait(next).await?; + } else if !reservation.grow(next) { + // 传输层违反 size_hint 上界时仍不绕过预算,也不拿着半份正文排队。 return Err(crate::resources::memory_busy()); } raw.extend_from_slice(&data); } let raw = raw.freeze(); let Some(coding) = coding else { + reservation.shrink(raw.len()); return Ok(reservation.wrap(raw)); }; headers.remove(header::CONTENT_ENCODING); - // 解压是纯 CPU 的活,几百兆要几百毫秒:不占 async 线程 - let slot = crate::resources::heavy_slot().await; - let decoded = tokio::task::spawn_blocking(move || { - // 客户端取消时阻塞任务仍可能在跑:CPU 名额和预算都跟任务走。 - let _slot = slot; - let decoded = coding.decode_with(&raw, max, |output| { - if reservation.grow(raw.len().saturating_add(output)) { - Ok(()) - } else { - Err(Decode::Busy) - } - })?; - Ok(reservation.wrap(decoded)) - }) - .await - .unwrap_or_else(|e| Err(Decode::Failed(e.to_string()))); + let decoded = decode(coding, raw, max, reservation).await; match decoded { Ok(b) => Ok(b), Err(Decode::Busy) => Err(crate::resources::memory_busy()), @@ -116,6 +122,51 @@ async fn read_with_budget( } } +async fn decode( + coding: Coding, + mut raw: Bytes, + max: usize, + mut reservation: crate::resources::Reservation, +) -> Result { + let mut retried = false; + loop { + // 解压不占 async 线程。取消时阻塞任务仍持有自己的 CPU 名额和预算。 + let slot = crate::resources::heavy_slot().await; + let (input, held, decoded) = tokio::task::spawn_blocking(move || { + let _slot = slot; + let decoded = coding.decode_with(&raw, max, |output| { + if reservation.grow(raw.len().saturating_add(output)) { + Ok(()) + } else { + Err(Decode::Busy) + } + }); + (raw, reservation, decoded) + }) + .await + .map_err(|e| Decode::Failed(e.to_string()))?; + raw = input; + reservation = held; + match decoded { + Ok(decoded) => { + // 原始压缩字节用完了,不让它的预留跟随解开的正文继续占着。 + drop(raw); + reservation.shrink(decoded.len()); + return Ok(reservation.wrap(decoded)); + } + Err(Decode::Busy) if !retried => { + // 失败的输出已释放。拿着 intake 名额等,后来的小正文不会抢走空间。 + // 不持 CPU 名额等内存,其他请求还需要 CPU 才能结束并归还预算。 + reservation.shrink(raw.len()); + reservation.exclusive().await.map_err(|_| Decode::Busy)?; + // 独占后所有 grow 都能成功,因此至多重新解压一次。 + retried = true; + } + Err(e) => return Err(e), + } + } +} + /// 这个请求的体压没压:没压是 `Ok(None)`,压成认不得的样子是 400。 /// /// **只认 gzip、deflate、zstd**(见 [`crate::inflate`]);叠了几层的(`gzip, zstd`)也不认 @@ -154,7 +205,7 @@ mod tests { assert_eq!(&b[..], b"12345"); } - #[tokio::test] + #[tokio::test(start_paused = true)] async fn concurrent_bodies_share_a_budget_until_the_last_clone_is_dropped() { let budget = crate::resources::Budget::new(30); let body = read_with_budget(&mut HeaderMap::new(), chunked(&[b"123456"]), 100, &budget) @@ -176,7 +227,7 @@ mod tests { ); } - #[tokio::test] + #[tokio::test(start_paused = true)] async fn cancelling_a_partial_read_returns_its_budget() { use futures::StreamExt; let budget = crate::resources::Budget::new(30); @@ -280,7 +331,7 @@ mod tests { h } - #[tokio::test] + #[tokio::test(start_paused = true)] async fn decompression_reserves_output_and_releases_everything_on_failure() { let budget = crate::resources::Budget::new(3_000); let held = read_with_budget( @@ -313,6 +364,280 @@ mod tests { assert_eq!(&decoded[..], &[b'x'; 1_000]); } + #[tokio::test(start_paused = true)] + async fn a_large_waiter_gets_budget_before_later_small_bodies() { + let budget = crate::resources::Budget::new(30); + let held = read_with_budget(&mut HeaderMap::new(), Body::from("123456"), 100, &budget) + .await + .unwrap(); + let mut large_headers = HeaderMap::new(); + let mut large = Box::pin(read_with_budget( + &mut large_headers, + Body::from("123456789"), + 100, + &budget, + )); + assert!(futures::poll!(&mut large).is_pending()); + let mut small_headers = HeaderMap::new(); + let mut small = Box::pin(read_with_budget( + &mut small_headers, + Body::from("1234"), + 100, + &budget, + )); + // 旧实现会让这份小正文直接进入,大正文的每次重试都可能被抢先。 + assert!(futures::poll!(&mut small).is_pending()); + drop(held); + let large = large.await.unwrap(); + assert_eq!(&large[..], b"123456789"); + assert!(futures::poll!(&mut small).is_pending()); + drop(large); + assert_eq!(&small.await.unwrap()[..], b"1234"); + } + + #[tokio::test(start_paused = true)] + async fn a_chunked_body_can_finish_growing_without_partial_read_deadlock() { + let budget = crate::resources::Budget::new(30); + let held = read_with_budget(&mut HeaderMap::new(), Body::from("123456"), 100, &budget) + .await + .unwrap(); + let mut first_headers = HeaderMap::new(); + let mut first = Box::pin(read_with_budget( + &mut first_headers, + chunked(&[b"123", b"456789"]), + 100, + &budget, + )); + assert!(futures::poll!(&mut first).is_pending()); + let mut second_headers = HeaderMap::new(); + let mut second = Box::pin(read_with_budget( + &mut second_headers, + chunked(&[b"123", b"456789"]), + 100, + &budget, + )); + assert!(futures::poll!(&mut second).is_pending()); + drop(held); + let first = first.await.unwrap(); + assert!(futures::poll!(&mut second).is_pending()); + drop(first); + assert_eq!(&second.await.unwrap()[..], b"123456789"); + } + + #[tokio::test(start_paused = true)] + async fn cancelling_the_head_waiter_returns_its_partial_body_and_queue_place() { + let budget = crate::resources::Budget::new(30); + let held = read_with_budget(&mut HeaderMap::new(), Body::from("123456"), 100, &budget) + .await + .unwrap(); + let mut large_headers = HeaderMap::new(); + let mut large = Box::pin(read_with_budget( + &mut large_headers, + chunked(&[b"123", b"456789"]), + 100, + &budget, + )); + assert!(futures::poll!(&mut large).is_pending()); + let mut small_headers = HeaderMap::new(); + let mut small = Box::pin(read_with_budget( + &mut small_headers, + Body::from("1234"), + 100, + &budget, + )); + assert!(futures::poll!(&mut small).is_pending()); + drop(large); + // 原来的竞争者还活着;只有取消者自己的预算和排队名额被归还。 + assert_eq!(&small.await.unwrap()[..], b"1234"); + drop(held); + } + + #[tokio::test(start_paused = true)] + async fn permanently_occupied_budget_has_a_bounded_wait_and_recovers() { + let budget = crate::resources::Budget::new(30); + let held = read_with_budget(&mut HeaderMap::new(), Body::from("123456"), 100, &budget) + .await + .unwrap(); + let started = tokio::time::Instant::now(); + let e = read_with_budget(&mut HeaderMap::new(), Body::from("123456789"), 100, &budget) + .await + .unwrap_err(); + assert_eq!(started.elapsed(), crate::resources::BODY_WAIT); + assert_eq!(e.detail.code, "gw.request.memory_busy"); + assert_eq!(e.status(), axum::http::StatusCode::TOO_MANY_REQUESTS); + assert_eq!(e.retry.unwrap().after_ms, 5_000); + drop(held); + assert!( + read_with_budget(&mut HeaderMap::new(), Body::from("123456789"), 100, &budget,) + .await + .is_ok() + ); + } + + #[tokio::test(start_paused = true)] + async fn an_idle_upload_releases_its_budget_and_admission_place() { + use futures::StreamExt; + let budget = crate::resources::Budget::new(30); + let stream = + futures::stream::iter([Ok::<_, std::io::Error>(Bytes::from_static(b"123456"))]) + .chain(futures::stream::pending()); + let e = read_with_budget( + &mut HeaderMap::new(), + Body::from_stream(stream), + 100, + &budget, + ) + .await + .unwrap_err(); + assert_eq!(e.detail.code, "gw.request.body_idle"); + assert!( + read_with_budget(&mut HeaderMap::new(), Body::from("123456789"), 100, &budget,) + .await + .is_ok() + ); + } + + #[tokio::test(start_paused = true)] + async fn decoded_bodies_return_the_compressed_input_headroom() { + let budget = crate::resources::Budget::new(3_003); + let decoded = read_with_budget( + &mut with_encoding("gzip"), + Body::from(gz(&[b'x'; 1_000])), + 10_000, + &budget, + ) + .await + .unwrap(); + assert_eq!(decoded.len(), 1_000); + // 解开的正文占 3,000;已经丢掉的压缩字节不能继续占着剩余 3 字节。 + let next = read_with_budget(&mut HeaderMap::new(), Body::from("x"), 10_000, &budget) + .await + .unwrap(); + assert_eq!(&next[..], b"x"); + drop((decoded, next)); + } + + #[tokio::test] + async fn http_large_requests_wait_and_recover_before_later_small_requests() { + use axum::{Router, extract::State, routing::post}; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::sync::mpsc; + type HttpState = ( + std::sync::Arc, + mpsc::UnboundedSender, + mpsc::UnboundedSender<(String, Bytes)>, + ); + async fn receive( + State((budget, entered, finished)): State, + mut headers: HeaderMap, + body: Body, + ) -> Result<(), GatewayError> { + let label = headers["x-test-request"].to_str().unwrap().to_owned(); + entered.send(label.clone()).unwrap(); + let body = read_with_budget(&mut headers, body, 10_000, &budget).await?; + // 模拟下游仍持有正文。测试决定何时归还,不依靠请求之间的延时碰运气。 + finished.send((label, body)).unwrap(); + Ok(()) + } + async fn event(receiver: &mut mpsc::UnboundedReceiver) -> T { + tokio::time::timeout(std::time::Duration::from_secs(5), receiver.recv()) + .await + .unwrap() + .unwrap() + } + let budget = crate::resources::Budget::new(3_000); + let json_body = |text: String| { + serde_json::json!({ + "model": "gpt-6.1-sol", + "input": [{"role": "user", "content": text}] + }) + .to_string() + }; + let held = read_with_budget( + &mut HeaderMap::new(), + Body::from(json_body("h".repeat(530))), + 10_000, + &budget, + ) + .await + .unwrap(); + let (entered, mut arrivals) = mpsc::unbounded_channel(); + let (finished, mut bodies) = mpsc::unbounded_channel(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let url = format!("http://{address}/v1/responses"); + let router = Router::new() + .route("/v1/responses", post(receive)) + .with_state((budget, entered, finished)); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + let client = reqwest::Client::new(); + let large = client + .post(&url) + .header("x-test-request", "large") + .header("content-type", "application/json") + .body(json_body("x".repeat(900))) + .send(); + let large = tokio::spawn(large); + assert_eq!(event(&mut arrivals).await, "large"); + let small = client + .post(&url) + .header("x-test-request", "small") + .body(json_body("hello".to_owned())) + .send(); + let small = tokio::spawn(small); + assert_eq!(event(&mut arrivals).await, "small"); + drop(held); + let (label, body) = event(&mut bodies).await; + assert_eq!(label, "large"); + assert_eq!(large.await.unwrap().unwrap().status(), 200); + assert!(!small.is_finished()); + drop(body); + let (label, body) = event(&mut bodies).await; + assert_eq!(label, "small"); + drop(body); + assert_eq!(small.await.unwrap().unwrap().status(), 200); + + // 已知长度的上传只发了一半时,其他放得下的正文仍能并行进入。 + let partial = json_body("p".repeat(530)); + let mut upload = tokio::net::TcpStream::connect(address).await.unwrap(); + let headers = format!( + "POST /v1/responses HTTP/1.1\r\nHost: {address}\r\n\ + Content-Length: {}\r\nx-test-request: partial\r\n\ + Connection: close\r\n\r\n", + partial.len() + ); + upload.write_all(headers.as_bytes()).await.unwrap(); + let halfway = partial.len() / 2; + upload + .write_all(&partial.as_bytes()[..halfway]) + .await + .unwrap(); + assert_eq!(event(&mut arrivals).await, "partial"); + let parallel = client + .post(&url) + .header("x-test-request", "parallel") + .body(json_body("hello".to_owned())) + .send(); + let parallel = tokio::spawn(parallel); + assert_eq!(event(&mut arrivals).await, "parallel"); + let (label, body) = event(&mut bodies).await; + assert_eq!(label, "parallel"); + assert_eq!(parallel.await.unwrap().unwrap().status(), 200); + drop(body); + upload + .write_all(&partial.as_bytes()[halfway..]) + .await + .unwrap(); + let (label, body) = event(&mut bodies).await; + assert_eq!(label, "partial"); + drop(body); + let mut response = Vec::new(); + upload.read_to_end(&mut response).await.unwrap(); + assert!(response.starts_with(b"HTTP/1.1 200")); + server.abort(); + let _ = server.await; + } + #[tokio::test] async fn a_gzip_body_is_decoded_and_the_encoding_header_is_dropped() { let packed = gz(b"12345"); diff --git a/docs/server.md b/docs/server.md index 407b7bd5..e3095e30 100644 --- a/docs/server.md +++ b/docs/server.md @@ -42,8 +42,15 @@ Incoming request bodies share a process-wide buffering budget: one eighth of the machine's memory (the cgroup limit on Linux containers), between 64 and 512 MiB, with threefold headroom for parsing and copies. The reservation follows the last reference to the buffered body, including while a request is queued. -When the budget is occupied, new bodies receive a retryable `429` with -`Retry-After`. A single large body can use the budget exclusively; the existing +Requests wait in arrival order for buffering space, so later small bodies cannot +keep a large request from entering. Known-size uncompressed uploads reserve their +space before reading and can upload concurrently. Unknown-size and compressed +bodies grow one at a time; decompression that runs out of space retries once after +obtaining exclusive space, without holding a CPU slot while waiting. Each budget +or admission wait is limited to 30 seconds, after which the request receives a +retryable `429` with `Retry-After` and Core logs the budget occupancy. An upload +that sends no body frame for 30 seconds is rejected and releases its reservation. +A single large body can use the budget exclusively; the existing 256 MiB per-request limit still applies. This budget covers request buffering, not the entire process RSS. diff --git a/docs/server.zh-CN.md b/docs/server.zh-CN.md index a44e1592..b245adff 100644 --- a/docs/server.zh-CN.md +++ b/docs/server.zh-CN.md @@ -18,7 +18,7 @@ Core 自动选择运行时规模,无需添加 `runtime` 配置或分配器环 解析、扫描、改写、解压和插件都先异步获取 CPU 名额。大正文立即交还 I/O 调度权;中小正文并发扫描、内联通道被占满时也会扩展执行线程。线程按实际负载增加,多余线程空闲 10 秒后回收;插件保留独立的沙箱栈,同样按需创建和回收线程。等待 CPU 名额时,流式回答、普通文件操作和 DNS 查询仍可继续。 -进来的请求体共用进程级缓冲预算:取机器内存的八分之一(Linux 容器按 cgroup 限额),范围为 64 至 512 MiB,并按正文的三倍预留解析和复制的余量。预算跟随正文的最后一份引用归还,请求排队期间也计入。预算被占用时,新请求返回可重试的 `429`,带有 `Retry-After`。单个大正文可以独占预算,原有的单请求 256 MiB 上限继续适用。这是请求缓冲的预算,并非整个进程 RSS 的硬上限。 +进来的请求体共用进程级缓冲预算:取机器内存的八分之一(Linux 容器按 cgroup 限额),范围为 64 至 512 MiB,并按正文的三倍预留解析和复制的余量。预算跟随正文的最后一份引用归还,请求排队期间也计入。新请求按到达顺序等预算,后来的小正文不能插队让大请求一直重试。已知上界的未压缩正文先留足空间,再并行读取;未知长度和压缩正文同一时刻只读一份,避免拿着半份正文互相等预算。解压空间不足时,释放 CPU 名额,等到可独占预算后重试一次。每次等准入或预算最多 30 秒,超时仍返回带 `Retry-After` 的可重试 `429`,并记录预算占用。上传连续 30 秒没有新的正文帧时会被拒绝并归还预算。单个大正文可以独占预算,原有的单请求 256 MiB 上限继续适用。这是请求缓冲的预算,并非整个进程 RSS 的硬上限。 发布包默认使用 mimalloc,减少内存碎片和空闲页留存。Linux 构建在内核允许时对 Core 进程禁用透明大页,降低小对象分配的驻留开销,不改系统设置。启动时记录选定的 worker 数、重任务名额、正文预算和分配器。CPU 和内存上限在启动时确定,执行线程在运行期间按负载自动增加和回收。 diff --git a/release-notes/0.98.1.md b/release-notes/0.98.1.md new file mode 100644 index 00000000..89f49759 --- /dev/null +++ b/release-notes/0.98.1.md @@ -0,0 +1,16 @@ +Large requests now wait their turn for the shared request-body budget instead of +immediately returning 429 while later small requests continue to enter. This +addresses the reproducible budget-starvation mechanism investigated in #403; +the reporter's original incident still lacks the response body and runtime +budget data needed to confirm its cause. + +Known-size uncompressed uploads reserve space before reading and remain +concurrent. Chunked and compressed requests grow under a single admission turn +to avoid partial-body deadlocks. Decompression can retry once with exclusive +space, releasing its CPU slot while waiting, and returns unused reservations +when the compressed input is discarded. + +Admission and budget waits are bounded at 30 seconds. Exhaustion still returns a +retryable 429 and now logs the budget occupancy. Uploads idle for 30 seconds are +rejected and release their budget. The existing memory policy and 256 MiB +per-request limit are unchanged.