diff --git a/crates/tw-gateway/src/error.rs b/crates/tw-gateway/src/error.rs index a59aa340..4379d93e 100644 --- a/crates/tw-gateway/src/error.rs +++ b/crates/tw-gateway/src/error.rs @@ -282,6 +282,11 @@ impl IntoResponse for GatewayError { "x-thinkwatch-error", HeaderValue::from_static(self.source.slug()), ); + // 保留粗分类和 SDK 的重试语义,另给诊断用的细码:只拿响应头也能区分 + // 本地正文预算、并发名额、密钥用量和上游限流。 + if let Ok(code) = HeaderValue::from_str(&self.detail.code) { + h.insert("x-thinkwatch-error-code", code); + } if let Some(r) = self.retry { let secs = r.after_ms.div_ceil(1000).max(1); h.insert(header::RETRY_AFTER, HeaderValue::from(secs)); @@ -414,6 +419,36 @@ mod tests { assert!(!r.headers().contains_key("retry-after")); } + #[tokio::test] + async fn body_budget_refusals_have_a_specific_code_without_changing_sdk_retries() { + for dialect in [ + Dialect::Anthropic, + Dialect::Chat, + Dialect::Responses, + Dialect::Gemini, + ] { + let r = crate::resources::memory_busy() + .in_dialect(dialect) + .into_response(); + assert_eq!(r.status(), StatusCode::TOO_MANY_REQUESTS); + assert_eq!(r.headers()["x-thinkwatch-error"], "rate_limited"); + assert_eq!( + r.headers()["x-thinkwatch-error-code"], + "gw.request.memory_busy" + ); + assert_eq!(r.headers()["retry-after"], "5"); + assert_eq!(r.headers()["retry-after-ms"], "5000"); + assert_eq!(r.headers()["x-should-retry"], "true"); + let body = to_bytes(r.into_body(), 64 * 1024).await.unwrap(); + let json: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert!(json["error"].is_object()); + } + let upstream = + GatewayError::rate_limited(msg!("t.upstream" => "slow down")).into_response(); + assert_eq!(upstream.headers()["x-thinkwatch-error"], "rate_limited"); + assert_eq!(upstream.headers()["x-thinkwatch-error-code"], "t.upstream"); + } + /// 密钥的用量上限拒绝时:多久之后能再来,到重置之前该不该重试。 #[tokio::test] async fn a_used_up_key_says_when_to_come_back_and_whether_to_retry() { diff --git a/crates/tw-gateway/src/resources.rs b/crates/tw-gateway/src/resources.rs index d9ec5234..cdf421a9 100644 --- a/crates/tw-gateway/src/resources.rs +++ b/crates/tw-gateway/src/resources.rs @@ -1,8 +1,9 @@ //! 进程级的自动资源调度。配置重载、多个运行时共用同一份预算。 //! //! CPU 重任务先异步等名额,再交给阻塞线程;等候者不会占满 Tokio 的线程。 -//! 正文按字节占预算,并为解析、复制留三倍余量。预算不足时先退避重试,避免已经 -//! 持有一半正文的请求互相等内存。名额跟着 Bytes 的最后一份引用归还。 +//! 正文按字节占预算,并为解析、复制留三倍余量。预算不足时让客户端退避重试,避免已经 +//! 持有一半正文的请求互相等内存。这不是公平队列:持续有其他正文占用时,大请求可能 +//! 反复被拒,小请求仍能通过。名额跟着 Bytes 的最后一份引用归还。 //! //! 单个合法的大正文可以独占预算:自动预算不改变 256 MiB 的单请求上限。这是 //! 正文缓冲的预算,不是对整个进程 RSS 的硬限制(插件、JSON 对象等另有开销)。 @@ -154,18 +155,65 @@ pub(crate) struct Reservation { charged: usize, } +/// 在锁内取得的拒绝快照。不能等请求释放预算后再读,否则日志会低报占用。 +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct BudgetDenied { + limit: usize, + used: usize, + reserved: usize, + body_bytes: usize, + charge: usize, +} + +impl BudgetDenied { + pub(crate) fn log( + &self, + stage: &'static str, + received_bytes: usize, + decoded_bytes: usize, + declared_bytes: Option, + encoding: &'static str, + ) { + // 只记大小和受控枚举,不记正文、请求头、URL 或客户端标识。 + tracing::warn!( + code = "gw.request.memory_busy", + core_version = env!("CARGO_PKG_VERSION"), + stage, + encoding, + ?declared_bytes, + received_bytes, + decoded_bytes, + body_bytes = self.body_bytes, + budget_bytes = self.limit, + used_bytes = self.used, + reserved_bytes = self.reserved, + requested_charge_bytes = self.charge, + additional_bytes = self.charge - self.reserved, + available_bytes = self.limit - self.used, + retry_after_ms = crate::error::BUSY_RETRY_AFTER_SECS * 1000, + "request rejected before routing: local body buffer budget exhausted" + ); + } +} + impl Reservation { /// 在追加正文或解压输出之前调用。失败不会改变已经占用的预算。 - pub(crate) fn grow(&mut self, bytes: usize) -> bool { + pub(crate) fn grow(&mut self, bytes: usize) -> Result<(), BudgetDenied> { let charge = bytes.saturating_mul(BODY_HEADROOM).min(self.budget.limit); let extra = charge.saturating_sub(self.charged); let mut used = self.budget.used.lock().unwrap_or_else(|e| e.into_inner()); if extra > self.budget.limit - *used { - return false; + return Err(BudgetDenied { + limit: self.budget.limit, + used: *used, + reserved: self.charged, + body_bytes: bytes, + charge, + }); } *used += extra; self.charged += extra; - true + Ok(()) } pub(crate) fn wrap(self, data: Bytes) -> Bytes { @@ -307,7 +355,7 @@ mod tests { fn reservations_survive_bytes_clones_and_slices() { let budget = Budget::new(30); let mut reservation = budget.reserve(); - assert!(reservation.grow(5)); + assert!(reservation.grow(5).is_ok()); let data = reservation.wrap(Bytes::from_static(b"hello")); let clone = data.clone(); let slice = clone.slice(1..3); @@ -324,11 +372,11 @@ mod tests { let budget = Budget::new(30); let mut first = budget.reserve(); let mut second = budget.reserve(); - assert!(first.grow(6)); - assert!(second.grow(4)); - assert!(!first.grow(7)); + assert!(first.grow(6).is_ok()); + assert!(second.grow(4).is_ok()); + assert!(first.grow(7).is_err()); drop(second); - assert!(first.grow(7)); + assert!(first.grow(7).is_ok()); drop(first); assert_eq!(*budget.used.lock().unwrap(), 0); } @@ -337,10 +385,35 @@ mod tests { fn a_large_legal_body_can_run_exclusively_without_deadlocking() { let budget = Budget::new(30); let mut large = budget.reserve(); - assert!(large.grow(100)); + assert!(large.grow(100).is_ok()); let mut small = budget.reserve(); - assert!(!small.grow(1)); + assert!(small.grow(1).is_err()); drop(large); - assert!(small.grow(1)); + assert!(small.grow(1).is_ok()); + } + + #[test] + fn denial_captures_occupancy_before_the_partial_body_is_released() { + let budget = Budget::new(30); + let mut held = budget.reserve(); + held.grow(6).unwrap(); + let mut reading = budget.reserve(); + reading.grow(3).unwrap(); + let denied = reading.grow(5).unwrap_err(); + assert_eq!( + denied, + BudgetDenied { + limit: 30, + used: 27, + reserved: 9, + body_bytes: 5, + charge: 15, + } + ); + drop(reading); + assert_eq!(*budget.used.lock().unwrap(), 18); + assert_eq!(denied.used, 27, "the diagnostic is the refusal snapshot"); + // 超过预算的合法正文按独占预算计费,日志要同时保留真实字节数和封顶后的费用。 + assert_eq!(budget.reserve().grow(100).unwrap_err().charge, 30); } } diff --git a/crates/tw-gateway/src/server/intake.rs b/crates/tw-gateway/src/server/intake.rs index 03da9957..f28d3c24 100644 --- a/crates/tw-gateway/src/server/intake.rs +++ b/crates/tw-gateway/src/server/intake.rs @@ -76,7 +76,14 @@ async fn read_with_budget( ))); } // 先记账再拷贝。用满就让客户端退避,不拿着半份正文等别的请求释放。 - if !reservation.grow(next) { + if let Err(denied) = reservation.grow(next) { + denied.log( + "read", + next, + 0, + declared, + coding.map_or("identity", Coding::name), + ); return Err(crate::resources::memory_busy()); } raw.extend_from_slice(&data); @@ -92,11 +99,12 @@ async fn read_with_budget( // 客户端取消时阻塞任务仍可能在跑:CPU 名额和预算都跟任务走。 let _slot = slot; let decoded = coding.decode_with(&raw, max, |output| { - if reservation.grow(raw.len().saturating_add(output)) { - Ok(()) - } else { - Err(Decode::Busy) - } + reservation + .grow(raw.len().saturating_add(output)) + .map_err(|denied| { + denied.log("decode", raw.len(), output, declared, coding.name()); + Decode::Busy + }) })?; Ok(reservation.wrap(decoded)) }) @@ -208,6 +216,41 @@ mod tests { ); } + #[tokio::test] + async fn a_long_body_can_be_repeatedly_refused_while_short_bodies_succeed() { + let budget = crate::resources::Budget::new(30); + let held = read_with_budget(&mut HeaderMap::new(), chunked(&[b"123456"]), 100, &budget) + .await + .unwrap(); + // 重试没有优先权。小正文能用剩下的 12 字节预算,大正文(包括 compact + // 传来的旧上下文)每次都会在第二块被拒;每次拒绝都应释放已读的第一块。 + for _ in 0..8 { + let e = read_with_budget( + &mut HeaderMap::new(), + chunked(&[b"123", b"456789"]), + 100, + &budget, + ) + .await + .unwrap_err(); + assert_eq!(e.detail.code, "gw.request.memory_busy"); + let short = read_with_budget(&mut HeaderMap::new(), chunked(&[b"1234"]), 100, &budget) + .await + .unwrap(); + drop(short); + } + drop(held); + let long = read_with_budget( + &mut HeaderMap::new(), + chunked(&[b"123", b"456789"]), + 100, + &budget, + ) + .await + .unwrap(); + assert_eq!(&long[..], b"123456789"); + } + #[tokio::test] async fn an_oversized_body_releases_its_partial_reservation() { let budget = crate::resources::Budget::new(30); diff --git a/docs/server.md b/docs/server.md index 407b7bd5..52277e1e 100644 --- a/docs/server.md +++ b/docs/server.md @@ -47,6 +47,27 @@ When the budget is occupied, new bodies receive a retryable `429` with 256 MiB per-request limit still applies. This budget covers request buffering, not the entire process RSS. +Body admission has no fairness queue. While other bodies keep occupying the +budget, a large request can be refused on every retry even though smaller ones +still fit. A context-compaction request must upload its old context through the +same check. Retrying is not a guarantee of admission; once the other reservations +are released, a legal large body can enter. Token counts and cache hits do not +measure this budget: it accounts for body bytes, including compressed input and +decoded output during decompression. + +For diagnosis, a local budget refusal emits a warning with +`code="gw.request.memory_busy"`, the Core version, `stage` (`read` or `decode`), +encoding, declared/received/decoded byte counts, and a snapshot of +`budget_bytes`, `used_bytes`, `reserved_bytes`, `requested_charge_bytes`, +`additional_bytes` and `available_bytes` at the refusal. No body content, +credentials or request URL is logged. This happens before routing and request +history, so absence from request history does not rule out a local refusal. +The response carries `x-thinkwatch-error-code: gw.request.memory_busy`; the +existing `x-thinkwatch-error: rate_limited`, retry headers and client-specific +JSON shape stay unchanged. The coarse `rate_limited` header and five-second +retry delay alone do not identify the cause. On older versions without these +diagnostics, collect the running Core version and error response body too. + Release binaries use mimalloc to reduce allocator fragmentation and retained free pages. Linux builds disable transparent huge pages for the Core process where the kernel permits it, reducing residency for small allocations without diff --git a/docs/server.zh-CN.md b/docs/server.zh-CN.md index a44e1592..e43ced75 100644 --- a/docs/server.zh-CN.md +++ b/docs/server.zh-CN.md @@ -22,6 +22,10 @@ Core 自动选择运行时规模,无需添加 `runtime` 配置或分配器环 发布包默认使用 mimalloc,减少内存碎片和空闲页留存。Linux 构建在内核允许时对 Core 进程禁用透明大页,降低小对象分配的驻留开销,不改系统设置。启动时记录选定的 worker 数、重任务名额、正文预算和分配器。CPU 和内存上限在启动时确定,执行线程在运行期间按负载自动增加和回收。 +正文准入没有公平队列。只要其他正文持续占用预算,大请求可能每次重试都被拒,小请求却仍能进入。压缩上下文的请求也要先上传旧上下文,经过同一道检查;重试不保证最终获得预算。其他预留释放后,合法的大正文可以进入。输入 token 数和缓存命中量不代表这里的占用:预算按正文字节数计费,解压期间包括压缩输入和解压输出。 + +本地预算不足时会输出一条警告,带 `code="gw.request.memory_busy"`、Core 版本、`stage`(`read` 或 `decode`)、编码、声明长度、已接收及解压字节数,以及拒绝当时的 `budget_bytes`、`used_bytes`、`reserved_bytes`、`requested_charge_bytes`、`additional_bytes`、`available_bytes` 快照。日志不含正文、凭据或请求 URL。拒绝发生在路由和请求记录之前,所以请求历史里没有记录也不能排除本地拒绝。响应另带 `x-thinkwatch-error-code: gw.request.memory_busy`,原有的 `x-thinkwatch-error: rate_limited`、重试头和客户端方言格式不变。仅凭 `rate_limited` 和五秒重试间隔不能确定原因;旧版没有这些诊断时,还需取得运行中的 Core 版本和失败响应正文。 + ## 1. 安装 ```sh