Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 35 additions & 0 deletions crates/tw-gateway/src/error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,11 @@ impl IntoResponse for GatewayError {
"x-thinkwatch-error",
HeaderValue::from_static(self.source.slug()),
);
// 保留粗分类和 SDK 的重试语义,另给诊断用的细码:只拿响应头也能区分
// 本地正文预算、并发名额、密钥用量和上游限流。
if let Ok(code) = HeaderValue::from_str(&self.detail.code) {
h.insert("x-thinkwatch-error-code", code);
}
if let Some(r) = self.retry {
let secs = r.after_ms.div_ceil(1000).max(1);
h.insert(header::RETRY_AFTER, HeaderValue::from(secs));
Expand Down Expand Up @@ -414,6 +419,36 @@ mod tests {
assert!(!r.headers().contains_key("retry-after"));
}

#[tokio::test]
async fn body_budget_refusals_have_a_specific_code_without_changing_sdk_retries() {
for dialect in [
Dialect::Anthropic,
Dialect::Chat,
Dialect::Responses,
Dialect::Gemini,
] {
let r = crate::resources::memory_busy()
.in_dialect(dialect)
.into_response();
assert_eq!(r.status(), StatusCode::TOO_MANY_REQUESTS);
assert_eq!(r.headers()["x-thinkwatch-error"], "rate_limited");
assert_eq!(
r.headers()["x-thinkwatch-error-code"],
"gw.request.memory_busy"
);
assert_eq!(r.headers()["retry-after"], "5");
assert_eq!(r.headers()["retry-after-ms"], "5000");
assert_eq!(r.headers()["x-should-retry"], "true");
let body = to_bytes(r.into_body(), 64 * 1024).await.unwrap();
let json: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert!(json["error"].is_object());
}
let upstream =
GatewayError::rate_limited(msg!("t.upstream" => "slow down")).into_response();
assert_eq!(upstream.headers()["x-thinkwatch-error"], "rate_limited");
assert_eq!(upstream.headers()["x-thinkwatch-error-code"], "t.upstream");
}

/// 密钥的用量上限拒绝时:多久之后能再来,到重置之前该不该重试。
#[tokio::test]
async fn a_used_up_key_says_when_to_come_back_and_whether_to_retry() {
Expand Down
99 changes: 86 additions & 13 deletions crates/tw-gateway/src/resources.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
//! 进程级的自动资源调度。配置重载、多个运行时共用同一份预算。
//!
//! CPU 重任务先异步等名额,再交给阻塞线程;等候者不会占满 Tokio 的线程。
//! 正文按字节占预算,并为解析、复制留三倍余量。预算不足时先退避重试,避免已经
//! 持有一半正文的请求互相等内存。名额跟着 Bytes 的最后一份引用归还。
//! 正文按字节占预算,并为解析、复制留三倍余量。预算不足时让客户端退避重试,避免已经
//! 持有一半正文的请求互相等内存。这不是公平队列:持续有其他正文占用时,大请求可能
//! 反复被拒,小请求仍能通过。名额跟着 Bytes 的最后一份引用归还。
//!
//! 单个合法的大正文可以独占预算:自动预算不改变 256 MiB 的单请求上限。这是
//! 正文缓冲的预算,不是对整个进程 RSS 的硬限制(插件、JSON 对象等另有开销)。
Expand Down Expand Up @@ -154,18 +155,65 @@ pub(crate) struct Reservation {
charged: usize,
}

/// 在锁内取得的拒绝快照。不能等请求释放预算后再读,否则日志会低报占用。
#[derive(Debug, PartialEq, Eq)]
pub(crate) struct BudgetDenied {
limit: usize,
used: usize,
reserved: usize,
body_bytes: usize,
charge: usize,
}

impl BudgetDenied {
pub(crate) fn log(
&self,
stage: &'static str,
received_bytes: usize,
decoded_bytes: usize,
declared_bytes: Option<u64>,
encoding: &'static str,
) {
// 只记大小和受控枚举,不记正文、请求头、URL 或客户端标识。
tracing::warn!(
code = "gw.request.memory_busy",
core_version = env!("CARGO_PKG_VERSION"),
stage,
encoding,
?declared_bytes,
received_bytes,
decoded_bytes,
body_bytes = self.body_bytes,
budget_bytes = self.limit,
used_bytes = self.used,
reserved_bytes = self.reserved,
requested_charge_bytes = self.charge,
additional_bytes = self.charge - self.reserved,
available_bytes = self.limit - self.used,
retry_after_ms = crate::error::BUSY_RETRY_AFTER_SECS * 1000,
"request rejected before routing: local body buffer budget exhausted"
);
}
}

impl Reservation {
/// 在追加正文或解压输出之前调用。失败不会改变已经占用的预算。
pub(crate) fn grow(&mut self, bytes: usize) -> bool {
pub(crate) fn grow(&mut self, bytes: usize) -> Result<(), BudgetDenied> {
let charge = bytes.saturating_mul(BODY_HEADROOM).min(self.budget.limit);
let extra = charge.saturating_sub(self.charged);
let mut used = self.budget.used.lock().unwrap_or_else(|e| e.into_inner());
if extra > self.budget.limit - *used {
return false;
return Err(BudgetDenied {
limit: self.budget.limit,
used: *used,
reserved: self.charged,
body_bytes: bytes,
charge,
});
}
*used += extra;
self.charged += extra;
true
Ok(())
}

pub(crate) fn wrap(self, data: Bytes) -> Bytes {
Expand Down Expand Up @@ -307,7 +355,7 @@ mod tests {
fn reservations_survive_bytes_clones_and_slices() {
let budget = Budget::new(30);
let mut reservation = budget.reserve();
assert!(reservation.grow(5));
assert!(reservation.grow(5).is_ok());
let data = reservation.wrap(Bytes::from_static(b"hello"));
let clone = data.clone();
let slice = clone.slice(1..3);
Expand All @@ -324,11 +372,11 @@ mod tests {
let budget = Budget::new(30);
let mut first = budget.reserve();
let mut second = budget.reserve();
assert!(first.grow(6));
assert!(second.grow(4));
assert!(!first.grow(7));
assert!(first.grow(6).is_ok());
assert!(second.grow(4).is_ok());
assert!(first.grow(7).is_err());
drop(second);
assert!(first.grow(7));
assert!(first.grow(7).is_ok());
drop(first);
assert_eq!(*budget.used.lock().unwrap(), 0);
}
Expand All @@ -337,10 +385,35 @@ mod tests {
fn a_large_legal_body_can_run_exclusively_without_deadlocking() {
let budget = Budget::new(30);
let mut large = budget.reserve();
assert!(large.grow(100));
assert!(large.grow(100).is_ok());
let mut small = budget.reserve();
assert!(!small.grow(1));
assert!(small.grow(1).is_err());
drop(large);
assert!(small.grow(1));
assert!(small.grow(1).is_ok());
}

#[test]
fn denial_captures_occupancy_before_the_partial_body_is_released() {
let budget = Budget::new(30);
let mut held = budget.reserve();
held.grow(6).unwrap();
let mut reading = budget.reserve();
reading.grow(3).unwrap();
let denied = reading.grow(5).unwrap_err();
assert_eq!(
denied,
BudgetDenied {
limit: 30,
used: 27,
reserved: 9,
body_bytes: 5,
charge: 15,
}
);
drop(reading);
assert_eq!(*budget.used.lock().unwrap(), 18);
assert_eq!(denied.used, 27, "the diagnostic is the refusal snapshot");
// 超过预算的合法正文按独占预算计费,日志要同时保留真实字节数和封顶后的费用。
assert_eq!(budget.reserve().grow(100).unwrap_err().charge, 30);
}
}
55 changes: 49 additions & 6 deletions crates/tw-gateway/src/server/intake.rs
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,14 @@ async fn read_with_budget(
)));
}
// 先记账再拷贝。用满就让客户端退避,不拿着半份正文等别的请求释放。
if !reservation.grow(next) {
if let Err(denied) = reservation.grow(next) {
denied.log(
"read",
next,
0,
declared,
coding.map_or("identity", Coding::name),
);
return Err(crate::resources::memory_busy());
}
raw.extend_from_slice(&data);
Expand All @@ -92,11 +99,12 @@ async fn read_with_budget(
// 客户端取消时阻塞任务仍可能在跑:CPU 名额和预算都跟任务走。
let _slot = slot;
let decoded = coding.decode_with(&raw, max, |output| {
if reservation.grow(raw.len().saturating_add(output)) {
Ok(())
} else {
Err(Decode::Busy)
}
reservation
.grow(raw.len().saturating_add(output))
.map_err(|denied| {
denied.log("decode", raw.len(), output, declared, coding.name());
Decode::Busy
})
})?;
Ok(reservation.wrap(decoded))
})
Expand Down Expand Up @@ -208,6 +216,41 @@ mod tests {
);
}

#[tokio::test]
async fn a_long_body_can_be_repeatedly_refused_while_short_bodies_succeed() {
let budget = crate::resources::Budget::new(30);
let held = read_with_budget(&mut HeaderMap::new(), chunked(&[b"123456"]), 100, &budget)
.await
.unwrap();
// 重试没有优先权。小正文能用剩下的 12 字节预算,大正文(包括 compact
// 传来的旧上下文)每次都会在第二块被拒;每次拒绝都应释放已读的第一块。
for _ in 0..8 {
let e = read_with_budget(
&mut HeaderMap::new(),
chunked(&[b"123", b"456789"]),
100,
&budget,
)
.await
.unwrap_err();
assert_eq!(e.detail.code, "gw.request.memory_busy");
let short = read_with_budget(&mut HeaderMap::new(), chunked(&[b"1234"]), 100, &budget)
.await
.unwrap();
drop(short);
}
drop(held);
let long = read_with_budget(
&mut HeaderMap::new(),
chunked(&[b"123", b"456789"]),
100,
&budget,
)
.await
.unwrap();
assert_eq!(&long[..], b"123456789");
}

#[tokio::test]
async fn an_oversized_body_releases_its_partial_reservation() {
let budget = crate::resources::Budget::new(30);
Expand Down
21 changes: 21 additions & 0 deletions docs/server.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,27 @@ When the budget is occupied, new bodies receive a retryable `429` with
256 MiB per-request limit still applies. This budget covers request buffering,
not the entire process RSS.

Body admission has no fairness queue. While other bodies keep occupying the
budget, a large request can be refused on every retry even though smaller ones
still fit. A context-compaction request must upload its old context through the
same check. Retrying is not a guarantee of admission; once the other reservations
are released, a legal large body can enter. Token counts and cache hits do not
measure this budget: it accounts for body bytes, including compressed input and
decoded output during decompression.

For diagnosis, a local budget refusal emits a warning with
`code="gw.request.memory_busy"`, the Core version, `stage` (`read` or `decode`),
encoding, declared/received/decoded byte counts, and a snapshot of
`budget_bytes`, `used_bytes`, `reserved_bytes`, `requested_charge_bytes`,
`additional_bytes` and `available_bytes` at the refusal. No body content,
credentials or request URL is logged. This happens before routing and request
history, so absence from request history does not rule out a local refusal.
The response carries `x-thinkwatch-error-code: gw.request.memory_busy`; the
existing `x-thinkwatch-error: rate_limited`, retry headers and client-specific
JSON shape stay unchanged. The coarse `rate_limited` header and five-second
retry delay alone do not identify the cause. On older versions without these
diagnostics, collect the running Core version and error response body too.

Release binaries use mimalloc to reduce allocator fragmentation and retained
free pages. Linux builds disable transparent huge pages for the Core process
where the kernel permits it, reducing residency for small allocations without
Expand Down
4 changes: 4 additions & 0 deletions docs/server.zh-CN.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,10 @@ Core 自动选择运行时规模,无需添加 `runtime` 配置或分配器环

发布包默认使用 mimalloc,减少内存碎片和空闲页留存。Linux 构建在内核允许时对 Core 进程禁用透明大页,降低小对象分配的驻留开销,不改系统设置。启动时记录选定的 worker 数、重任务名额、正文预算和分配器。CPU 和内存上限在启动时确定,执行线程在运行期间按负载自动增加和回收。

正文准入没有公平队列。只要其他正文持续占用预算,大请求可能每次重试都被拒,小请求却仍能进入。压缩上下文的请求也要先上传旧上下文,经过同一道检查;重试不保证最终获得预算。其他预留释放后,合法的大正文可以进入。输入 token 数和缓存命中量不代表这里的占用:预算按正文字节数计费,解压期间包括压缩输入和解压输出。

本地预算不足时会输出一条警告,带 `code="gw.request.memory_busy"`、Core 版本、`stage`(`read` 或 `decode`)、编码、声明长度、已接收及解压字节数,以及拒绝当时的 `budget_bytes`、`used_bytes`、`reserved_bytes`、`requested_charge_bytes`、`additional_bytes`、`available_bytes` 快照。日志不含正文、凭据或请求 URL。拒绝发生在路由和请求记录之前,所以请求历史里没有记录也不能排除本地拒绝。响应另带 `x-thinkwatch-error-code: gw.request.memory_busy`,原有的 `x-thinkwatch-error: rate_limited`、重试头和客户端方言格式不变。仅凭 `rate_limited` 和五秒重试间隔不能确定原因;旧版没有这些诊断时,还需取得运行中的 Core 版本和失败响应正文。

## 1. 安装

```sh
Expand Down
Loading