From f07fbcd7d93280f0fc7f2afb169b86125c010f3b Mon Sep 17 00:00:00 2001 From: fylorn <249551762+fylorn@users.noreply.github.com> Date: Thu, 19 Nov 2026 19:39:57 +0800 Subject: [PATCH 1/3] feat(routing): apply route changes before dispatch Fence obsolete requests and retries while letting dispatched responses finish. Preserve account-bound history and reject incompatible requests before sending them. Make manual selection use a single upstream. Prepare Core 0.99.0. --- Cargo.lock | 44 +- Cargo.toml | 2 +- crates/tw-api/msg-codes.txt | 5 + crates/tw-api/src/lib.rs | 5 + crates/tw-control/src/lib.rs | 1 + crates/tw-engine/src/engine.rs | 32 +- crates/tw-gateway/src/affinity.rs | 22 +- crates/tw-gateway/src/error.rs | 43 +- crates/tw-gateway/src/handover.rs | 9 + crates/tw-gateway/src/handover/wire.rs | 2 + crates/tw-gateway/src/lib.rs | 1 + crates/tw-gateway/src/oauth.rs | 14 + crates/tw-gateway/src/route_switch.rs | 824 ++++++++++++++++++ crates/tw-gateway/src/server.rs | 3 +- crates/tw-gateway/src/server/pipeline.rs | 65 +- crates/tw-gateway/src/server/pipeline/hop.rs | 120 ++- .../tw-gateway/src/server/pipeline/relay.rs | 22 + .../src/server/pipeline/web/socket.rs | 37 +- crates/tw-gateway/src/server/upgrade.rs | 24 +- crates/tw-gateway/src/state.rs | 42 +- crates/tw-gateway/src/wire.rs | 2 +- crates/tw-gateway/src/ws.rs | 149 +++- crates/tw-gateway/tests/sealed.rs | 333 +++++-- crates/tw-gateway/tests/ws.rs | 126 +++ 24 files changed, 1769 insertions(+), 158 deletions(-) create mode 100644 crates/tw-gateway/src/route_switch.rs diff --git a/Cargo.lock b/Cargo.lock index 92be819c..208add36 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4148,7 +4148,7 @@ dependencies = [ [[package]] name = "tw-api" -version = "0.98.1" +version = "0.99.0" dependencies = [ "serde", "serde_json", @@ -4159,7 +4159,7 @@ dependencies = [ [[package]] name = "tw-bedrock" -version = "0.98.1" +version = "0.99.0" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -4180,7 +4180,7 @@ dependencies = [ [[package]] name = "tw-breaker" -version = "0.98.1" +version = "0.99.0" dependencies = [ "serde", "serde_json", @@ -4188,7 +4188,7 @@ dependencies = [ [[package]] name = "tw-config" -version = "0.98.1" +version = "0.99.0" dependencies = [ "blake3", "icu_locale_core", @@ -4216,7 +4216,7 @@ dependencies = [ [[package]] name = "tw-control" -version = "0.98.1" +version = "0.99.0" dependencies = [ "axum", "base64 0.22.1", @@ -4262,7 +4262,7 @@ dependencies = [ [[package]] name = "tw-dialect" -version = "0.98.1" +version = "0.99.0" dependencies = [ "memchr", "serde", @@ -4271,7 +4271,7 @@ dependencies = [ [[package]] name = "tw-engine" -version = "0.98.1" +version = "0.99.0" dependencies = [ "chrono", "serde", @@ -4285,7 +4285,7 @@ dependencies = [ [[package]] name = "tw-gateway" -version = "0.98.1" +version = "0.99.0" dependencies = [ "arc-swap", "async-stream", @@ -4340,7 +4340,7 @@ dependencies = [ [[package]] name = "tw-guard" -version = "0.98.1" +version = "0.99.0" dependencies = [ "aho-corasick", "base64 0.22.1", @@ -4361,7 +4361,7 @@ dependencies = [ [[package]] name = "tw-handover" -version = "0.98.1" +version = "0.99.0" dependencies = [ "libc", "serde", @@ -4374,7 +4374,7 @@ dependencies = [ [[package]] name = "tw-link" -version = "0.98.1" +version = "0.99.0" dependencies = [ "serde", "serde_json", @@ -4388,7 +4388,7 @@ dependencies = [ [[package]] name = "tw-observe" -version = "0.98.1" +version = "0.99.0" dependencies = [ "tokio", "tracing", @@ -4397,7 +4397,7 @@ dependencies = [ [[package]] name = "tw-pack" -version = "0.98.1" +version = "0.99.0" dependencies = [ "age", "blake3", @@ -4421,7 +4421,7 @@ dependencies = [ [[package]] name = "tw-plugin" -version = "0.98.1" +version = "0.99.0" dependencies = [ "libc", "rand 0.10.2", @@ -4437,7 +4437,7 @@ dependencies = [ [[package]] name = "tw-pricing" -version = "0.98.1" +version = "0.99.0" dependencies = [ "arc-swap", "flate2", @@ -4450,14 +4450,14 @@ dependencies = [ [[package]] name = "tw-secret" -version = "0.98.1" +version = "0.99.0" dependencies = [ "thiserror 2.0.20", ] [[package]] name = "tw-store" -version = "0.98.1" +version = "0.99.0" dependencies = [ "blake3", "bytes", @@ -4477,7 +4477,7 @@ dependencies = [ [[package]] name = "tw-sync" -version = "0.98.1" +version = "0.99.0" dependencies = [ "age", "axum", @@ -4505,7 +4505,7 @@ dependencies = [ [[package]] name = "tw-types" -version = "0.98.1" +version = "0.99.0" dependencies = [ "serde", "serde_json", @@ -4514,7 +4514,7 @@ dependencies = [ [[package]] name = "tw-watch" -version = "0.98.1" +version = "0.99.0" dependencies = [ "libc", "notify", @@ -4525,7 +4525,7 @@ dependencies = [ [[package]] name = "tw-yaml" -version = "0.98.1" +version = "0.99.0" dependencies = [ "saphyr-parser", "serde_yaml_ng", @@ -4535,7 +4535,7 @@ dependencies = [ [[package]] name = "twcore" -version = "0.98.1" +version = "0.99.0" dependencies = [ "anyhow", "axum", diff --git a/Cargo.toml b/Cargo.toml index 1b55d1ca..3fe09b7e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,7 +61,7 @@ categories = ["network-programming", "web-programming::http-server"] # 二进制走 CalVer,crate 走 SemVer —— 两者是两回事,卖给不同的人。 # 这里是 crate 的版本。 -version = "0.98.1" +version = "0.99.0" [workspace.dependencies] # ── 内部 crate ─────────────────────────────────────────── diff --git a/crates/tw-api/msg-codes.txt b/crates/tw-api/msg-codes.txt index e347a970..f3594929 100644 --- a/crates/tw-api/msg-codes.txt +++ b/crates/tw-api/msg-codes.txt @@ -436,13 +436,18 @@ gw.request.decoded_body_over_limit gw.request.memory_busy gw.request.unsupported_encoding gw.route.all_selected_disabled +gw.route.changed gw.route.denied gw.route.failed +gw.route.history.content +gw.route.history.reasoning +gw.route.history.stored gw.route.no_upstream_alive gw.route.protocol_mismatch gw.route.rule_failed gw.route.sealed_upstream_gone gw.route.selected_upstream_missing +gw.route.socket_target gw.route.upstream_missing gw.toolcall.connection_cut gw.toolcall.response_cut diff --git a/crates/tw-api/src/lib.rs b/crates/tw-api/src/lib.rs index 4454612b..c8623f07 100644 --- a/crates/tw-api/src/lib.rs +++ b/crates/tw-api/src/lib.rs @@ -2585,6 +2585,11 @@ pub struct ConfigRejection { #[derive(Debug, Clone, Serialize, Deserialize)] #[cfg_attr(feature = "ts", derive(ts_rs::TS))] pub struct Overview { + /// Routing changes take effect on the next request, including manual selection. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[cfg_attr(feature = "ts", ts(optional))] + pub hard_route_switch: Option, + /// 配置文件现在的版本号。和 `GET /config` 的 `version`、改配置时带的 /// `base_version` 是同一个。 /// diff --git a/crates/tw-control/src/lib.rs b/crates/tw-control/src/lib.rs index 561121e2..be0de954 100644 --- a/crates/tw-control/src/lib.rs +++ b/crates/tw-control/src/lib.rs @@ -336,6 +336,7 @@ async fn overview(State(s): State) -> Json { None => 0, }; Json(tw_api::Overview { + hard_route_switch: Some(true), // **磁盘上那一份的版本,不是正在服务的那一份。**改配置时核对的是 // 文件(见 `ConfigManager::write`)。两者只在文件被改坏、旧配置还在 // 服务时不一样:这时拿文件的版本去改,得到的是「文件读不懂」,而不是 diff --git a/crates/tw-engine/src/engine.rs b/crates/tw-engine/src/engine.rs index c2b8efac..71708dd8 100644 --- a/crates/tw-engine/src/engine.rs +++ b/crates/tw-engine/src/engine.rs @@ -252,6 +252,14 @@ impl Group { self.providers.iter().any(|m| m.name == name) } + /// The operator's selection, or the first member in older configs without one. + pub fn selection(&self) -> Option<&str> { + self.selected + .as_deref() + .filter(|name| self.has(name)) + .or_else(|| self.providers.first().map(|member| member.name.as_str())) + } + /// 成员 `name` 的权重。不是成员的按 1 算 pub fn weight(&self, name: &str) -> u32 { self.providers @@ -1523,23 +1531,7 @@ impl Engine { match g.kind { // 顺序就是优先级。第一个健康的就用,缓存持续命中。 GroupType::Fallback => g.names(), - GroupType::Select => { - // 选中的排头,其余仍然留着做故障转移 —— **手动选一家不 - // 等于放弃容错**,那家挂了照样该切。 - let mut out = Vec::with_capacity(g.providers.len()); - if let Some(sel) = &g.selected - && g.has(sel) - { - out.push(sel.clone()); - } - out.extend( - g.providers - .iter() - .filter(|p| Some(&p.name) != g.selected.as_ref()) - .map(|p| p.name.clone()), - ); - out - } + GroupType::Select => g.selection().map(str::to_owned).into_iter().collect(), // 这三种要运行时的数字才排得出来。**这里只给集合, // 顺序由 `order` 定** —— 它是纯函数,数据面和试算页都调它。 GroupType::LoadBalance | GroupType::UrlTest | GroupType::Cheapest => g.names(), @@ -1892,9 +1884,7 @@ mod tests { } #[test] - fn select_puts_the_chosen_one_first_but_keeps_the_rest_for_failover() { - // **手动选一家不等于放弃容错。**那家挂了照样该切,否则「select」 - // 就变成了「单点故障」。 + fn select_uses_only_the_chosen_member() { let g = Group { name: "pool".into(), kind: GroupType::Select, @@ -1907,7 +1897,7 @@ mod tests { vec![g], vec![route("x", "{}", "pool")], ); - assert_eq!(candidates(&e, &facts("x")), ["b", "a", "c"]); + assert_eq!(candidates(&e, &facts("x")), ["b"]); } #[test] diff --git a/crates/tw-gateway/src/affinity.rs b/crates/tw-gateway/src/affinity.rs index 1952620f..879e3a38 100644 --- a/crates/tw-gateway/src/affinity.rs +++ b/crates/tw-gateway/src/affinity.rs @@ -22,8 +22,7 @@ //! //! **带着上游封存内容的对话另当别论**(见 [`tw_dialect::sealed`]):那段历史只有封存它的 //! 那个账号读得出来,换到别家整个请求被拒,而且之后每一轮都被拒。所以每段对话还记着上次 -//! 回答它的那一家([`Affinity::holder`]),不管策略组、不管缓存凉没凉、配置改没改、那一家 -//! 中途没没了声音。怎么用它见 `server::pipeline::route`。 +//! 回答它的那一家([`Affinity::holder`])。用户修改路由后,通过 `route_switch` 校验密文的账号归属。 //! //! **只在内存里**,最多记 [`CAP`] 段、每段最多 [`KEEP_MS`](带过封存内容的 [`SEALED_KEEP_MS`]): //! core 重启之后第一轮按策略重新排,最坏是一次缓存重建 —— 带着封存内容的对话这时认不出 @@ -142,6 +141,25 @@ impl Conversation { turn, } } + + pub(crate) fn scoped( + mut self, + client: &crate::route_switch::ClientRoute, + strict: bool, + ) -> Self { + let revision = if strict && client.revision == 0 { + u64::MAX + } else { + client.revision + }; + self.key = format!( + "{}\n{}\n{}", + blake3::Hash::from_bytes(client.key), + revision, + self.key + ); + self + } } /// 一段对话记着的。 diff --git a/crates/tw-gateway/src/error.rs b/crates/tw-gateway/src/error.rs index a59aa340..ef4bf462 100644 --- a/crates/tw-gateway/src/error.rs +++ b/crates/tw-gateway/src/error.rs @@ -23,6 +23,7 @@ pub enum Source { Upstream, /// 请求本身有问题 Request, + RouteConflict, /// 上游说「慢点」:对面的额度到顶了。回 502 的话客户端会当成「服务器 /// 坏了」而不是「该退避了」,而它们该做的事完全不同。 RateLimited, @@ -76,6 +77,7 @@ impl Source { Source::Config => "config", Source::Upstream => "upstream", Source::Request => "request", + Source::RouteConflict => "route_conflict", Source::RateLimited | Source::Busy => "rate_limited", Source::Denied => "denied", Source::NotSupported | Source::TooLarge => "request", @@ -89,6 +91,7 @@ impl Source { Source::Config => StatusCode::INTERNAL_SERVER_ERROR, Source::Upstream => StatusCode::BAD_GATEWAY, Source::Request => StatusCode::BAD_REQUEST, + Source::RouteConflict => StatusCode::CONFLICT, // 429 而不是 503:客户端至少知道这是限流,可以退避。 Source::RateLimited | Source::Busy => StatusCode::TOO_MANY_REQUESTS, Source::Denied => StatusCode::FORBIDDEN, @@ -198,6 +201,15 @@ impl GatewayError { pub fn upstream(detail: Msg) -> Self { Self::new(Source::Upstream, detail) } + pub(crate) fn conflict(detail: Msg) -> Self { + Self::new(Source::RouteConflict, detail) + } + + pub(crate) fn route_changed() -> Self { + Self::conflict(tw_types::msg!("gw.route.changed" => + "The route changed before the request could be sent. Send the request again to use the current route.")) + } + pub fn request(detail: Msg) -> Self { Self::new(Source::Request, detail) } @@ -238,11 +250,20 @@ impl GatewayError { /// 客户端看到的是一个**戛然而止的流**,而截断和「答完了」在 SSE /// 里长得一模一样。帧的形状见 [`tw_dialect::convert::error_frame`]。 pub fn sse_frame(&self) -> String { - tw_dialect::convert::error_frame( + let frame = tw_dialect::convert::error_frame( self.dialect, self.source.status().as_u16(), &self.client_message(), - ) + ); + if self.source == Source::RouteConflict + && self.dialect == Dialect::Responses + && let Some(event) = tw_dialect::frame::parse(frame.as_bytes()) + && let Ok(mut value) = serde_json::from_str::(&event.data) + { + value["response"]["error"]["code"] = self.detail.code.clone().into(); + return format!("event: response.failed\ndata: {value}\n\n"); + } + frame } /// 这个错误的响应体,**客户端格式的形状**(见 @@ -258,12 +279,20 @@ impl GatewayError { &self.client_message(), ); // 自然周期用完了:OpenAI 格式里写成额度用完(见 [`Retry`]) - if self.retry.is_some_and(|r| r.until_reset) + let code = if self.source == Source::RouteConflict && self.dialect != Dialect::Gemini { + Some(self.detail.code.as_str()) + } else if self.retry.is_some_and(|r| r.until_reset) && matches!(self.dialect, Dialect::Chat | Dialect::Responses) + { + Some("insufficient_quota") + } else { + None + }; + if let Some(code) = code && let Ok(mut v) = serde_json::from_slice::(&body) && let Some(e) = v.get_mut("error").and_then(|e| e.as_object_mut()) { - e.insert("code".into(), "insufficient_quota".into()); + e.insert("code".into(), code.into()); return v.to_string().into_bytes(); } body @@ -282,6 +311,12 @@ impl IntoResponse for GatewayError { "x-thinkwatch-error", HeaderValue::from_static(self.source.slug()), ); + if self.source == Source::RouteConflict { + h.insert("x-should-retry", HeaderValue::from_static("false")); + if let Ok(code) = HeaderValue::from_str(&self.detail.code) { + h.insert("x-thinkwatch-error-code", code); + } + } if let Some(r) = self.retry { let secs = r.after_ms.div_ceil(1000).max(1); h.insert(header::RETRY_AFTER, HeaderValue::from(secs)); diff --git a/crates/tw-gateway/src/handover.rs b/crates/tw-gateway/src/handover.rs index 2e849fd5..c6c9b5b9 100644 --- a/crates/tw-gateway/src/handover.rs +++ b/crates/tw-gateway/src/handover.rs @@ -161,6 +161,10 @@ impl AppState { wire::AFFINITY, serde_json::to_value(self.affinity.handover_export()), ); + put( + wire::ROUTE_HISTORY, + serde_json::to_value(self.route_history.export(&self.runtime().routing)), + ); put( wire::SEALS, serde_json::to_value(self.seals.handover_export()), @@ -224,6 +228,11 @@ impl AppState { .map_or(0, |at| crate::server::now_ms().saturating_sub(at)); let cfg = self.config(); self.affinity.handover_import(entries(m, wire::AFFINITY)); + if let Some(history) = section(m, wire::ROUTE_HISTORY) { + let rt = self.runtime(); + self.route_history.import(&rt.routing, history); + self.route_history.sync_routing(&rt.routing); + } self.seals.handover_import(entries(m, wire::SEALS)); self.cache_marks .handover_import(entries(m, wire::CACHE_MARKS)); diff --git a/crates/tw-gateway/src/handover/wire.rs b/crates/tw-gateway/src/handover/wire.rs index 0c098857..f0705ee9 100644 --- a/crates/tw-gateway/src/handover/wire.rs +++ b/crates/tw-gateway/src/handover/wire.rs @@ -15,6 +15,8 @@ use crate::key_limits::books::{Amount, Period}; /// 快照里各节的名字。**一节读不懂只丢那一节**(见 [`crate::state::AppState::handover_import`]) pub(crate) const AT_MS: &str = "at_ms"; pub(crate) const AFFINITY: &str = "affinity"; +/// Optional section. Older peers can skip it. +pub(crate) const ROUTE_HISTORY: &str = "route_history"; pub(crate) const SEALS: &str = "seals"; pub(crate) const CACHE_MARKS: &str = "cache_marks"; pub(crate) const SESSIONS: &str = "sessions"; diff --git a/crates/tw-gateway/src/lib.rs b/crates/tw-gateway/src/lib.rs index 7f93c3c9..ba37777d 100644 --- a/crates/tw-gateway/src/lib.rs +++ b/crates/tw-gateway/src/lib.rs @@ -57,6 +57,7 @@ pub mod quota; pub mod quote; pub mod resources; pub mod retry; +mod route_switch; pub mod seal; pub mod sent; pub mod server; diff --git a/crates/tw-gateway/src/oauth.rs b/crates/tw-gateway/src/oauth.rs index 73ec8f67..e64cb906 100644 --- a/crates/tw-gateway/src/oauth.rs +++ b/crates/tw-gateway/src/oauth.rs @@ -338,6 +338,20 @@ impl Cache { .or_else(|| cfg.access.clone().filter(|a| !a.is_empty())) } + /// A credential written back by our own refresh is still the same account. + pub(crate) fn wrote_rotation(&self, name: &str, old: &OAuth, new: &OAuth) -> bool { + self.inner + .lock() + .unwrap_or_else(|p| p.into_inner()) + .get(name) + .is_some_and(|live| { + live.obtained.is_some() + && live.fp == fingerprint(old) + && live.refresh == new.refresh + && new.access.as_deref() == Some(live.access.as_str()) + }) + } + /// 这家最近一次刷新失败的原因,和是不是要重新登录。没失败过、或者已经恢复,是 `None`。 pub fn failure(&self, provider: &str, cfg: &OAuth) -> Option<(Msg, bool)> { let g = self.inner.lock().expect("lock not poisoned"); diff --git a/crates/tw-gateway/src/route_switch.rs b/crates/tw-gateway/src/route_switch.rs new file mode 100644 index 00000000..bdc9075d --- /dev/null +++ b/crates/tw-gateway/src/route_switch.rs @@ -0,0 +1,824 @@ +//! Live routing revisions and account provenance for upstream-bound history. + +use std::collections::{HashMap, HashSet}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +use serde_json::Value; +use tw_config::{Config, Provider}; +use tw_dialect::ir::Dialect; +use tw_dialect::sealed::Sealed; +use tw_engine::{Engine, Group, GroupType, Target}; +use tw_types::msg; + +use crate::error::GatewayError; +use crate::state::{AppState, Runtime}; + +type Fingerprint = [u8; 32]; + +fn hash(value: impl AsRef<[u8]>) -> Fingerprint { + *blake3::hash(value.as_ref()).as_bytes() +} + +#[derive(Clone)] +pub(crate) struct ClientRoute { + pub key: Fingerprint, + shape: Fingerprint, + pub revision: u64, +} + +#[derive(Clone)] +pub(crate) struct Routing { + revision: u64, + clients: HashMap, + accounts: HashMap, +} + +#[derive(Clone, Copy)] +struct AccountIdentity { + configured: Fingerprint, + without_refresh: Option, + verified: Fingerprint, +} + +impl Routing { + pub fn build( + config: &Config, + engine: &Engine, + previous: Option<&Runtime>, + oauth: Option<&crate::oauth::Cache>, + ) -> Self { + let revision = previous.map_or(0, |rt| rt.routing.revision + 1); + let mut known = HashMap::new(); + if let Some(rt) = previous { + for old in &rt.config.providers { + if let Some(identity) = rt.routing.accounts.get(&old.name) { + known.insert(identity.configured, identity.verified); + } + } + if let Some(oauth) = oauth { + for provider in config.providers.iter().filter(|p| p.oauth.is_some()) { + let Some(old) = rt.config.providers.iter().find(|p| p.name == provider.name) + else { + continue; + }; + if let (Some(was), Some(now)) = (&old.oauth, &provider.oauth) + && was.refresh != now.refresh + && let Some(identity) = rt.routing.accounts.get(&old.name) + && identity.without_refresh == Some(account(provider, false)) + && oauth.wrote_rotation(&provider.name, was, now) + { + known.insert(account(provider, true), identity.verified); + } + } + } + } + let accounts: HashMap<_, _> = config + .providers + .iter() + .map(|p| { + let configured = account(p, true); + let without_refresh = p.oauth.as_ref().map(|_| account(p, false)); + let verified = known.get(&configured).copied().unwrap_or(configured); + ( + p.name.clone(), + AccountIdentity { + configured, + without_refresh, + verified, + }, + ) + }) + .collect(); + let clients = config + .clients + .iter() + .filter(|client| !client.disabled) + .map(|client| { + let key = hash(&client.key); + let route = engine.route_of(&client.name); + let rules = engine.rules_for_client(&client.name); + let mut names = HashSet::new(); + let mut groups = Vec::new(); + for target in rules.iter().filter_map(|rule| rule.to.as_ref()) { + match target { + Target::Name(name) => { + if let Some(group) = engine.groups().iter().find(|g| g.name == *name) { + groups.push(group); + if group.kind == GroupType::Select { + names.extend(group.selection()); + } else { + names.extend(group.providers.iter().map(|m| m.name.as_str())); + } + } else { + names.insert(name.as_str()); + } + } + Target::Models(models) => { + names.extend(models.iter().map(|p| p.provider.as_str())); + } + } + } + let providers: Vec<_> = config + .providers + .iter() + .filter(|p| names.contains(p.name.as_str())) + .map(|p| { + ( + &p.name, + accounts[&p.name].verified, + p.disabled, + &p.models, + &p.models_only, + ) + }) + .collect(); + let shape = hash( + serde_json::to_vec(&( + route, + rules, + groups, + providers, + &config.aliases, + &client.allow, + config.web_tools.mode, + )) + .expect("routing configuration serializes"), + ); + let revision = previous + .and_then(|rt| rt.routing.client(&client.name)) + .filter(|old| old.key == key && old.shape == shape) + .map_or(revision, |old| old.revision); + ( + client.name.clone(), + ClientRoute { + key, + shape, + revision, + }, + ) + }) + .collect(); + Self { + revision, + clients, + accounts, + } + } + + pub fn client(&self, name: &str) -> Option<&ClientRoute> { + self.clients.get(name) + } + + pub fn account(&self, provider: &str) -> Option { + self.accounts + .get(provider) + .map(|identity| identity.verified) + } +} + +impl ClientRoute { + pub fn matches(&self, rt: &Runtime, client: &str) -> bool { + rt.routing + .client(client) + .is_some_and(|now| now.key == self.key && now.revision == self.revision) + } + + pub fn current(&self, state: &AppState, client: &str) -> bool { + self.matches(&state.runtime(), client) + } +} + +pub(crate) fn manual(engine: &Engine, group: Option<&str>) -> bool { + group + .and_then(|name| engine.groups().iter().find(|g| g.name == name)) + .is_some_and(|g: &Group| g.kind == GroupType::Select) +} + +fn account(provider: &Provider, include_refresh: bool) -> Fingerprint { + let token = provider + .key + .as_ref() + .map(|key| key.resolve().unwrap_or_else(|_| key.raw().into())); + let mut headers: Vec<_> = provider + .headers + .iter() + .map(|header| { + ( + header.name.trim().to_ascii_lowercase(), + header + .value + .resolve() + .unwrap_or_else(|_| header.value.raw().into()), + ) + }) + .collect(); + headers.sort(); + let oauth = provider.oauth.as_ref().map(|oauth| { + // ChatGPT config has a stable account identifier even as its tokens rotate. + let has_account = headers + .iter() + .any(|(name, value)| name == "chatgpt-account-id" && !value.is_empty()); + ( + &oauth.endpoint, + &oauth.client_id, + (include_refresh && !has_account).then_some(&oauth.refresh), + ) + }); + hash( + serde_json::to_vec(&( + provider.base_url.trim_end_matches('/'), + provider.effective_protocol(), + token, + headers, + oauth, + &provider.aws, + )) + .expect("account configuration serializes"), + ) +} + +#[derive(Clone, Copy)] +pub(crate) enum Kind { + Reasoning, + Content, + Stored, +} + +impl Kind { + pub fn conflict(self) -> GatewayError { + GatewayError::conflict(match self { + Self::Reasoning => msg!("gw.route.history.reasoning" => + "The selected upstream account cannot be verified to read this conversation's encrypted reasoning. Start a new conversation on the selected route."), + Self::Content => msg!("gw.route.history.content" => + "The selected upstream account cannot be verified to read this conversation's encrypted history, such as compacted context or tool results. Start a new conversation on the selected route."), + Self::Stored => msg!("gw.route.history.stored" => + "This request references history stored by an upstream account. It cannot be verified for the selected account. Start a new conversation on the selected route."), + }) + } +} + +pub(crate) struct Required { + pub kind: Kind, + artifacts: Vec, + unknown: bool, +} + +fn artifact(namespace: &str, text: &str) -> Fingerprint { + let mut h = blake3::Hasher::new(); + h.update(namespace.as_bytes()) + .update(b"\0") + .update(text.as_bytes()); + *h.finalize().as_bytes() +} + +impl Required { + pub fn read(dialect: Dialect, body: &Value) -> Option { + if dialect == Dialect::Anthropic { + let mut artifacts = Vec::new(); + for message in body["messages"].as_array().into_iter().flatten() { + for block in message["content"].as_array().into_iter().flatten() { + anthropic_block(block, &mut artifacts); + } + } + return (!artifacts.is_empty()).then_some(Self { + kind: Kind::Reasoning, + artifacts, + unknown: false, + }); + } + if dialect != Dialect::Responses { + return None; + } + let mut artifacts = Vec::new(); + let mut unknown = false; + let mut stored = false; + for name in ["previous_response_id", "conversation", "prompt"] { + let v = &body[name]; + if v.is_null() || v.as_str() == Some("") { + continue; + } + stored = true; + if let Some(id) = v.as_str().or_else(|| v["id"].as_str()) + && name != "prompt" + { + let namespace = if name == "previous_response_id" { + "response" + } else { + name + }; + artifacts.push(artifact(namespace, id)); + } else { + unknown = true; + } + } + if let Some(input) = body["input"].as_array() { + for item in input { + if item["type"] == "item_reference" { + stored = true; + if let Some(id) = item["id"].as_str() { + artifacts.push(artifact("item", id)); + } else { + unknown = true; + } + } + ciphertext(item, &mut artifacts); + } + } + let kind = match (stored, tw_dialect::sealed::sealed(body)) { + (true, _) => Kind::Stored, + (false, Some(Sealed::Content)) => Kind::Content, + (false, Some(Sealed::Reasoning)) => Kind::Reasoning, + _ => return None, + }; + Some(Self { + kind, + artifacts, + unknown, + }) + } +} + +fn anthropic_block(block: &Value, out: &mut Vec) { + let (field, namespace) = match block["type"].as_str() { + Some("thinking") => ("signature", "anthropic/sig"), + Some("redacted_thinking") => ("data", "anthropic_redacted"), + _ => return, + }; + if let Some(value) = block[field].as_str() + && !value.is_empty() + && !value.starts_with(tw_dialect::ir::CARRIED) + { + out.push(artifact(namespace, value)); + } +} + +fn ciphertext(value: &Value, out: &mut Vec) { + match value { + Value::Object(fields) => { + for (name, value) in fields { + if name == "encrypted_content" + && let Some(s) = value.as_str() + && !s.is_empty() + && !s.starts_with(tw_dialect::ir::CARRIED) + { + out.push(artifact("ciphertext", s)); + } + ciphertext(value, out); + } + } + Value::Array(values) => { + for value in values { + ciphertext(value, out); + } + } + _ => {} + } +} + +const CAP: usize = 65_536; +const SNAPSHOT_LIMIT: usize = 4096; +const KEEP_MS: u64 = crate::affinity::SEALED_KEEP_MS; + +#[derive(Clone, Copy, serde::Serialize, serde::Deserialize)] +struct Origin { + account: Option, + at: u64, +} + +#[derive(Default)] +pub(crate) struct History { + entries: Mutex>, + inherited_cuts: Mutex>, + disk: Mutex>, + save_queued: AtomicBool, +} + +type RouteSnapshot = Vec<(Fingerprint, Fingerprint, bool)>; + +#[derive(serde::Serialize, serde::Deserialize)] +pub(crate) struct Snapshot { + routes: RouteSnapshot, + origins: Vec<(Fingerprint, Fingerprint, Origin)>, +} + +impl History { + pub fn strict(&self, route: &ClientRoute) -> bool { + route.revision != 0 + || self + .inherited_cuts + .lock() + .unwrap_or_else(|p| p.into_inner()) + .contains(&route.key) + } + + fn routes(&self, routing: &Routing) -> RouteSnapshot { + let mut routes: RouteSnapshot = routing + .clients + .values() + .map(|route| (route.key, route.shape, self.strict(route))) + .collect(); + routes.sort_unstable_by_key(|(key, _, _)| *key); + routes + } + + pub fn export(&self, routing: &Routing) -> Snapshot { + self.snapshot(self.routes(routing)) + } + + fn snapshot(&self, routes: RouteSnapshot) -> Snapshot { + let now = crate::server::now_ms(); + let mut origins: Vec<_> = self + .entries + .lock() + .unwrap_or_else(|p| p.into_inner()) + .iter() + .filter(|(_, origin)| now.saturating_sub(origin.at) < KEEP_MS) + .map(|((key, id), origin)| (*key, *id, *origin)) + .collect(); + origins.sort_unstable_by_key(|(_, _, origin)| std::cmp::Reverse(origin.at)); + origins.truncate(SNAPSHOT_LIMIT); + Snapshot { routes, origins } + } + + /// Keep enough proof to enforce a cut across a restart. The file contains + /// fingerprints only and is written atomically when routes change. + pub fn set_dir(&self, dir: &Path, routing: &Routing) { + let path = dir.join("route-history.json"); + let saved: Option = std::fs::metadata(&path) + .ok() + .filter(|m| m.len() <= 4 * 1024 * 1024) + .and_then(|_| std::fs::read(&path).ok()) + .and_then(|bytes| serde_json::from_slice(&bytes).ok()); + if let Some(snapshot) = saved { + self.import(routing, snapshot); + } + *self.disk.lock().unwrap_or_else(|p| p.into_inner()) = Some((path, Vec::new())); + self.sync_routing(routing); + } + + pub fn sync_routing(&self, routing: &Routing) { + let routes = self.routes(routing); + // The safe-mode fallback has no authenticated clients. + if routes.is_empty() { + return; + } + let mut disk = self.disk.lock().unwrap_or_else(|p| p.into_inner()); + let Some((path, saved)) = disk.as_mut() else { + return; + }; + if *saved == routes { + return; + } + if self.write_snapshot(path, &routes) { + *saved = routes; + } + } + + fn write_snapshot(&self, path: &Path, routes: &RouteSnapshot) -> bool { + let Ok(json) = serde_json::to_string(&self.snapshot(routes.clone())) else { + return false; + }; + match tw_config::store::write_atomic(path, &json) { + Ok(_) => true, + Err(error) => { + tracing::warn!("could not save routing history fingerprints: {error}"); + false + } + } + } + + // Output can arrive every few milliseconds. Group the writes and keep all + // file I/O off the response path. + fn queue_save(self: &Arc) { + if self.save_queued.load(Ordering::Acquire) + || self + .disk + .lock() + .unwrap_or_else(|p| p.into_inner()) + .is_none() + || self.save_queued.swap(true, Ordering::AcqRel) + { + return; + } + let history = self.clone(); + tokio::spawn(async move { + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + let _ = tokio::task::spawn_blocking(move || { + history.save_queued.store(false, Ordering::Release); + let disk = history.disk.lock().unwrap_or_else(|p| p.into_inner()); + if let Some((path, routes)) = disk.as_ref().filter(|(_, routes)| !routes.is_empty()) + { + history.write_snapshot(path, routes); + } + }) + .await; + }); + } + + pub fn import(&self, routing: &Routing, snapshot: Snapshot) { + let shapes: HashMap<_, _> = routing + .clients + .values() + .map(|route| (route.key, route.shape)) + .collect(); + self.inherited_cuts + .lock() + .unwrap_or_else(|p| p.into_inner()) + .extend( + snapshot + .routes + .into_iter() + .filter_map(|(key, shape, strict)| { + shapes + .get(&key) + .is_some_and(|current| strict || *current != shape) + .then_some(key) + }), + ); + let mut entries = self.entries.lock().unwrap_or_else(|p| p.into_inner()); + for (key, id, imported) in snapshot.origins.into_iter().take(SNAPSHOT_LIMIT) { + if entries.len() >= CAP { + break; + } + entries + .entry((key, id)) + .and_modify(|origin| { + if origin.account != imported.account { + origin.account = None; + } + origin.at = origin.at.max(imported.at); + }) + .or_insert(imported); + } + } + + pub fn check( + &self, + required: &Required, + key: Fingerprint, + account: Fingerprint, + strict: bool, + ) -> Result<(), GatewayError> { + let now = crate::server::now_ms(); + let entries = self.entries.lock().unwrap_or_else(|p| p.into_inner()); + let readable = (!strict || !required.unknown) + && required.artifacts.iter().all(|id| { + entries + .get(&(key, *id)) + .filter(|origin| now.saturating_sub(origin.at) < KEEP_MS) + .map_or(!strict, |origin| origin.account == Some(account)) + }); + if readable { + Ok(()) + } else { + Err(required.kind.conflict()) + } + } + + pub fn record(self: &Arc, key: Fingerprint, account: Fingerprint, value: &Value) { + let mut artifacts = Vec::new(); + match value["type"].as_str() { + Some("response.output_item.done") => output_item(&value["item"], &mut artifacts), + Some("response.created" | "response.completed" | "response.incomplete") => { + output_response(&value["response"], &mut artifacts); + } + _ if value["object"] == "response" || value.get("output").is_some() => { + output_response(value, &mut artifacts); + } + _ => {} + } + self.record_artifacts(key, account, artifacts); + } + + fn record_artifacts( + self: &Arc, + key: Fingerprint, + account: Fingerprint, + artifacts: Vec, + ) { + if artifacts.is_empty() { + return; + } + let now = crate::server::now_ms(); + let mut entries = self.entries.lock().unwrap_or_else(|p| p.into_inner()); + if entries.len() + artifacts.len() > CAP { + entries.retain(|_, origin| now.saturating_sub(origin.at) < KEEP_MS); + if entries.len() + artifacts.len() > CAP { + let mut newest: Vec<_> = entries.drain().collect(); + newest.sort_unstable_by_key(|(_, origin)| std::cmp::Reverse(origin.at)); + entries.extend(newest.into_iter().take(CAP / 2)); + } + } + use std::collections::hash_map::Entry; + let mut changed = false; + for id in artifacts { + if entries.len() >= CAP && !entries.contains_key(&(key, id)) { + break; + } + match entries.entry((key, id)) { + Entry::Occupied(mut entry) => { + let origin = entry.get_mut(); + if origin.account.is_some_and(|old| old != account) { + origin.account = None; + changed = true; + } + origin.at = now; + } + Entry::Vacant(entry) => { + entry.insert(Origin { + account: Some(account), + at: now, + }); + changed = true; + } + } + } + drop(entries); + if changed { + self.queue_save(); + } + } +} + +fn output_response(response: &Value, out: &mut Vec) { + if let Some(id) = response["id"].as_str() { + out.push(artifact("response", id)); + } + // A response's own ID and output were issued here. Any echoed conversation + // reference is already an input dependency and is left out of this record. + if let Some(items) = response["output"].as_array() { + for item in items { + output_item(item, out); + } + } +} + +fn output_item(item: &Value, out: &mut Vec) { + if let Some(id) = item["id"].as_str() { + out.push(artifact("item", id)); + } + ciphertext(item, out); +} + +/// Observe Responses output without changing what is sent to the client. +pub(crate) struct Capture { + history: std::sync::Arc, + key: Fingerprint, + account: Fingerprint, + dialect: Dialect, + sse: bool, + signatures: HashMap, + pending: Vec, + overflow: bool, +} + +impl Capture { + pub fn new( + state: &AppState, + rt: &Runtime, + key: &str, + provider: &str, + dialect: Dialect, + sse: bool, + ) -> Option { + Some(Self { + history: state.route_history.clone(), + key: rt.routing.client(key)?.key, + account: rt.routing.account(provider)?, + dialect, + sse, + signatures: HashMap::new(), + pending: Vec::new(), + overflow: false, + }) + } + + pub fn feed(&mut self, chunk: &[u8]) { + const MAX_CAPTURE: usize = 4 * 1024 * 1024; + if self.overflow { + return; + } + if self.pending.len().saturating_add(chunk.len()) > MAX_CAPTURE { + self.pending.clear(); + self.overflow = true; + return; + } + self.pending.extend_from_slice(chunk); + if self.sse { + let mut read = 0; + while let Some((end, sep)) = tw_dialect::frame::frame_end(&self.pending[read..]) { + if let Some(frame) = tw_dialect::frame::parse(&self.pending[read..read + end]) + && let Ok(value) = serde_json::from_str::(&frame.data) + { + self.record(value); + } + read += end + sep; + } + self.pending.drain(..read); + } else if let Ok(value) = serde_json::from_slice::(&self.pending) { + self.record(value); + self.pending.clear(); + } + } + + pub fn finish(&mut self) { + if self.sse + && let Some(frame) = tw_dialect::frame::parse(&self.pending) + && let Ok(value) = serde_json::from_str::(&frame.data) + { + self.record(value); + } + } + + fn record(&mut self, value: Value) { + if self.dialect == Dialect::Responses { + self.history.record(self.key, self.account, &value); + return; + } + let mut artifacts = Vec::new(); + match value["type"].as_str() { + Some("content_block_start") => anthropic_block(&value["content_block"], &mut artifacts), + Some("content_block_delta") if value["delta"]["type"] == "signature_delta" => { + if let (Some(index), Some(delta)) = ( + value["index"].as_u64(), + value["delta"]["signature"].as_str(), + ) && (self.signatures.contains_key(&index) || self.signatures.len() < 32) + { + let signature = self.signatures.entry(index).or_default(); + if signature.len() + delta.len() <= 64 * 1024 { + signature.push_str(delta); + } + } + } + Some("content_block_stop") => { + if let Some(signature) = value["index"] + .as_u64() + .and_then(|i| self.signatures.remove(&i)) + { + anthropic_block( + &serde_json::json!({"type":"thinking","signature":signature}), + &mut artifacts, + ); + } + } + Some("message") => { + for block in value["content"].as_array().into_iter().flatten() { + anthropic_block(block, &mut artifacts); + } + } + _ => {} + } + self.history + .record_artifacts(self.key, self.account, artifacts); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn signed_anthropic_history_matches_only_the_issuing_account() { + let key = hash("key"); + let owner = hash("account-a"); + let history = Arc::new(History::default()); + let mut capture = Capture { + history: history.clone(), + key, + account: owner, + dialect: Dialect::Anthropic, + sse: true, + signatures: HashMap::new(), + pending: Vec::new(), + overflow: false, + }; + for value in [ + json!({"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"part1"}}), + json!({"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"part2"}}), + json!({"type":"content_block_stop","index":0}), + json!({"type":"content_block_start","index":1,"content_block":{"type":"redacted_thinking","data":"sealed"}}), + ] { + let frame = format!("data: {value}\n\n"); + for chunk in frame.as_bytes().chunks(7) { + capture.feed(chunk); + } + } + let body = json!({"messages":[{"role":"assistant","content":[ + {"type":"thinking","thinking":"...","signature":"part1part2"}, + {"type":"redacted_thinking","data":"sealed"} + ]}]}); + let required = Required::read(Dialect::Anthropic, &body).unwrap(); + assert!(history.check(&required, key, owner, true).is_ok()); + assert_eq!( + history + .check(&required, key, hash("account-b"), true) + .unwrap_err() + .detail + .code, + "gw.route.history.reasoning" + ); + } +} diff --git a/crates/tw-gateway/src/server.rs b/crates/tw-gateway/src/server.rs index ca3a43f9..35a5125b 100644 --- a/crates/tw-gateway/src/server.rs +++ b/crates/tw-gateway/src/server.rs @@ -129,7 +129,7 @@ async fn passthrough( ), )); } - let (client_name, position) = state.identify(&headers, query.as_deref())?; + let (client_name, position) = AppState::identify_in(&rt, &headers, query.as_deref())?; // 除了密钥的名字,这条请求是谁发的 let from = Sender { peer: crate::hint::peer_of(peer.ip()), @@ -153,6 +153,7 @@ async fn passthrough( started, live, from, + peer.ip(), ) .await; } diff --git a/crates/tw-gateway/src/server/pipeline.rs b/crates/tw-gateway/src/server/pipeline.rs index 378302f8..94c16f78 100644 --- a/crates/tw-gateway/src/server/pipeline.rs +++ b/crates/tw-gateway/src/server/pipeline.rs @@ -128,7 +128,8 @@ pub(super) async fn pipeline( match plan { Ok(Some(plan)) => { let (reading, _) = read(&req, String::new(), Some(&body), (state.local_time)()); - let early = commit::early(&state, &req, &reading); + let early = commit::early(&state, &req, &reading) + .filter(|_| crate::route_switch::Required::read(req.dialect, &body).is_none()); let dialect = req.dialect; let abort = req.abort.clone(); let run = web::run(state, rt, req, live, plan, body); @@ -175,8 +176,18 @@ async fn forward_pipeline( read(&req, intent, parsed.as_ref(), (state.local_time)()) }) .await; - let conv = conversation(&rt, &req, &reading, fp.as_deref()); - let (choice, decision) = match route(&state, &rt, &req, &reading, conv.as_ref()) { + let conv = conversation(&state, &rt, &req, &reading, fp.as_deref()); + let required = parsed + .as_ref() + .and_then(|body| crate::route_switch::Required::read(req.dialect, body)); + let (choice, decision) = match route( + &state, + &rt, + &req, + &reading, + conv.as_ref(), + required.as_ref(), + ) { Routed::Go(choice, decision) => (choice, decision), // 请求一家上游都不会去:还没配上游、规则没做出决定、规则拒绝了、模型准入没过、规则 // 选中的上游都服务不了。**照样开始、照样报路由**(尝试链是空的),失败由 @@ -595,6 +606,7 @@ fn read( /// **数得出轮次的才算**:要解码出整段对话才知道最后一条是不是工具结果。计 token、 /// 嵌入、解不开的请求不记 —— 它们照常按策略排序。 fn conversation( + state: &AppState, rt: &Runtime, req: &Inbound, reading: &crate::client_api::Reading, @@ -604,11 +616,15 @@ fn conversation( return None; }; let id = crate::affinity::identity(&req.headers, fp)?; - Some(crate::affinity::Conversation::new( - rt.engine.route_of(&req.client_name), - &id, - crate::affinity::turn_of(&d.request), - )) + let client = rt.routing.client(&req.client_name)?; + Some( + crate::affinity::Conversation::new( + rt.engine.route_of(&req.client_name), + &id, + crate::affinity::turn_of(&d.request), + ) + .scoped(client, state.route_history.strict(client)), + ) } /// 输入超出了这个决定所选模型的上下文窗口:这一轮沿用的决定要重新求值。 @@ -836,6 +852,7 @@ fn route( req: &Inbound, reading: &crate::client_api::Reading, conv: Option<&crate::affinity::Conversation>, + required: Option<&crate::route_switch::Required>, ) -> Routed { let facts = &reading.facts; let route = rt.engine.route_of(&req.client_name).to_string(); @@ -849,6 +866,13 @@ fn route( why, ) }; + let Some(client_route) = rt + .routing + .client(&req.client_name) + .filter(|client| client.current(state, &req.client_name)) + else { + return undecided(GatewayError::route_changed()); + }; // 首次运行还没配完是正常状态,不是配置错误。这条要在规则求值之前挡, // 因为「一个 provider 都没有」时任何路由结果都是空的,而那条错误 // 说不清下一步。 @@ -912,11 +936,13 @@ fn route( }), stayed_on: None, }; - // 带着上游封存内容的对话,只有封存它的那一家读得出来(见 `tw_dialect::sealed`)。认得出是 - // 谁封存的(上次回答它的那一家,见 `crate::affinity`),就送回它那儿:规则这一次选了谁、 - // 配置改没改、缓存凉没凉都不管。认不出来的照常路由 —— 被拒了,失败的说明会讲清楚 + let strict_history = state.route_history.strict(client_route) + || crate::route_switch::manual(&rt.engine, decision.via_group.as_deref()); + // Ordinary automatic routing keeps its existing affinity. Once the + // operator changes the route, proof comes from the actual response items. let sealed = reading .sealed + .filter(|_| !strict_history) .zip(conv) .and_then(|(kind, c)| Some((kind, state.affinity.holder(c, now)?))); if let Some((tw_dialect::sealed::Sealed::Content, holder)) = &sealed { @@ -975,6 +1001,23 @@ fn route( conv, now, ); + if let Some(required) = required.filter(|_| strict_history) { + // Use the same health filtering as dispatch before deciding which + // account must be able to read this request. + let (alive, _) = state.health.filter(&decision.candidates); + if let Some(leader) = alive.first() { + let account = rt.routing.account(leader).expect("usable provider"); + if let Err(err) = state + .route_history + .check(required, client_route.key, account, true) + { + return Routed::Refused(choice, err); + } + decision + .candidates + .retain(|name| rt.routing.account(name) == Some(account)); + } + } let mut stayed = arranged.stayed; // 封存的只有推理:封存它的那一家排到头上,推理接得上;别家照旧留着做故障转移 —— 去掉推理 // 也接得下去(见 `crate::seal`) diff --git a/crates/tw-gateway/src/server/pipeline/hop.rs b/crates/tw-gateway/src/server/pipeline/hop.rs index 29328ccd..5f08540f 100644 --- a/crates/tw-gateway/src/server/pipeline/hop.rs +++ b/crates/tw-gateway/src/server/pipeline/hop.rs @@ -256,7 +256,17 @@ pub(super) async fn try_upstreams<'a>( // 这一跳发出去的推理强度和客户端要的不一样:那一跳在尝试链上的位置和那一项 let mut effort_note: Option<(usize, tw_api::EffortChange)> = None; + let route = rt + .routing + .client(&req.client_name) + .expect("authenticated key"); + let strict_history = state.route_history.strict(route) + || crate::route_switch::manual(&rt.engine, decision.via_group.as_deref()); loop { + if !route.current(state, &req.client_name) { + halt = Some(GatewayError::route_changed()); + break; + } stamp_queued(&mut chain, queued.take()); stamp_effort(&mut chain, effort_note.take()); // 候选都试过了,最后一跳在内容之前暂时出错:等待期限之内退避一阵,把暂时出错的那几家再 @@ -803,7 +813,11 @@ pub(super) async fn try_upstreams<'a>( .map_or(&started.ledger, |r| &r.ledger); let (body, ledger, differs) = super::heavy(&out.body, || { // 这一家在这段对话里拒过的别家封存的推理:发之前先去掉(见 `crate::seal`) - let unsealed = unseal_upfront(state, req, started, provider, &out); + let unsealed = if strict_history { + out.body.clone() + } else { + unseal_upfront(state, req, started, provider, &out) + }; // 发出去的就是客户端原文的那些字节(同格式直通、一个字节都没改):开头那一遍在 // 它上面找到的就是这一跳要换的,不再找一遍 let known = started.hits.as_deref().filter(|_| { @@ -939,6 +953,7 @@ pub(super) async fn try_upstreams<'a>( let sending = async { let sent = send( state, + route, req, provider, http, @@ -953,6 +968,7 @@ pub(super) async fn try_upstreams<'a>( match sent { Ok(r) => { let resend = Resend { + route, out: &out, body: &body, headers: &upstream_headers, @@ -971,6 +987,7 @@ pub(super) async fn try_upstreams<'a>( ) .await?; let resend = Resend { + route, out: &out, body: uncached.as_ref().unwrap_or(&body), headers: &upstream_headers, @@ -978,8 +995,12 @@ pub(super) async fn try_upstreams<'a>( conversation: started.conversation.as_deref(), tap: &tap, }; - resend_unsealed(state, req, provider, http, resend, r, &mut repair_used) - .await + if strict_history { + Ok(r) + } else { + resend_unsealed(state, req, provider, http, resend, r, &mut repair_used) + .await + } } Err(e) => Err(e), } @@ -1777,6 +1798,16 @@ pub(super) async fn try_upstreams<'a>( }); break; } + Err(SendError::RouteChanged { dispatched }) => { + if !dispatched { + usage_hold.release(); + } + if let Some(p) = probe.take() { + p.withdraw(); + } + halt = Some(GatewayError::route_changed()); + break; + } Err(e) => { note_health( &state.bus, @@ -1785,6 +1816,7 @@ pub(super) async fn try_upstreams<'a>( state.health.record_failure(&provider.name), ); let err = match e { + SendError::RouteChanged { .. } => unreachable!("handled above"), // 连不上:上游一个字节都没收到,这一跳不算 SendError::Http(e) if e.is_connect() => { usage_hold.release(); @@ -2964,6 +2996,7 @@ fn unseal_upfront( /// 重发一次要用的:这一跳原来发的那些 struct Resend<'a> { + route: &'a crate::route_switch::ClientRoute, out: &'a Outbound, body: &'a Bytes, headers: &'a [(String, String)], @@ -3018,6 +3051,7 @@ async fn resend_uncached( let stripped = Bytes::from(stripped); let r = send( state, + resend.route, req, provider, http, @@ -3027,7 +3061,8 @@ async fn resend_uncached( resend.aws, &resend.tap.resent(), ) - .await?; + .await + .map_err(SendError::after_dispatch)?; Ok((r, Some(stripped))) } @@ -3076,6 +3111,7 @@ async fn resend_unsealed( ); send( state, + resend.route, req, provider, http, @@ -3086,6 +3122,7 @@ async fn resend_unsealed( &resend.tap.resent(), ) .await + .map_err(SendError::after_dispatch) } /// 这一跳发往的完整地址。 @@ -3105,6 +3142,18 @@ enum SendError { Http(reqwest::Error), /// Bedrock 的请求没签成名 Sign(tw_bedrock::SignError), + RouteChanged { + dispatched: bool, + }, +} + +impl SendError { + fn after_dispatch(self) -> Self { + match self { + Self::RouteChanged { .. } => Self::RouteChanged { dispatched: true }, + other => other, + } + } } /// 发出这一跳。OAuth 上游回 401 时换一个 token 再发一次。 @@ -3115,6 +3164,7 @@ enum SendError { #[allow(clippy::too_many_arguments)] async fn send( state: &AppState, + route: &crate::route_switch::ClientRoute, req: &Inbound, provider: &tw_config::Provider, http: &reqwest::Client, @@ -3201,6 +3251,16 @@ async fn send( .map(|c| (c, provider.bedrock_region().unwrap_or_default())); // 发出去的请求体按线上的样子记:压缩过的是压缩之后的(见 `crate::traffic`) let (traffic, size) = (&req.traffic, body.len()); + let dispatch = |request| { + dispatch( + state, + route, + &req.client_name, + traffic, + size, + http.execute(request), + ) + }; if let Some((credentials, region)) = signer { let mut request = build(&upstream_headers) .body(body.clone()) @@ -3208,9 +3268,7 @@ async fn send( .map_err(SendError::Http)?; crate::bedrock::sign_request(&mut request, credentials, region).map_err(SendError::Sign)?; tap.sending(&request, &plain, &upstream_headers); - return dispatch(traffic, size, http.execute(request)) - .await - .map_err(SendError::Http); + return dispatch(request).await; } // 定稿了再发:报文记录记下真正发出去的请求头 let request = build(&upstream_headers) @@ -3218,7 +3276,7 @@ async fn send( .build() .map_err(SendError::Http)?; tap.sending(&request, &plain, &upstream_headers); - let mut sent = dispatch(traffic, size, http.execute(request)).await; + let mut sent = dispatch(request).await; // **OAuth 上游回 401:换一个 access token 再发一次,只一次。**token 可能在别处被 // 吊销了、提前失效了;不重试的话,这个请求连同之后每一个请求都会原样失败,直到 // 缓存里那个 token 按时间过期。换回来的还是 401,说明问题不在 token @@ -3228,9 +3286,9 @@ async fn send( Ok(fresh) if fresh != upstream_headers => match build(&fresh).body(body).build() { Ok(request) => { tap.sending(&request, &plain, &fresh); - sent = dispatch(traffic, size, http.execute(request)).await; + sent = dispatch(request).await.map_err(SendError::after_dispatch); } - Err(e) => sent = Err(e), + Err(e) => sent = Err(SendError::Http(e)), }, Ok(_) => {} Err(e) => { @@ -3238,7 +3296,7 @@ async fn send( } } } - sent.map_err(SendError::Http) + sent } /// 发出去一次:请求体(`size` 字节)记进这个请求的流量,响应套上一层、读到的每一块都记上 @@ -3246,21 +3304,45 @@ async fn send( /// 后面读它的每一处拿到的都是解开的字节。**连不上的退回**:地址不通、代理拒绝、握手失败时 /// 上游一个字节都没收到。别的错误(超时、发到一半断了)照记:请求体多半已经出去了 async fn dispatch( + state: &AppState, + route: &crate::route_switch::ClientRoute, + client: &str, traffic: &std::sync::Arc, size: usize, sending: impl std::future::Future>, -) -> reqwest::Result { - traffic.sending(size); - match sending.await { +) -> Result { + let mut sending = std::pin::pin!(sending); + let mut started = false; + let result = std::future::poll_fn(|cx| { + let _gate = if started { + None + } else { + let gate = state + .dispatch_gate + .lock() + .unwrap_or_else(|p| p.into_inner()); + if !route.current(state, client) { + return std::task::Poll::Ready(Err(SendError::RouteChanged { dispatched: false })); + } + traffic.sending(size); + started = true; + Some(gate) + }; + sending + .as_mut() + .poll(cx) + .map(|r| r.map_err(SendError::Http)) + }) + .await; + match result { Ok(r) => Ok(crate::inflate::inflated(crate::traffic::metered( r, traffic, ))), - Err(e) => { - if e.is_connect() { - traffic.unsent(size); - } - Err(e) + Err(SendError::Http(e)) if e.is_connect() => { + traffic.unsent(size); + Err(SendError::Http(e)) } + Err(e) => Err(e), } } diff --git a/crates/tw-gateway/src/server/pipeline/relay.rs b/crates/tw-gateway/src/server/pipeline/relay.rs index cddc9973..9dc8c95d 100644 --- a/crates/tw-gateway/src/server/pipeline/relay.rs +++ b/crates/tw-gateway/src/server/pipeline/relay.rs @@ -25,6 +25,8 @@ use crate::failure::Cause; use crate::forward; use crate::server::{because, flagged}; use crate::state::{AppState, Runtime}; +use tw_dialect::ir::Dialect; + use tw_types::msg; /// `asked_model` 是客户端请求里写的模型名:发出去的不是它、上游答的又是发出去的那个 @@ -143,6 +145,20 @@ pub(super) fn respond( session.as_ref(), ); let upstream_dialect = upstream_dialect(req, session.as_ref()); + let mut history = (status.is_success() + && req.dialect == upstream_dialect + && matches!(upstream_dialect, Dialect::Responses | Dialect::Anthropic)) + .then(|| { + crate::route_switch::Capture::new( + state, + rt, + &req.client_name, + &provider.name, + upstream_dialect, + plan.is_sse, + ) + }) + .flatten(); let mut mode_completion = (stream_trial.is_some() && plan.is_sse) .then(|| crate::stream_mode::Completion::new(upstream_dialect)); let stream_modes = rt.stream_modes.clone(); @@ -339,6 +355,9 @@ pub(super) fn respond( // **旁路嗅探和留档,不缓冲**:看的是上游原话(带占位符的那一版), // usage 和请求记录不受后面的还原、转换影响。先认错误,再记这一家的成败。 ending.feed(&chunk); + if let Some(capture) = history.as_mut() { + capture.feed(&chunk); + } if let Some(proof) = mode_completion.as_mut() { proof.feed(&chunk); } @@ -391,6 +410,9 @@ pub(super) fn respond( } else { (Some(chunks), Some(_slot)) }; + if let Some(capture) = history.as_mut() { + capture.finish(); + } let mode_complete = stream_trial.is_some() && match mode_completion { Some(proof) => proof.finish(), None => crate::stream_mode::whole_answer(upstream_dialect, &relay.whole), diff --git a/crates/tw-gateway/src/server/pipeline/web/socket.rs b/crates/tw-gateway/src/server/pipeline/web/socket.rs index 03447ec7..38e1bd36 100644 --- a/crates/tw-gateway/src/server/pipeline/web/socket.rs +++ b/crates/tw-gateway/src/server/pipeline/web/socket.rs @@ -13,13 +13,12 @@ type Writer = futures::stream::SplitSink; pub(in crate::server) async fn serve_socket( socket: WebSocket, state: AppState, - rt: Arc, - client_name: String, uri: axum::http::Uri, query: Option, headers: axum::http::HeaderMap, from: crate::server::Sender, _live: crate::live::Pass, + peer: std::net::IpAddr, ) { let (mut writer, mut reader) = socket.split(); let mut previous: Option<(String, Value, Vec)> = None; @@ -56,6 +55,25 @@ pub(in crate::server) async fn serve_socket( .await; continue; } + let rt = state.runtime(); + if !rt.allow.allows(peer) { + send_error( + &mut writer, + "permission_error", + "This source is no longer allowed", + ) + .await; + return; + } + let client_name = match AppState::identify_in(&rt, &headers, query.as_deref()) { + Ok((client, _)) => client, + Err(err) => { + let _ = writer + .send(Message::Text(crate::ws::failed_frame(err, None).into())) + .await; + return; + } + }; body.as_object_mut() .expect("response.create object") .remove("type"); @@ -140,8 +158,19 @@ pub(in crate::server) async fn serve_socket( send_error(&mut writer, "cancelled", "Response was cancelled").await; continue; } - let response = - response.unwrap_or_else(|e| e.in_dialect(Dialect::Responses).into_response()); + let response = match response { + Err(err) if err.source == crate::error::Source::RouteConflict => { + if writer + .send(Message::Text(crate::ws::failed_frame(err, None).into())) + .await + .is_err() + { + return; + } + continue; + } + other => other.unwrap_or_else(|e| e.in_dialect(Dialect::Responses).into_response()), + }; let response = match seat.get() { Some(capture) => capture.client_answer(axum::http::Version::HTTP_11, response), None => response, diff --git a/crates/tw-gateway/src/server/upgrade.rs b/crates/tw-gateway/src/server/upgrade.rs index 1feb83e5..09a03aeb 100644 --- a/crates/tw-gateway/src/server/upgrade.rs +++ b/crates/tw-gateway/src/server/upgrade.rs @@ -34,6 +34,7 @@ pub(super) async fn ws_upgrade( started: std::time::Instant, live: crate::live::Pass, from: Sender, + peer: std::net::IpAddr, ) -> Result { // 升级请求没有体,所以性质里只有客户端名字 —— 按模型路由的规则 // 对它不适用,而那是对的:这条连接上会跑什么模型,现在还不知道。这次的决定 @@ -48,17 +49,7 @@ pub(super) async fn ws_upgrade( return Ok(ws .max_message_size(16 * 1024 * 1024) .on_upgrade(move |socket| { - super::pipeline::web_socket( - socket, - state, - rt, - client_name, - uri, - query, - headers, - from, - live, - ) + super::pipeline::web_socket(socket, state, uri, query, headers, from, live, peer) })); } let realtime = crate::ws::realtime(uri.path()); @@ -164,6 +155,17 @@ pub(super) async fn ws_upgrade( // 按这次的决定定(指定模型、阶段一的改写),别名对到这一家(见 `crate::ws::Naming`) let naming_for = |provider: &tw_config::Provider| crate::ws::Naming { config: rt.config.clone(), + route: rt + .routing + .client(&client_name) + .expect("authenticated key") + .clone(), + account: rt + .routing + .account(&provider.name) + .expect("configured provider"), + peer, + engine: rt.engine.clone(), decision: decision.clone(), provider: provider.clone(), diff --git a/crates/tw-gateway/src/state.rs b/crates/tw-gateway/src/state.rs index f3f0ba5b..ab4f0618 100644 --- a/crates/tw-gateway/src/state.rs +++ b/crates/tw-gateway/src/state.rs @@ -23,6 +23,8 @@ mod upstream; /// 了,正在排队的请求会失去它们的位置;事件流丢了,界面上的实时列表会 /// 断一次。这些都不该因为改了一条路由规则而发生。 pub struct Runtime { + pub(crate) routing: crate::route_switch::Routing, + pub web_tools: Arc, pub config: Arc, /// 路由引擎。**和配置一起建,一起换** —— 分开持有会让「规则改了但 @@ -72,6 +74,15 @@ impl Runtime { config: tw_config::Config, previous: Option<&Runtime>, plugins: &crate::plugin::Plugins, + ) -> Result { + Self::build_with_oauth(config, previous, plugins, None) + } + + fn build_with_oauth( + config: tw_config::Config, + previous: Option<&Runtime>, + plugins: &crate::plugin::Plugins, + oauth: Option<&crate::oauth::Cache>, ) -> Result { let mut clients = std::collections::HashMap::new(); for p in &config.providers { @@ -128,9 +139,12 @@ impl Runtime { web_tools.history = previous.web_tools.history.clone(); } let web_tools = Arc::new(web_tools); + let engine = Arc::new(config.engine()); + let routing = crate::route_switch::Routing::build(&config, &engine, previous, oauth); Ok(Self { + routing, web_tools, - engine: Arc::new(config.engine()), + engine, config: Arc::new(config), clients, allow, @@ -148,6 +162,7 @@ impl Runtime { /// 同一份配置、换一份插件。插件文件变了时走这条:配置没变,别的都不用重建 fn with_plugins(&self, plugins: crate::plugin::PluginSet) -> Self { Self { + routing: self.routing.clone(), web_tools: self.web_tools.clone(), config: self.config.clone(), engine: self.engine.clone(), @@ -291,6 +306,8 @@ pub struct AppState { pub live: crate::live::Live, /// 每段对话此刻归到哪一次会话(见 [`crate::session::Sessions`])。**跨重载存活** pub sessions: Arc, + /// Account fingerprints for history that can only be read by its upstream. + pub(crate) route_history: Arc, /// 每段对话这一轮的路由决定、上次回答它的那一家(见 [`crate::affinity`])。**跨重载存活** pub affinity: Arc, /// 每个 `load-balance` 组轮到了谁(见 [`crate::balance`])。**跨重载存活**,改了的组从头轮 @@ -310,6 +327,9 @@ pub struct AppState { /// 存回去 —— 不排队的话,插件那一路可能拿着换配置之前的那份配置,把刚换进去的 /// 新配置又换回去 swap: Arc>, + /// Orders the start of a send against publication of a new routing table. + pub(crate) dispatch_gate: Arc>, + /// Anthropic 流里上游静默多久就补一个 `ping`(见 `relay`)。**测试会把它调短**, /// 否则一条心跳的测试要干等十五秒 pub ping_every: std::time::Duration, @@ -408,6 +428,7 @@ impl AppState { proxies: Arc::new(std::sync::Mutex::new(Default::default())), live: crate::live::Live::default(), sessions: Default::default(), + route_history: Default::default(), affinity: Default::default(), balance: Default::default(), seals: Default::default(), @@ -415,6 +436,7 @@ impl AppState { effort_memory: Default::default(), plugins, swap: Default::default(), + dispatch_gate: Default::default(), ping_every: crate::PING_EVERY, ping_for: crate::PING_FOR, plugin_pool: Arc::new(crate::plugin::pool::Pool::default_size()), @@ -641,7 +663,7 @@ impl AppState { .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); let old = self.rt.load(); - let next = Runtime::build(config, Some(&old), &self.plugins)?; + let next = Runtime::build_with_oauth(config, Some(&old), &self.plugins, Some(&self.oauth))?; let broken = crate::plugin::load::newly_broken(&old.plugins, &next.plugins); // 比的是写法不是解析出来的地址:网卡名要问系统,而那是监听那一边的事 let (was, now) = (&old.config.listen.gateway, &next.config.listen.gateway); @@ -657,7 +679,11 @@ impl AppState { self.throttles.configure(&next.config); // 删掉的、地址密钥协议变了的上游,记着的推理强度作废 self.effort_memory.reconcile(&next.config); - self.rt.store(Arc::new(next)); + { + let _dispatch = self.dispatch_gate.lock().unwrap_or_else(|p| p.into_inner()); + self.rt.store(Arc::new(next)); + } + self.route_history.sync_routing(&self.runtime().routing); self.announce_broken(broken); // 模型汇总马上按新配置重算:删掉、停用的上游的模型必须立刻消失(列表 // 即承诺),改了范围的立刻生效。新加的、地址凭据变了的在后台补问 @@ -695,6 +721,7 @@ impl AppState { pub fn set_config_dir(&self, dir: std::path::PathBuf) { // 记着的推理强度也存在这个目录里(见 `crate::effort_memory`) self.effort_memory.set_dir(dir.clone()); + self.route_history.set_dir(&dir, &self.runtime().routing); if self.plugins.set_dir(dir) { self.reload_plugins(); } @@ -726,6 +753,14 @@ impl AppState { &self, headers: &HeaderMap, query: Option<&str>, + ) -> Result<(String, crate::auth::KeyPosition), GatewayError> { + Self::identify_in(&self.runtime(), headers, query) + } + + pub(crate) fn identify_in( + rt: &Runtime, + headers: &HeaderMap, + query: Option<&str>, ) -> Result<(String, crate::auth::KeyPosition), GatewayError> { let Some((key, position)) = crate::auth::extract_key_with_position(headers, query) else { return Err(GatewayError::auth(msg!( @@ -734,7 +769,6 @@ impl AppState { keys under `clients` in config.yaml." ))); }; - let rt = self.rt.load(); let found = rt.config.clients.iter().find(|c| key_eq(&c.key, &key)); match found { // **停用的密钥要说清楚是停用了。**这一条和「密钥无效」不同: diff --git a/crates/tw-gateway/src/wire.rs b/crates/tw-gateway/src/wire.rs index 67999ef4..fb71e429 100644 --- a/crates/tw-gateway/src/wire.rs +++ b/crates/tw-gateway/src/wire.rs @@ -10,7 +10,7 @@ impl From for FailureSource { Source::Auth => Self::Auth, Source::Config => Self::Config, Source::Upstream => Self::Upstream, - Source::Request => Self::Request, + Source::Request | Source::RouteConflict => Self::Request, Source::RateLimited | Source::Busy => Self::RateLimited, Source::Denied => Self::Denied, Source::NotSupported | Source::TooLarge => Self::Request, diff --git a/crates/tw-gateway/src/ws.rs b/crates/tw-gateway/src/ws.rs index 74b2086c..b72f6b67 100644 --- a/crates/tw-gateway/src/ws.rs +++ b/crates/tw-gateway/src/ws.rs @@ -259,6 +259,10 @@ pub struct Plugins { /// Responses 的连接每一帧用它([`Self::frame`])。Realtime 的连接没有 `response.create` 这种 /// 请求,模型写在升级请求的查询串里:升级时用一次([`Self::connect`])。 pub struct Naming { + pub(crate) route: crate::route_switch::ClientRoute, + pub(crate) account: [u8; 32], + pub(crate) peer: std::net::IpAddr, + /// 升级那一刻的配置和路由引擎:一条连接活多久就用它多久,和 [`Rules`] 一样 pub config: Arc, pub engine: Arc, @@ -350,6 +354,28 @@ pub enum Connect { } impl Naming { + fn current(&self, state: &AppState) -> bool { + let rt = state.runtime(); + self.route.matches(&rt, &self.client) && rt.allow.allows(self.peer) + } + + fn check_history( + &self, + state: &AppState, + value: &serde_json::Value, + ) -> Result<(), GatewayError> { + let Some(required) = + crate::route_switch::Required::read(tw_dialect::ir::Dialect::Responses, value) + else { + return Ok(()); + }; + let strict = state.route_history.strict(&self.route) + || crate::route_switch::manual(&self.engine, self.decision.via_group.as_deref()); + state + .route_history + .check(&required, self.route.key, self.account, strict) + } + /// 客户端那一侧的名称发给这一家时叫什么([`crate::models::resolve`])。`None` = 这一家 /// 服务不了这个别名 fn resolve(&self, catalog: &tw_engine::Catalog, name: &str) -> Option { @@ -444,8 +470,9 @@ impl Naming { &self, catalog: &tw_engine::Catalog, facts: &tw_engine::RequestFacts, + strict: bool, ) -> Result> { - let decision = self.this_frame(facts)?; + let decision = self.this_frame(facts, strict)?; let p = &self.provider; let (mut set, renamed, two) = match self.engine.phase_two(facts, &p.name, &decision.set) { Ok(tw_engine::Outcome2::Proceed { @@ -493,8 +520,18 @@ impl Naming { fn this_frame( &self, facts: &tw_engine::RequestFacts, + strict: bool, ) -> Result> { match self.engine.route(facts) { + Ok(tw_engine::Outcome::Route(d)) if strict => { + if d.candidates.contains(&self.provider.name) { + return Ok(d); + } + Err(NotSent::plain(GatewayError::conflict(msg!( + "gw.route.socket_target" => + "The current rules select an upstream outside this WebSocket connection. Send this request over HTTP to use the selected route." + )))) + } Ok(tw_engine::Outcome::Route(d)) => Ok(tw_engine::Decision { set: d.set, rewritten_by: d.rewritten_by, @@ -663,6 +700,7 @@ struct Pipes { plugins: Option, /// 每个 `response.create` 发出去的模型名怎么定。Responses 的连接才有 naming: Option, + reconnect: bool, /// 最近一次 `response.create`:客户端要的模型、发出去的模型(别名对过、规则或插件 /// 换过的那个)和它的密钥映射。回答钩子用 requested_model: String, @@ -814,6 +852,12 @@ pub(crate) async fn proxy( naming: Option, ) { let hop_started = std::time::Instant::now(); + if let Some(naming) = &naming + && !naming.current(&state) + { + close_with(client, GatewayError::route_changed().message()).await; + return; + } let connected = connect(&upstream).await; // **连不上也要报。**和 HTTP 那条路一样,失败的时候恰恰最需要看这一跳; // 报在结局之前,存储层落库时手上才有它 @@ -965,6 +1009,7 @@ pub(crate) async fn proxy( written, plugins, naming, + reconnect: false, requested_model: String::new(), sent_model: String::new(), bridge: None, @@ -1166,7 +1211,7 @@ async fn pump( break 'pump end; } } - if draining + if (draining || p.reconnect) && waiting.is_none() && held.is_empty() && p.turns.as_ref().is_some_and(turn::Turns::is_empty) @@ -1293,7 +1338,12 @@ async fn pump( let _ = c_tx .send(Message::Close(Some(axum::extract::ws::CloseFrame { code: CLOSE_RESTART, - reason: "the gateway is restarting; reconnect".into(), + reason: if p.reconnect { + "the route changed; reconnect" + } else { + "the gateway is restarting; reconnect" + } + .into(), }))) .await; } @@ -1318,6 +1368,14 @@ async fn client_frame( waiting: &mut Option, ending: &mut Option, ) -> Step { + if p.turns.is_some() + && let Some(naming) = &p.naming + && !naming.current(state) + && matches!(m, Message::Binary(_)) + { + p.reconnect = true; + return reply_failed(c_tx, GatewayError::route_changed()).await; + } let out = match m { Message::Text(t) => { if p.turns.is_some() @@ -1401,6 +1459,12 @@ async fn begin_turn( let (Some(naming), Some(turns)) = (p.naming.as_ref(), p.turns.as_ref()) else { return Step::Go; }; + let route_check = if naming.current(state) { + naming.check_history(state, &frame) + } else { + p.reconnect = true; + Err(GatewayError::route_changed()) + }; let screening = { let s = &p.rules.screen; if s.mode.detects() { @@ -1437,7 +1501,13 @@ async fn begin_turn( let line = turns.line.clone(); // 别名对到这一家、密钥的模型范围继承时看的清单:这一帧(一次请求)用同一份 let catalog = state.catalog.load(); - let out = match naming.frame(&catalog, &reading.facts) { + let out = match route_check.map_err(NotSent::plain).and_then(|()| { + naming.frame( + &catalog, + &reading.facts, + state.route_history.strict(&naming.route), + ) + }) { Ok(out) => out, // 只是这一帧不发:照样留一行,替它回一个 `response.failed`,连接照常 Err(no) => { @@ -1519,12 +1589,61 @@ async fn admitted( return reply_failed(c_tx, err).await; } }; + if let Some(naming) = &p.naming { + let problem = if !naming.current(state) { + p.reconnect = true; + Err(GatewayError::route_changed()) + } else { + serde_json::from_str(&text).map_or(Ok(()), |frame| naming.check_history(state, &frame)) + }; + if let Err(err) = problem { + turn.unsent(Vec::new(), err.source.into(), err.detail.clone()); + return reply_failed(c_tx, err).await; + } + } let model = Some(p.sent_model.clone()).filter(|m| !m.is_empty() && *m != p.requested_model); let out = outbound(state, p, turn.id, text); - turn.sent(model.clone()); let size = payload(&out); - match u_tx.send(out).await { + let sent = if let Some(naming) = &p.naming { + use futures::Sink; + use std::task::Poll; + let mut outgoing = Some(out); + let enqueued = std::future::poll_fn(|cx| { + let mut sink = std::pin::Pin::new(&mut *u_tx); + // Keep checking while the socket is full. Once start_send accepts + // the frame, its flush can finish across a reload. + let _gate = state + .dispatch_gate + .lock() + .unwrap_or_else(|p| p.into_inner()); + if !naming.current(state) { + return Poll::Ready(None); + } + match sink.as_mut().poll_ready(cx) { + Poll::Ready(Ok(())) => Poll::Ready(Some( + sink.start_send(outgoing.take().expect("only enqueued once")), + )), + Poll::Ready(Err(err)) => Poll::Ready(Some(Err(err))), + Poll::Pending => Poll::Pending, + } + }) + .await; + match enqueued { + Some(Ok(())) => u_tx.flush().await, + Some(Err(err)) => Err(err), + None => { + p.reconnect = true; + let err = GatewayError::route_changed(); + turn.unsent(Vec::new(), err.source.into(), err.detail.clone()); + return reply_failed(c_tx, err).await; + } + } + } else { + u_tx.send(out).await + }; + match sent { Ok(()) => { + turn.sent(model.clone()); turn.sending(size); if let Some(t) = p.turns.as_mut() { t.push(turn); @@ -1642,6 +1761,22 @@ async fn upstream_text( e.count(t.len()); } let kind = frame_kind(t); + if let Some(naming) = &p.naming + && matches!( + kind.as_deref(), + Some( + "response.created" + | "response.completed" + | "response.incomplete" + | "response.output_item.done" + ) + ) + && let Ok(value) = serde_json::from_str(t) + { + state + .route_history + .record(naming.route.key, naming.account, &value); + } // Realtime 的一次回答收了尾:它的用量加到这条连接的那一行上。**看的是上游原话**,和 // 别的路一样(占位符不影响数字) if p.realtime @@ -2316,7 +2451,7 @@ fn response_id(frame: &str) -> Option { /// 替被切掉的那次回答(或者没发出去的那一帧)发的 `response.failed`:和 SSE 那条路同一个 /// 形状(`tw_dialect` 的错误帧),id 换成这次回答的。没发出去的那一帧没有回答,id 是新的 -fn failed_frame(err: GatewayError, response: Option<&str>) -> String { +pub(crate) fn failed_frame(err: GatewayError, response: Option<&str>) -> String { let until_reset = err.retry.is_some_and(|r| r.until_reset); let sse = err .in_dialect(tw_dialect::ir::Dialect::Responses) diff --git a/crates/tw-gateway/tests/sealed.rs b/crates/tw-gateway/tests/sealed.rs index 909a86b0..fc5922b3 100644 --- a/crates/tw-gateway/tests/sealed.rs +++ b/crates/tw-gateway/tests/sealed.rs @@ -56,10 +56,20 @@ async fn account(name: &'static str, in_stream: bool) -> Account { .unwrap(); } let answer = format!( - r#"{{"id":"resp_1","object":"response","status":"completed","model":"gpt-5","output":[{{"type":"reasoning","id":"rs_1","summary":[],"encrypted_content":"gAAAA-{name}"}},{{"type":"message","role":"assistant","content":[{{"type":"output_text","text":"ok"}}]}}],"usage":{{"input_tokens":3,"output_tokens":1,"total_tokens":4}}}}"# + r#"{{"id":"resp_{name}","object":"response","status":"completed","model":"gpt-5","output":[{{"type":"reasoning","id":"rs_1","summary":[],"encrypted_content":"gAAAA-{name}"}},{{"type":"message","role":"assistant","content":[{{"type":"output_text","text":"ok"}}]}}],"usage":{{"input_tokens":3000,"input_tokens_details":{{"cached_tokens":2048}},"output_tokens":1,"total_tokens":4}}}}"# ); + let answer = if body.contains("warm-cache") { + answer + } else { + answer.replace(r#""cached_tokens":2048"#, r#""cached_tokens":0"#) + }; + let (format, answer) = if body.contains(r#""stream":true"#) { + ("text/event-stream", format!("event: response.completed\ndata: {{\"type\":\"response.completed\",\"response\":{answer}}}\n\n")) + } else { + ("application/json", answer) + }; builder - .header("content-type", "application/json") + .header("content-type", format) .body(axum::body::Body::from(answer)) .unwrap() } @@ -143,6 +153,8 @@ fn body(turn: usize, sealed: &str, agents: bool) -> String { /// 结局:路由事件里试过的上游、留下的理由,和失败时的那句话 struct Outcome { status: u16, + no_retry: bool, + reply: String, attempts: Vec, stayed: Option, failed: Option, @@ -154,7 +166,7 @@ async fn ask(gw: SocketAddr, rx: &mut Receiver, body: String) -> Outcome /// [`ask`],在会话 `session` 里 async fn ask_in(gw: SocketAddr, rx: &mut Receiver, session: &str, body: String) -> Outcome { - let status = reqwest::Client::builder() + let response = reqwest::Client::builder() .no_proxy() .build() .unwrap() @@ -164,9 +176,13 @@ async fn ask_in(gw: SocketAddr, rx: &mut Receiver, session: &str, body: S .body(body) .send() .await - .unwrap() - .status() - .as_u16(); + .unwrap(); + let status = response.status().as_u16(); + let no_retry = response + .headers() + .get("x-should-retry") + .is_some_and(|v| v == "false"); + let reply = response.text().await.unwrap(); let mut routed = (Vec::new(), None); loop { let ev = tokio::time::timeout(Duration::from_secs(5), rx.recv()) @@ -180,6 +196,8 @@ async fn ask_in(gw: SocketAddr, rx: &mut Receiver, session: &str, body: S Event::RequestFinished { .. } => { return Outcome { status, + no_retry, + reply, attempts: routed.0, stayed: routed.1, failed: None, @@ -188,6 +206,8 @@ async fn ask_in(gw: SocketAddr, rx: &mut Receiver, session: &str, body: S Event::RequestFailed { message, .. } => { return Outcome { status, + no_retry, + reply, attempts: routed.0, stayed: routed.1, failed: Some(message), @@ -206,63 +226,84 @@ fn served_by(o: &Outcome) -> Vec<(&str, AttemptOutcome)> { } #[tokio::test] -async fn sealed_agent_messages_go_back_to_the_account_that_sealed_them() { +async fn cutover_refuses_nonportable_history_before_dispatch() { let (a, b) = (account("甲", false).await, account("乙", false).await); - // 一个请求最多等一秒;退避的一秒是一百毫秒,不浮动 - let waiting = |to: &str| { - let mut c = cfg(&[("甲", &a), ("乙", &b)], to); - c.failover.wait_secs = 1; - c - }; - let mut state = tw_gateway::AppState::new(waiting("甲")).unwrap(); - state.retry_backoff = tw_gateway::retry::Backoff { - tick: Duration::from_millis(100), - jitter: 0.0, - }; + let state = tw_gateway::AppState::new(cfg(&[("甲", &a), ("乙", &b)], "甲")).unwrap(); let (gw, mut rx) = serve(state.clone()).await; - let o = ask(gw, &mut rx, body(1, "甲", false)).await; - assert_eq!(served_by(&o), [("甲", AttemptOutcome::Served)]); + let initial = ask(gw, &mut rx, body(1, "甲", false)).await; + assert_eq!(served_by(&initial), [("甲", AttemptOutcome::Served)]); - // 改了配置:规则现在送去乙。带着甲封存的、子代理交回来的话的这段对话照旧回甲,乙不做备选 - state.reload(waiting("乙")).unwrap(); - let o = ask(gw, &mut rx, body(2, "甲", true)).await; - assert_eq!(o.status, 200); - assert_eq!(served_by(&o), [("甲", AttemptOutcome::Served)]); - assert_eq!(o.stayed, Some(Stay::Sealed)); + state.reload(cfg(&[("甲", &a), ("乙", &b)], "乙")).unwrap(); + for (request, code) in [ + (body(2, "甲", false), "gw.route.history.reasoning"), + (body(2, "甲", true), "gw.route.history.content"), + (r#"{"model":"gpt-5","previous_response_id":"resp_甲","input":"continue","stream":true}"#.into(), + "gw.route.history.stored"), + (r#"{"model":"gpt-5","input":[{"type":"item_reference","id":"rs_1"}]}"#.into(), + "gw.route.history.stored"), + ] { + let o = ask(gw, &mut rx, request).await; + assert_eq!(o.status, 409, "{}", o.reply); + assert!(o.no_retry); + assert!(o.attempts.is_empty(), "{:#?}", o.attempts); + assert_eq!(o.failed.unwrap().code, code); + assert_eq!(serde_json::from_str::(&o.reply).unwrap()["error"]["code"], code); + } + assert_eq!(a.hits.load(Ordering::SeqCst), 1); assert_eq!(b.hits.load(Ordering::SeqCst), 0); - // 甲发不出去:退避着重试的也只是甲,到期限了报甲的错,不转给解不开的乙 - a.down.store(true, Ordering::SeqCst); - let o = ask(gw, &mut rx, body(3, "甲", true)).await; - assert_ne!(o.status, 200); - assert!(o.attempts.len() > 1, "{:#?}", o.attempts); - assert!( - o.attempts.iter().all(|a| a.provider == "甲"), - "{:#?}", - o.attempts + let replayable = body(2, "甲", false).replace( + r#",{"type":"reasoning","id":"rs_0","summary":[],"encrypted_content":"gAAAA-甲"}"#, + "", ); - assert_eq!(b.hits.load(Ordering::SeqCst), 0); - - // 新的对话照新的规则走 - let fresh = body(1, "甲", false).replace("对话-1", "对话-2"); - let o = ask_in(gw, &mut rx, "codex-2", fresh).await; + let o = ask(gw, &mut rx, replayable).await; assert_eq!(served_by(&o), [("乙", AttemptOutcome::Served)]); } #[tokio::test] -async fn a_conversation_whose_sealing_upstream_was_removed_is_refused_plainly() { - let (a, b) = (account("甲", false).await, account("乙", false).await); - let state = tw_gateway::AppState::new(cfg(&[("甲", &a), ("乙", &b)], "甲")).unwrap(); +async fn same_account_history_survives_a_route_change_and_provider_rename() { + let a = account("甲", false).await; + let state = tw_gateway::AppState::new(cfg(&[("甲", &a)], "甲")).unwrap(); let (gw, mut rx) = serve(state.clone()).await; - ask(gw, &mut rx, body(1, "甲", false)).await; + assert_eq!(ask(gw, &mut rx, body(1, "甲", false)).await.status, 200); + state.reload(cfg(&[("new name", &a)], "new name")).unwrap(); + for request in [ + body(2, "甲", true), + r#"{"model":"gpt-5","previous_response_id":"resp_甲","input":"continue"}"#.into(), + ] { + let o = ask(gw, &mut rx, request).await; + assert_eq!( + served_by(&o), + [("new name", AttemptOutcome::Served)], + "{}", + o.reply + ); + } + assert_eq!(a.hits.load(Ordering::SeqCst), 3); +} - state.reload(cfg(&[("乙", &b)], "乙")).unwrap(); - let o = ask(gw, &mut rx, body(2, "甲", true)).await; +#[tokio::test] +async fn manual_cutover_ignores_warm_affinity_and_has_no_implicit_fallback() { + let (a, b) = (account("甲", false).await, account("乙", false).await); + let selected = |name: &str| { + let mut c = cfg(&[("甲", &a), ("乙", &b)], "池"); + c.groups[0].kind = tw_engine::GroupType::Select; + c.groups[0].selected = Some(name.into()); + c.failover.wait_secs = 0; + c + }; + let warm = || body(1, "甲", false).replace("对话-1", "warm-cache"); + let state = tw_gateway::AppState::new(selected("甲")).unwrap(); + let (gw, mut rx) = serve(state.clone()).await; + assert_eq!(ask(gw, &mut rx, warm()).await.status, 200); + state.reload(selected("乙")).unwrap(); + let o = ask(gw, &mut rx, warm()).await; + assert_eq!(served_by(&o), [("乙", AttemptOutcome::Served)]); + b.down.store(true, Ordering::SeqCst); + let o = ask(gw, &mut rx, warm()).await; assert_ne!(o.status, 200); - assert!(o.attempts.is_empty(), "{:#?}", o.attempts); - let m = o.failed.expect("该失败"); - assert_eq!(m.code, "gw.route.sealed_upstream_gone", "{m:?}"); - assert_eq!(b.hits.load(Ordering::SeqCst), 0); + assert!(o.attempts.iter().all(|a| a.provider == "乙")); + assert_eq!(a.hits.load(Ordering::SeqCst), 1); } #[tokio::test] @@ -306,3 +347,195 @@ async fn an_upstream_that_cannot_read_the_history_says_so_plainly() { assert_eq!(m.args.get("upstream").map(String::as_str), Some("乙")); } } + +#[tokio::test] +async fn a_queued_retry_cannot_use_its_previous_route_after_a_round_trip() { + let (a, b) = (account("甲", false).await, account("乙", false).await); + let config = |to| { + let mut cfg = cfg(&[("甲", &a), ("乙", &b)], to); + cfg.failover.wait_secs = 3; + cfg + }; + let mut state = tw_gateway::AppState::new(config("甲")).unwrap(); + state.retry_backoff = tw_gateway::retry::Backoff { + tick: Duration::from_millis(600), + jitter: 0.0, + }; + let (gw, mut rx) = serve(state.clone()).await; + a.down.store(true, Ordering::SeqCst); + let request = tokio::spawn(async move { + reqwest::Client::new() + .post(format!("http://{gw}/v1/responses")) + .header("authorization", "Bearer tw-k") + .body(body(1, "甲", false)) + .send() + .await + .unwrap() + }); + tokio::time::timeout(Duration::from_secs(5), async { + while !matches!(rx.recv().await.unwrap(), Event::RequestRetrying { .. }) {} + }) + .await + .unwrap(); + state.reload(config("乙")).unwrap(); + state.reload(config("甲")).unwrap(); + a.down.store(false, Ordering::SeqCst); + let result = request.await.unwrap(); + assert_eq!(result.status(), 409); + assert_eq!(result.headers()["x-should-retry"], "false"); + assert_eq!(a.hits.load(Ordering::SeqCst), 1); + assert_eq!(b.hits.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn a_default_change_preserves_explicit_assignment_for_the_same_route() { + let (a, b) = (account("甲", false).await, account("乙", false).await); + let mut config = cfg(&[("甲", &a), ("乙", &b)], "甲"); + let old = config.routes[0].name.clone(); + let mut new_route = config.routes[0].clone(); + new_route.name = "next".into(); + new_route.rules[0].to = Some("乙".into()); + config.routes.push(new_route); + config.clients.push(Client { + name: "pinned".into(), + key: "tw-pinned".into(), + route: Some(old.clone()), + ..Default::default() + }); + let state = tw_gateway::AppState::new(config.clone()).unwrap(); + let (gw, _rx) = serve(state.clone()).await; + let client = reqwest::Client::new(); + let send = |key: &str, payload: String| { + client + .post(format!("http://{gw}/v1/responses")) + .bearer_auth(key) + .body(payload) + .send() + }; + + for key in ["tw-k", "tw-pinned"] { + let result = send(key, body(1, "甲", false)).await.unwrap(); + assert_eq!(result.status(), 200); + result.bytes().await.unwrap(); + } + config.default_route = Some("next".into()); + state.reload(config.clone()).unwrap(); + assert_eq!( + send("tw-k", body(2, "甲", false)).await.unwrap().status(), + 409 + ); + assert_eq!( + send("tw-pinned", body(2, "甲", false)) + .await + .unwrap() + .status(), + 200 + ); + + config.clients[0].route = Some(old); + state.reload(config).unwrap(); + assert_eq!( + send("tw-k", body(2, "甲", false)).await.unwrap().status(), + 200 + ); + assert_eq!(b.hits.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn handover_compares_the_effective_route_and_keeps_history_proof() { + let (a, b) = (account("甲", false).await, account("乙", false).await); + let old = tw_gateway::AppState::new(cfg(&[("甲", &a), ("乙", &b)], "甲")).unwrap(); + let (gw, mut rx) = serve(old.clone()).await; + assert_eq!(ask(gw, &mut rx, body(1, "甲", false)).await.status, 200); + let new = tw_gateway::AppState::new(cfg(&[("甲", &a), ("乙", &b)], "乙")).unwrap(); + new.handover_import(&old.handover_export()); + let (gw, mut rx) = serve(new.clone()).await; + assert_eq!(ask(gw, &mut rx, body(2, "甲", true)).await.status, 409); + assert_eq!(b.hits.load(Ordering::SeqCst), 0); + new.reload(cfg(&[("alias", &a), ("乙", &b)], "alias")) + .unwrap(); + assert_eq!(ask(gw, &mut rx, body(2, "甲", true)).await.status, 200); +} + +#[tokio::test] +async fn hosted_responses_socket_uses_the_current_route_on_each_turn() { + use futures::{SinkExt, StreamExt}; + use tokio_tungstenite::tungstenite::{Message, client::IntoClientRequest}; + let (a, b) = (account("甲", false).await, account("乙", false).await); + let config = |to| { + let mut cfg = cfg(&[("甲", &a), ("乙", &b)], to); + cfg.web_tools.mode = tw_api::WebToolsMode::Core; + cfg + }; + let state = tw_gateway::AppState::new(config("甲")).unwrap(); + let (gw, _) = serve(state.clone()).await; + let mut req = format!("ws://{gw}/v1/responses") + .into_client_request() + .unwrap(); + req.headers_mut() + .insert("authorization", "Bearer tw-k".parse().unwrap()); + let (mut socket, _) = tokio_tungstenite::connect_async(req).await.unwrap(); + let create = |request: String| { + let mut value: serde_json::Value = serde_json::from_str(&request).unwrap(); + value["type"] = "response.create".into(); + Message::Text(value.to_string().into()) + }; + socket.send(create(body(1, "甲", false))).await.unwrap(); + let result = tokio::time::timeout(Duration::from_secs(5), socket.next()) + .await + .unwrap() + .unwrap() + .unwrap(); + assert!(result.into_text().unwrap().contains("response.completed")); + + state.reload(config("乙")).unwrap(); + socket + .send(create( + r#"{"model":"gpt-5","previous_response_id":"resp_甲","input":"continue"}"#.into(), + )) + .await + .unwrap(); + let result = tokio::time::timeout(Duration::from_secs(5), socket.next()) + .await + .unwrap() + .unwrap() + .unwrap(); + let value: serde_json::Value = serde_json::from_str(&result.into_text().unwrap()).unwrap(); + assert_eq!(value["type"], "response.failed"); + assert_eq!( + value["response"]["error"]["code"], + "gw.route.history.reasoning" + ); + assert_eq!(a.hits.load(Ordering::SeqCst), 1); + assert_eq!(b.hits.load(Ordering::SeqCst), 0); + + socket.send(create(body(1, "乙", false))).await.unwrap(); + let result = tokio::time::timeout(Duration::from_secs(5), socket.next()) + .await + .unwrap() + .unwrap() + .unwrap(); + assert!(result.into_text().unwrap().contains("resp_乙")); +} + +#[tokio::test] +async fn a_restart_after_cutover_keeps_the_current_route_and_history_proof() { + let dir = tempfile::tempdir().unwrap(); + let (a, b) = (account("甲", false).await, account("乙", false).await); + let config = |to| cfg(&[("甲", &a), ("乙", &b)], to); + let before = tw_gateway::AppState::new(config("甲")).unwrap(); + before.set_config_dir(dir.path().into()); + let (gw, mut rx) = serve(before.clone()).await; + assert_eq!(ask(gw, &mut rx, body(1, "甲", false)).await.status, 200); + before.reload(config("乙")).unwrap(); + + let after = tw_gateway::AppState::new(config("乙")).unwrap(); + after.set_config_dir(dir.path().into()); + let (gw, mut rx) = serve(after.clone()).await; + let o = ask(gw, &mut rx, body(2, "甲", false)).await; + assert_eq!(o.status, 409); + assert!(o.no_retry); + assert_eq!(b.hits.load(Ordering::SeqCst), 0); + after.reload(config("甲")).unwrap(); + assert_eq!(ask(gw, &mut rx, body(2, "甲", false)).await.status, 200); +} diff --git a/crates/tw-gateway/tests/ws.rs b/crates/tw-gateway/tests/ws.rs index 06aff559..6f606c6c 100644 --- a/crates/tw-gateway/tests/ws.rs +++ b/crates/tw-gateway/tests/ws.rs @@ -2898,3 +2898,129 @@ async fn closing_frames_belong_to_the_answer_in_progress_unless_they_are_strays( } } } + +#[tokio::test] +async fn a_native_responses_socket_finishes_the_sent_turn_and_fences_the_next() { + let (release, hold) = tokio::sync::watch::channel(false); + let (a, a_seen) = responder(Some(hold)).await; + let (b, b_seen) = responder(None).await; + let (gw, _, state) = turns_gateway( + a, + |cfg| { + cfg.providers.push(Provider { + name: "next".into(), + base_url: format!("http://{b}"), + protocol: Some(tw_config::Protocol::OpenaiResponses), + ..Default::default() + }); + cfg.routes + .push(tw_engine::RouteSet::default_with(vec![tw_engine::Rule { + name: "choose".into(), + when: serde_yaml_ng::from_str("{}").unwrap(), + to: Some("up".into()), + set: None, + deny: None, + }])); + }, + None, + ) + .await; + let mut socket = connect(gw).await; + socket.send(create("already sent")).await.unwrap(); + let first = tokio::time::timeout(Duration::from_secs(5), socket.next()) + .await + .unwrap() + .unwrap() + .unwrap(); + assert!(first.into_text().unwrap().contains("response.created")); + let mut next = (*state.config()).clone(); + next.routes[0].rules[0].to = Some("next".into()); + state.reload(next).unwrap(); + socket.send(create("after switch")).await.unwrap(); + let failed = one_answer(&mut socket).await; + assert_eq!( + failed.last().unwrap()["response"]["error"]["code"], + "gw.route.changed" + ); + release.send(true).unwrap(); + assert_eq!( + one_answer(&mut socket).await.last().unwrap()["type"], + "response.completed" + ); + let frame = tokio::time::timeout(Duration::from_secs(5), socket.next()) + .await + .unwrap() + .unwrap() + .unwrap(); + assert!(frame.is_close()); + assert_eq!(a_seen.lock().unwrap().len(), 1); + assert!(b_seen.lock().unwrap().is_empty()); + + let mut socket = connect(gw).await; + let stored = serde_json::json!({"type":"response.create","model":"gpt-5","previous_response_id":"resp_1","input":"continue"}); + socket + .send(tokio_tungstenite::tungstenite::Message::Text( + stored.to_string().into(), + )) + .await + .unwrap(); + let failed = one_answer(&mut socket).await; + assert_eq!( + failed.last().unwrap()["response"]["error"]["code"], + "gw.route.history.stored" + ); + assert!(b_seen.lock().unwrap().is_empty()); + + socket.send(create("full replay")).await.unwrap(); + assert_eq!( + one_answer(&mut socket).await.last().unwrap()["type"], + "response.completed" + ); + assert_eq!(b_seen.lock().unwrap().len(), 1); +} + +#[tokio::test] +async fn a_reconnected_socket_refuses_a_model_now_routed_to_another_upstream() { + let (a, a_seen) = responder(None).await; + let (b, b_seen) = responder(None).await; + let (gw, _, state) = turns_gateway( + a, + |cfg| { + cfg.providers.push(Provider { + name: "next".into(), + base_url: format!("http://{b}"), + protocol: Some(tw_config::Protocol::OpenaiResponses), + ..Default::default() + }); + }, + None, + ) + .await; + let mut cfg = (*state.config()).clone(); + cfg.routes.push(tw_engine::RouteSet::default_with(vec![ + tw_engine::Rule { + name: "model rule".into(), + when: serde_yaml_ng::from_str("{model: gpt-5}").unwrap(), + to: Some("next".into()), + set: None, + deny: None, + }, + tw_engine::Rule { + name: "fallback".into(), + when: serde_yaml_ng::from_str("{}").unwrap(), + to: Some("up".into()), + set: None, + deny: None, + }, + ])); + state.reload(cfg).unwrap(); + let mut socket = connect(gw).await; + socket.send(create("model routed elsewhere")).await.unwrap(); + let failed = one_answer(&mut socket).await; + assert_eq!( + failed.last().unwrap()["response"]["error"]["code"], + "gw.route.socket_target" + ); + assert!(a_seen.lock().unwrap().is_empty()); + assert!(b_seen.lock().unwrap().is_empty()); +} From f5a1c73bbabb851ee8854f95cb11c10087593f99 Mon Sep 17 00:00:00 2001 From: fylorn <249551762+fylorn@users.noreply.github.com> Date: Thu, 19 Nov 2026 19:46:21 +0800 Subject: [PATCH 2/3] ci: provision the plugin sandbox toolchain for enterprise checks --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 81b1863b..12dae62f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -167,6 +167,11 @@ jobs: - uses: dtolnay/rust-toolchain@stable if: steps.changed.outputs.layer1 == 'true' + with: + targets: wasm32-unknown-unknown + - name: Toolchain for the plugin sandbox + if: steps.changed.outputs.layer1 == 'true' + run: bash core/scripts/wasm-toolchain.sh # 第一层全部指向这次的 core - name: Point enterprise at this core if: steps.changed.outputs.layer1 == 'true' From f177180bdae72ba9aaa30d84637e5daa47a3230d Mon Sep 17 00:00:00 2001 From: fylorn <249551762+fylorn@users.noreply.github.com> Date: Thu, 19 Nov 2026 19:49:06 +0800 Subject: [PATCH 3/3] fix(release): update the plugin sandbox to Wasmtime 49.0.6 Use the same Ubuntu image for the enterprise compatibility check as for the Linux Core check. It provides the LLVM toolchain used by the sandbox. --- .github/workflows/ci.yml | 2 +- Cargo.lock | 108 ++++++++++++++++++------------------ crates/tw-plugin/Cargo.toml | 4 +- 3 files changed, 57 insertions(+), 57 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 12dae62f..04a8e2c1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -127,7 +127,7 @@ jobs: # 两个仓库都是公开的,不需要令牌。企业版没有 `sqlx::query!` 这类编译期 # 连库的宏,`cargo check` 不需要数据库。 enterprise: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 with: diff --git a/Cargo.lock b/Cargo.lock index 208add36..5adcf83b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -882,27 +882,27 @@ dependencies = [ [[package]] name = "cranelift-assembler-x64" -version = "0.136.5" +version = "0.136.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "249a61efca6a54425e027f5704a6f68a0232b4df527e05a3ef947d89dbdcb078" +checksum = "aecefa88696b62f833cc0c997d19c93246b0268e2aaf23e0e8692e527f6de088" dependencies = [ "cranelift-assembler-x64-meta", ] [[package]] name = "cranelift-assembler-x64-meta" -version = "0.136.5" +version = "0.136.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5336a46e05db2d515440b1ce8a9363e7bc04b608c0f66cc98dddc5492a66fbf" +checksum = "1391a2546f5edf06a0ff5e9430d73c2713fd536aefec53200dc21846d1467f53" dependencies = [ "cranelift-srcgen", ] [[package]] name = "cranelift-bforest" -version = "0.136.5" +version = "0.136.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01b3675b13214270fddbd257d5fbf4109be8b02d92fc636fe96e2829514a7235" +checksum = "9528915fbc2c950702329e08c6d72aaff8ece9b7ecdbd225742825c09a445227" dependencies = [ "cranelift-entity", "wasmtime-internal-core", @@ -910,9 +910,9 @@ dependencies = [ [[package]] name = "cranelift-bitset" -version = "0.136.5" +version = "0.136.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7c1d67aa02b6e174bd2b165c113e94af9498fcc20e2892d0f4d5ab8c9618d09" +checksum = "9dd69a6db85c1bcff989809e0e0b3b06506425f32b545c46aeafda5fe7775d6a" dependencies = [ "serde", "serde_derive", @@ -921,9 +921,9 @@ dependencies = [ [[package]] name = "cranelift-codegen" -version = "0.136.5" +version = "0.136.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26fe6f3ec5344c32602c414e986dcbe699fa501a3e3e7b02f887d9d12b14ae42" +checksum = "24c2db6c26cd96393cf785c4b51b900738963cc7422aa0045416a7641ae2f6cd" dependencies = [ "bumpalo", "cranelift-assembler-x64", @@ -952,9 +952,9 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.136.5" +version = "0.136.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c5e6323d458486d95781defdb5016a1176fc911d8bc8e64b696adbda6e9044" +checksum = "e284270142e7f1dcb73a992a62a8b3716b4062a65a52d8f2732c6ebf6f7f342f" dependencies = [ "cranelift-assembler-x64-meta", "cranelift-codegen-shared", @@ -965,24 +965,24 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" -version = "0.136.5" +version = "0.136.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1260aa93b0bd48e8e55c1c7fd9ff6a260f1a1bc46e1f0151a5055b124e5b15d" +checksum = "6595667b7d505cf85d5bdd014e6f57ad6128183238f8e372319d9384782472f5" [[package]] name = "cranelift-control" -version = "0.136.5" +version = "0.136.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50c179632a09850f81f1a1e9116cb09330c734a86689f3ee4c3f4ec81543acbd" +checksum = "f26ffdbfcd24c3833d730849e6d132af34a400a2b89b44446015c01eecbea21d" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.136.5" +version = "0.136.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8c071188cb1c3b1803e23ae415a8d5f58d741d3a4b26a79b83dd61921112eac" +checksum = "9ffb08ccd8a77fb31cdbbda2afa2333d8f85299b1bc0d1c194259ba66d160d74" dependencies = [ "cranelift-bitset", "serde", @@ -992,9 +992,9 @@ dependencies = [ [[package]] name = "cranelift-frontend" -version = "0.136.5" +version = "0.136.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2b85ba0a55d7685117b275fe6f36c819d438862e4223d28f0b97902460ff0ac" +checksum = "b447e4f2bb44b1188185a77f478e5a91b7723a40b3b58c3e2d5b26df2ed5a665" dependencies = [ "cranelift-codegen", "hashbrown 0.17.1", @@ -1005,15 +1005,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.136.5" +version = "0.136.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20ed364edcaed769da7b2709d540f8f7d0ff257749ba193f475e22dba6a0178b" +checksum = "2f0989e943f3a9d41989595826bba544df9fda64bcddd4df0c06fb2e26af3cb3" [[package]] name = "cranelift-native" -version = "0.136.5" +version = "0.136.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "418d98c3dce08f0d13526692bbd1888bff4c5f45207f5093d168b5a224fbc88c" +checksum = "5fa10df59e776c42dae5b6ba5216ba5321707a2136abd2219e615c54b234fc1b" dependencies = [ "cranelift-codegen", "libc", @@ -1022,9 +1022,9 @@ dependencies = [ [[package]] name = "cranelift-srcgen" -version = "0.136.5" +version = "0.136.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f016b9994e39cad68107f33fa5b8dcee8fb7e38332bde3f97690e5e8abe1b10" +checksum = "36508559c8839a8b6ee091f9deaa9cfe620a82f1abb5e358a0a0c65290583622" [[package]] name = "crc32fast" @@ -2727,9 +2727,9 @@ dependencies = [ [[package]] name = "pulley-interpreter" -version = "49.0.5" +version = "49.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d74df9e20cba0f6dd07f01c3140535a6b23e40aecccfcacfb921621b41d0acb" +checksum = "335e459f9c7809af04768799880504d014d07e8e2fc6ae682fe2a105d8ced33f" dependencies = [ "cranelift-bitset", "log", @@ -2739,9 +2739,9 @@ dependencies = [ [[package]] name = "pulley-macros" -version = "49.0.5" +version = "49.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8415513bfa407c06de6f11dbe9437f2352e075a5520ff491a131166d3d90aed1" +checksum = "dd492ac929a6ca0357362d9fa0a09e31290feac5914550ccfb539b21f89c4b9e" dependencies = [ "proc-macro2", "quote", @@ -4856,9 +4856,9 @@ dependencies = [ [[package]] name = "wasmtime" -version = "49.0.5" +version = "49.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d45bf3b13781b9febb0eef42e1e145146c92ef891f345fd918e0e0fa1568c992" +checksum = "f32390bdc8d7cdb6259a1b855178bfed88226b4004b209fd48fb30e0deb01cd3" dependencies = [ "addr2line", "async-trait", @@ -4895,9 +4895,9 @@ dependencies = [ [[package]] name = "wasmtime-environ" -version = "49.0.5" +version = "49.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "833f00f3dc6128159284c5b71e211f24c97d45e052230a14f767fd830a6fb8b0" +checksum = "9c10f8ef1c9ee5e0f125c775fe56a4d07b8525db30a96e6358c61d93bd11dee4" dependencies = [ "anyhow", "cpp_demangle", @@ -4926,15 +4926,15 @@ dependencies = [ [[package]] name = "wasmtime-internal-component-util" -version = "49.0.5" +version = "49.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b168d39f2d6b896d89fe096afa4ccdd7a8e927b01469b590af84f02fca3126a1" +checksum = "011e31ca343d9ba0ec623153cd94131b6f03973ef1a0ab791161f642e4a16a93" [[package]] name = "wasmtime-internal-core" -version = "49.0.5" +version = "49.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ad86a65d12c6a9202584ceb658c9a20cf5b25c32eed61c9be5e5f863f88f0d9" +checksum = "ab15e50a060b746493e6e95ef3ce4e6e7ebfdd7cc336a7dff0f0277e01b447ef" dependencies = [ "hashbrown 0.17.1", "libm", @@ -4943,9 +4943,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-cranelift" -version = "49.0.5" +version = "49.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a71f0fd1dd40e4e18d4a20cdba29ff8e0eb2c0f21b12c2eb5869e0fea789dd05" +checksum = "dc3d27440a83fba26dd2784a997e13d9acb3606a75fb127c324929b52221c5c5" dependencies = [ "cranelift-codegen", "cranelift-control", @@ -4969,9 +4969,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-fiber" -version = "49.0.5" +version = "49.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa4dfa39e58735b24ce3d28bfc8c2b3041603cc8a941bb268395351beff90172" +checksum = "6e54b28d0d453ca8932e4504065ae66a78f01fcb22b1a2ee287128d732fa9d6f" dependencies = [ "cc", "libc", @@ -4983,9 +4983,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-jit-debug" -version = "49.0.5" +version = "49.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2eb98d4486c7b0163447d77dea06f3cc87e5e5ba5fc93613f8ec42e27ea181d" +checksum = "9293e75a3048b900f7b7c524702abca16d8a2332f970324eefc519f454cd5d69" dependencies = [ "cc", "wasmtime-internal-versioned-export-macros", @@ -4993,9 +4993,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-jit-icache-coherence" -version = "49.0.5" +version = "49.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852cae234735d79db6c58fe0d04b32f4078ad01fdb3fd6d488ac366132ba9785" +checksum = "b852e568831b913d410541bb37edc45a5002e472208fbdbaf844ed801dd4255f" dependencies = [ "libc", "wasmtime-internal-core", @@ -5004,9 +5004,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-unwinder" -version = "49.0.5" +version = "49.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9289378c82b9db7e7cf933d6bb441ad968536ac48f3f38c55b7cf9d3e8d87e1" +checksum = "a1f7349c9834a542275ad757340079e6453fccb429efce581454d23f85af92c0" dependencies = [ "cranelift-codegen", "log", @@ -5016,9 +5016,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-versioned-export-macros" -version = "49.0.5" +version = "49.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc5c60c732d99dbcc5d670b71ba6ccf2b015cd16fc828990eaad2c4b8805ebdb" +checksum = "30b3e0768a450d9dea3e16b83f2498f0b08db0e32bad695e35564eef017be3f1" dependencies = [ "proc-macro2", "quote", @@ -5027,9 +5027,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-winch" -version = "49.0.5" +version = "49.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c236aa2967d9ed27babc3a76d478f6323cb5bf4c77d20102e282828226646458" +checksum = "3386d498f0bf0ccb226a086ee84964b0e72ec63fc0d84b3a12e09dc49737fd96" dependencies = [ "cranelift-codegen", "gimli", @@ -5116,9 +5116,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "winch-codegen" -version = "49.0.5" +version = "49.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98e378f1851ec513f230e3aa1b77a3b4c2e0ebed8db0e551b0c51144f43c133d" +checksum = "16ab4fd4d63582b1ca285b1c623dda445f23f1c8adcbef453808145bd00a5de6" dependencies = [ "cranelift-assembler-x64", "cranelift-codegen", diff --git a/crates/tw-plugin/Cargo.toml b/crates/tw-plugin/Cargo.toml index fa9c35f9..2ad5e0ce 100644 --- a/crates/tw-plugin/Cargo.toml +++ b/crates/tw-plugin/Cargo.toml @@ -18,7 +18,7 @@ description = "Runs one script plugin's hooks inside a WebAssembly sandbox (Quic # **只有运行时**:没有 Cranelift,没有池分配器。沙箱的机器码在 build.rs 里预编译 # 好、嵌进二进制。版本钉死,并且和下面构建依赖里的是同一个:预编译产物只肯在 # 同一版本的 Wasmtime 里加载 -wasmtime = { version = "=49.0.5", default-features = false, features = ["runtime", "std"] } +wasmtime = { version = "=49.0.6", default-features = false, features = ["runtime", "std"] } serde = { workspace = true } serde_json = { workspace = true } sha2 = { workspace = true } @@ -36,7 +36,7 @@ windows-sys = { workspace = true, features = ["Win32_Foundation", "Win32_System_ [build-dependencies] # 构建时的编译器:Cranelift 把快照编成目标平台的机器码;all-arch 让交叉编译 # (比如在 x64 上编 aarch64 的 Windows)也能编 -wasmtime = { version = "=49.0.5", default-features = false, features = ["runtime", "std", "cranelift", "all-arch", "parallel-compilation"] } +wasmtime = { version = "=49.0.6", default-features = false, features = ["runtime", "std", "cranelift", "all-arch", "parallel-compilation"] } # 改写快照的数据段。和 Wasmtime 49 自己用的是同一版 wasmparser = "0.258" wasm-encoder = "0.258"