diff --git a/Cargo.lock b/Cargo.lock index 7498487f..03ab7f4c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -79,6 +79,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2fb6cfd47bf496ff64095c20eaba0c201404ee38714d4142fcfa1dc334fcc7a" + +[[package]] +name = "alloc-stdlib" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5c1865780388bfa186411ab5f247819487fc4864c6e9c3106611fa347586e1" +dependencies = [ + "alloc-no-stdlib", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -177,6 +192,18 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" +[[package]] +name = "async-compression" +version = "0.4.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ad47a740fb906fffdd891e8b6d5fde2511600f7bfaebd68939fa08a14999459" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "async-stream" version = "0.3.6" @@ -520,6 +547,27 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "brotli" +version = "9.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "627b7cfb0578f0eafce087a90fcd40add1231900b69e4d4de2d456260a4f5d81" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "6.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75c74c9c51b9140663e196b5fb88bc94c334d7a497c8311cb759d8557874747f" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -709,6 +757,26 @@ dependencies = [ "memchr", ] +[[package]] +name = "compression-codecs" +version = "0.4.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84f840cff84d10085260e460023934d0af699e71212b09b2a41452e8bd9cda43" +dependencies = [ + "brotli", + "compression-core", + "flate2", + "memchr", + "zstd", + "zstd-safe", +] + +[[package]] +name = "compression-core" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ebdda89edc534bf8d5dd2857562633b6495557e5846552ade994b99ed13c36a" + [[package]] name = "const-oid" version = "0.10.2" @@ -721,6 +789,17 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" +[[package]] +name = "cookie" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a373e3602691c3cdea496d2f0ee5935151e6168fe87739483c463db1b2f2f87" +dependencies = [ + "percent-encoding", + "time", + "version_check", +] + [[package]] name = "cookie-factory" version = "0.3.3" @@ -730,6 +809,24 @@ dependencies = [ "futures", ] +[[package]] +name = "cookie_store" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b2c103cf610ec6cae3da84a766285b42fd16aad564758459e6ecf128c75206" +dependencies = [ + "cookie", + "document-features", + "idna", + "log", + "publicsuffix", + "serde", + "serde_derive", + "serde_json", + "time", + "url", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -1107,6 +1204,15 @@ dependencies = [ "syn 3.0.5", ] +[[package]] +name = "document-features" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" +dependencies = [ + "litrs", +] + [[package]] name = "dtoa" version = "1.0.11" @@ -2104,6 +2210,12 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" +[[package]] +name = "litrs" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4744e383959f0db86ede514b809b1c53251889093803c05267acc7d4e7030d70" + [[package]] name = "lock_api" version = "0.4.14" @@ -2597,6 +2709,22 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "psl-types" +version = "2.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac" + +[[package]] +name = "publicsuffix" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42ea446cab60335f76979ec15e12619a2165b5ae2c12166bef27d283a9fadf" +dependencies = [ + "idna", + "psl-types", +] + [[package]] name = "pulley-interpreter" version = "49.0.5" @@ -2873,6 +3001,8 @@ checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ "base64 0.22.1", "bytes", + "cookie", + "cookie_store", "futures-core", "futures-util", "h2", @@ -3864,12 +3994,17 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ + "async-compression", "bitflags", "bytes", + "futures-core", "futures-util", "http", "http-body", + "http-body-util", "pin-project-lite", + "tokio", + "tokio-util", "tower", "tower-layer", "tower-service", @@ -4013,7 +4148,7 @@ dependencies = [ [[package]] name = "tw-api" -version = "0.97.0" +version = "0.98.0" dependencies = [ "serde", "serde_json", @@ -4024,7 +4159,7 @@ dependencies = [ [[package]] name = "tw-bedrock" -version = "0.97.0" +version = "0.98.0" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -4045,7 +4180,7 @@ dependencies = [ [[package]] name = "tw-breaker" -version = "0.97.0" +version = "0.98.0" dependencies = [ "serde", "serde_json", @@ -4053,7 +4188,7 @@ dependencies = [ [[package]] name = "tw-config" -version = "0.97.0" +version = "0.98.0" dependencies = [ "blake3", "icu_locale_core", @@ -4081,7 +4216,7 @@ dependencies = [ [[package]] name = "tw-control" -version = "0.97.0" +version = "0.98.0" dependencies = [ "axum", "base64 0.22.1", @@ -4127,7 +4262,7 @@ dependencies = [ [[package]] name = "tw-dialect" -version = "0.97.0" +version = "0.98.0" dependencies = [ "memchr", "serde", @@ -4136,7 +4271,7 @@ dependencies = [ [[package]] name = "tw-engine" -version = "0.97.0" +version = "0.98.0" dependencies = [ "chrono", "serde", @@ -4150,7 +4285,7 @@ dependencies = [ [[package]] name = "tw-gateway" -version = "0.97.0" +version = "0.98.0" dependencies = [ "arc-swap", "async-stream", @@ -4198,13 +4333,14 @@ dependencies = [ "tw-pricing", "tw-secret", "tw-types", + "url", "uuid", "zstd", ] [[package]] name = "tw-guard" -version = "0.97.0" +version = "0.98.0" dependencies = [ "aho-corasick", "base64 0.22.1", @@ -4225,7 +4361,7 @@ dependencies = [ [[package]] name = "tw-handover" -version = "0.97.0" +version = "0.98.0" dependencies = [ "libc", "serde", @@ -4238,7 +4374,7 @@ dependencies = [ [[package]] name = "tw-link" -version = "0.97.0" +version = "0.98.0" dependencies = [ "serde", "serde_json", @@ -4252,7 +4388,7 @@ dependencies = [ [[package]] name = "tw-observe" -version = "0.97.0" +version = "0.98.0" dependencies = [ "tokio", "tracing", @@ -4261,7 +4397,7 @@ dependencies = [ [[package]] name = "tw-pack" -version = "0.97.0" +version = "0.98.0" dependencies = [ "age", "blake3", @@ -4285,7 +4421,7 @@ dependencies = [ [[package]] name = "tw-plugin" -version = "0.97.0" +version = "0.98.0" dependencies = [ "libc", "rand 0.10.2", @@ -4301,7 +4437,7 @@ dependencies = [ [[package]] name = "tw-pricing" -version = "0.97.0" +version = "0.98.0" dependencies = [ "arc-swap", "flate2", @@ -4314,14 +4450,14 @@ dependencies = [ [[package]] name = "tw-secret" -version = "0.97.0" +version = "0.98.0" dependencies = [ "thiserror 2.0.20", ] [[package]] name = "tw-store" -version = "0.97.0" +version = "0.98.0" dependencies = [ "blake3", "bytes", @@ -4341,7 +4477,7 @@ dependencies = [ [[package]] name = "tw-sync" -version = "0.97.0" +version = "0.98.0" dependencies = [ "age", "axum", @@ -4369,7 +4505,7 @@ dependencies = [ [[package]] name = "tw-types" -version = "0.97.0" +version = "0.98.0" dependencies = [ "serde", "serde_json", @@ -4378,7 +4514,7 @@ dependencies = [ [[package]] name = "tw-watch" -version = "0.97.0" +version = "0.98.0" dependencies = [ "libc", "notify", @@ -4389,7 +4525,7 @@ dependencies = [ [[package]] name = "tw-yaml" -version = "0.97.0" +version = "0.98.0" dependencies = [ "saphyr-parser", "serde_yaml_ng", @@ -4399,7 +4535,7 @@ dependencies = [ [[package]] name = "twcore" -version = "0.97.0" +version = "0.98.0" dependencies = [ "anyhow", "axum", diff --git a/Cargo.toml b/Cargo.toml index 2fd657c0..13067cf5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,7 +61,7 @@ categories = ["network-programming", "web-programming::http-server"] # 二进制走 CalVer,crate 走 SemVer —— 两者是两回事,卖给不同的人。 # 这里是 crate 的版本。 -version = "0.97.0" +version = "0.98.0" [workspace.dependencies] # ── 内部 crate ─────────────────────────────────────────── diff --git a/bin/twcore/src/upgrade.rs b/bin/twcore/src/upgrade.rs index c2dcb45a..c6415730 100644 --- a/bin/twcore/src/upgrade.rs +++ b/bin/twcore/src/upgrade.rs @@ -285,6 +285,12 @@ fn client() -> Result { /// 走环境变量里的代理(`HTTPS_PROXY`):服务器上出网常常只有这一条路。 fn builder() -> reqwest::ClientBuilder { reqwest::Client::builder() + // Search opts into content decoding; release downloads retain their + // existing behavior when those reqwest features are unified. + .no_gzip() + .no_brotli() + .no_deflate() + .no_zstd() .user_agent(concat!("twcore/", env!("CARGO_PKG_VERSION"))) .connect_timeout(Duration::from_secs(15)) .timeout(Duration::from_secs(300)) diff --git a/crates/tw-api/msg-codes.txt b/crates/tw-api/msg-codes.txt index 08b1a660..a8e64872 100644 --- a/crates/tw-api/msg-codes.txt +++ b/crates/tw-api/msg-codes.txt @@ -476,6 +476,7 @@ gw.wait.rate_limited gw.wait.used_up gw.wait.window_full gw.web_tools.failed +gw.web_tools.query_mismatch gw.ws.alias_unserved gw.ws.alias_unserved_rewritten gw.ws.bad_header diff --git a/crates/tw-api/src/web_tools.rs b/crates/tw-api/src/web_tools.rs index 96684c01..0f7ea7ad 100644 --- a/crates/tw-api/src/web_tools.rs +++ b/crates/tw-api/src/web_tools.rs @@ -17,6 +17,7 @@ pub enum WebToolsMode { pub enum SearchEngine { Bing, Duckduckgo, + GoogleDocs, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -77,6 +78,18 @@ pub struct WebToolsView { pub config_version: String, /// Detected on the Core host, which may be remote. Empty fields mean unavailable. pub system_locale: WebSearchLocale, + /// Engine capabilities advertised by the connected Core. + #[serde(default)] + pub available_engines: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct WebSearchEngine { + pub id: SearchEngine, + pub name: String, + /// The engine's minimum applies even if the user's configured interval is lower. + pub min_interval_ms: u32, } #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] diff --git a/crates/tw-config/tests/manual/schema.rs b/crates/tw-config/tests/manual/schema.rs index 31e5167f..31c31549 100644 --- a/crates/tw-config/tests/manual/schema.rs +++ b/crates/tw-config/tests/manual/schema.rs @@ -1310,8 +1310,8 @@ pub fn sections() -> Vec
{ Kind::Strs, Def::Is("[]"), t( - "Empty selects available engines automatically. Otherwise list bing and/or duckduckgo in priority order.", - "空列表自动选择可用引擎;手动填写 bing、duckduckgo,按顺序尝试。", + "Empty tries google_docs, duckduckgo, then bing. Otherwise list these engine IDs in priority order.", + "空列表依次尝试 google_docs、duckduckgo、bing;也可手动按优先顺序填写这些引擎 ID。", ), ), row( @@ -1403,8 +1403,8 @@ pub fn sections() -> Vec
{ Kind::Int, Def::Is("1000"), t( - "Minimum interval per search engine, 250–60000 milliseconds.", - "同一搜索引擎的最小请求间隔,250–60000 毫秒。", + "Minimum interval per search engine, 250–60000 milliseconds. A source may enforce a higher minimum (google_docs: 6000).", + "同一搜索引擎的最小请求间隔,250–60000 毫秒。搜索源可要求更长间隔(google_docs:6000)。", ), ), row( diff --git a/crates/tw-control/src/sync.rs b/crates/tw-control/src/sync.rs index f3790f88..ec055a2e 100644 --- a/crates/tw-control/src/sync.rs +++ b/crates/tw-control/src/sync.rs @@ -218,6 +218,12 @@ impl Host for CoreHost { /// 同步位置的密码就跟着去了),证书照常校验 fn http(cfg: &tw_config::Config, proxy: &str) -> Result { let mut b = reqwest::Client::builder() + // Search enables reqwest's decoding features workspace-wide. Keep sync + // downloads' existing byte semantics independent of that feature set. + .no_gzip() + .no_brotli() + .no_deflate() + .no_zstd() .connect_timeout(Duration::from_secs(10)) // 一轮里的请求是接连发的;两轮之间的空闲连接不留:很多服务端几秒就关掉空闲连接, // 留着的下一轮一发就断(发请求那一处另有一次重发) diff --git a/crates/tw-control/src/web_tools.rs b/crates/tw-control/src/web_tools.rs index ea5eac71..f504ffd5 100644 --- a/crates/tw-control/src/web_tools.rs +++ b/crates/tw-control/src/web_tools.rs @@ -15,6 +15,7 @@ async fn view(State(s): State) -> Json { config: s.config().web_tools.clone(), config_version: s.cfg.current().map(|c| c.version()).unwrap_or_default(), system_locale: tw_config::web_tools::system_locale(), + available_engines: tw_gateway::web_tools::available_engines(), }) } diff --git a/crates/tw-gateway/Cargo.toml b/crates/tw-gateway/Cargo.toml index 0785b080..2217796d 100644 --- a/crates/tw-gateway/Cargo.toml +++ b/crates/tw-gateway/Cargo.toml @@ -34,7 +34,8 @@ bytes = { workspace = true } chrono = { workspace = true } futures = { workspace = true } http-body-util = { workspace = true } -reqwest = { workspace = true } +url = { workspace = true } +reqwest = { workspace = true, features = ["cookies", "gzip", "brotli", "deflate", "zstd"] } rustls = { workspace = true } rustls-platform-verifier = { workspace = true } tokio-rustls = { workspace = true } diff --git a/crates/tw-gateway/src/outbound.rs b/crates/tw-gateway/src/outbound.rs index 64e39411..ff5bba36 100644 --- a/crates/tw-gateway/src/outbound.rs +++ b/crates/tw-gateway/src/outbound.rs @@ -7,6 +7,12 @@ use tw_types::msg; /// 的那批和不走代理的那批会慢慢长出不同的超时行为。 pub(crate) fn base_client_builder() -> reqwest::ClientBuilder { reqwest::Client::builder() + // Web search opts into browser content decoding. Other outbound paths + // preserve their existing wire-body semantics despite feature unification. + .no_gzip() + .no_brotli() + .no_deflate() + .no_zstd() // 分段超时。**没有整体超时** —— 一个跑了六分钟的 // Opus 任务不该被中间层掐断,让客户端自己决定何时放弃。 .connect_timeout(std::time::Duration::from_secs(10)) diff --git a/crates/tw-gateway/src/web_tools/cache.rs b/crates/tw-gateway/src/web_tools/cache.rs new file mode 100644 index 00000000..440407cf --- /dev/null +++ b/crates/tw-gateway/src/web_tools/cache.rs @@ -0,0 +1,89 @@ +//! Bounded, caller-keyed cache slots. Holding a slot coalesces concurrent work; +//! eviction only removes one idle slot and cannot split an in-flight request. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; +use std::time::Instant; +use tokio::sync::Mutex as AsyncMutex; +use tw_types::Msg; + +type Slot = Arc>>; +struct Entry { + used: Instant, + slot: Slot, +} + +pub(super) struct Cache { + capacity: usize, + entries: Mutex>>, +} + +impl Cache { + pub fn new(capacity: usize) -> Self { + Self { + capacity, + entries: Mutex::new(HashMap::new()), + } + } + + pub fn get(&self, key: String) -> Result, Msg> { + let mut entries = self.entries.lock().unwrap_or_else(|e| e.into_inner()); + if let Some(entry) = entries.get_mut(&key) { + entry.used = Instant::now(); + return Ok(entry.slot.clone()); + } + if entries.len() >= self.capacity { + let oldest = entries + .iter() + .filter(|(_, e)| Arc::strong_count(&e.slot) == 1) + .min_by_key(|(_, e)| e.used) + .map(|(key, _)| key.clone()); + if let Some(oldest) = oldest { + entries.remove(&oldest); + } else { + return Err(super::error("too many pending web operations")); + } + } + let slot = Arc::new(AsyncMutex::new(None)); + entries.insert( + key, + Entry { + used: Instant::now(), + slot: slot.clone(), + }, + ); + Ok(slot) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn eviction_retains_hot_values_and_coalesces_in_flight_work() { + let cache = Cache::new(2); + let first = cache.get("first".into()).unwrap(); + *first.lock().await = Some((Instant::now(), "saved")); + let pending = cache.get("pending".into()).unwrap(); + let guard = pending.lock().await; + let joined = cache.get("pending".into()).unwrap(); + assert!(Arc::ptr_eq(&pending, &joined)); + assert!(joined.try_lock().is_err()); + assert!( + cache.get("new".into()).is_err(), + "active work cannot be evicted" + ); + drop(guard); + drop(joined); + drop(pending); + drop(first); + // Touch the completed result: inserting another key must evict the older + // idle slot, not flush all successfully cached results. + drop(cache.get("first".into()).unwrap()); + drop(cache.get("new".into()).unwrap()); + let first = cache.get("first".into()).unwrap(); + assert_eq!(first.lock().await.as_ref().unwrap().1, "saved"); + assert_eq!(cache.entries.lock().unwrap().len(), 2); + } +} diff --git a/crates/tw-gateway/src/web_tools/collect.rs b/crates/tw-gateway/src/web_tools/collect.rs index 561a8593..b1819ac5 100644 --- a/crates/tw-gateway/src/web_tools/collect.rs +++ b/crates/tw-gateway/src/web_tools/collect.rs @@ -11,18 +11,12 @@ fn clean(s: &str) -> String { s.split_whitespace().collect::>().join(" ") } -pub(super) fn search(engine: SearchEngine, body: &str) -> Result, Msg> { - if body.contains("anomaly.js") || body.contains("challenge-form") || body.contains("b_captcha") - { - return Err(error("search engine requires a verification challenge")); - } - match engine { - SearchEngine::Bing => rss(body), - SearchEngine::Duckduckgo => duckduckgo(body), - } -} - -fn source(engine: SearchEngine, url: &str, title: &str, text: &str) -> Option { +pub(super) fn source( + engine: SearchEngine, + url: &str, + title: &str, + text: &str, +) -> Option { let mut url = super::public::parse(url).ok()?; url.set_fragment(None); let params: Vec<_> = url @@ -44,126 +38,17 @@ fn source(engine: SearchEngine, url: &str, title: &str, text: &str) -> Option Result, Msg> { - use quick_xml::events::Event; - let mut reader = quick_xml::Reader::from_str(body); - reader.config_mut().trim_text(false); - let mut items = Vec::new(); - let mut in_item = false; - let mut rss = false; - let (mut tag, mut title, mut url, mut text, mut date) = ( - Vec::new(), - String::new(), - String::new(), - String::new(), - String::new(), - ); - loop { - match reader.read_event() { - Ok(Event::Start(e)) => { - tag = e.name().as_ref().as_bytes().to_vec(); - if tag == b"rss" { - rss = true; - } - if tag == b"item" { - in_item = true; - title.clear(); - url.clear(); - text.clear(); - date.clear(); - } - } - Ok(event @ (Event::Text(_) | Event::CData(_) | Event::GeneralRef(_))) if in_item => { - let value = match &event { - Event::Text(e) => e.as_ref().to_string(), - Event::CData(e) => e.as_ref().to_string(), - Event::GeneralRef(e) => { - quick_xml::escape::unescape(&format!("&{};", e.as_ref())) - .map_err(|_| error("invalid search XML entity"))? - .into_owned() - } - _ => unreachable!(), - }; - match tag.as_slice() { - b"title" => title.push_str(&value), - b"link" => url.push_str(&value), - b"description" => text.push_str(&value), - b"pubDate" => date.push_str(&value), - _ => {} - } - } - Ok(Event::End(e)) if e.name().as_ref() == "item" => { - if let Some(mut s) = source( - SearchEngine::Bing, - &url, - &title, - &clean( - &Html::parse_fragment(&text) - .root_element() - .text() - .collect::>() - .join(" "), - ), - ) { - if !date.is_empty() { - s.published_at = Some(date.clone()); - } - items.push(s); - } - in_item = false; - } - Ok(Event::Eof) => break, - Err(_) => return Err(error("invalid search XML")), - _ => {} - } - } - if !rss { - return Err(error("search engine did not return RSS results")); - } - Ok(items) -} - -fn duckduckgo(body: &str) -> Result, Msg> { - let doc = Html::parse_document(body); - let mut result = Vec::new(); - for row in doc.select(&selector(".result")) { - let Some(link) = row.select(&selector(".result__a")).next() else { - continue; - }; - let Some(href) = link.value().attr("href") else { - continue; - }; - let absolute = if href.starts_with("//") { - format!("https:{href}") - } else { - href.to_string() - }; - let Ok(url) = reqwest::Url::parse(&absolute) else { - continue; - }; - let destination = url - .query_pairs() - .find(|(k, _)| k == "uddg") - .map(|(_, v)| v.into_owned()) - .unwrap_or(absolute); - let title = link.text().collect::>().join(" "); - let text = row - .select(&selector(".result__snippet")) - .next() - .map(|n| n.text().collect::>().join(" ")) - .unwrap_or_default(); - if let Some(s) = source(SearchEngine::Duckduckgo, &destination, &title, &text) { - result.push(s); - } - } - if result.is_empty() && !body.contains("no-results") { - return Err(error("search page layout was not recognized")); - } - Ok(result) -} - -pub(super) fn page(body: &str) -> (String, String) { +pub(super) fn page(body: &str) -> Result<(String, String), Msg> { let mut doc = Html::parse_document(body); + if doc + .select(&selector( + "#challenge-form,#cf-challenge-running,#challenge-running", + )) + .next() + .is_some() + { + return Err(error("page requires a verification challenge")); + } let remove: Vec<_> = doc .select(&selector( "script,style,noscript,nav,footer,header,aside,form,svg,template", @@ -181,16 +66,66 @@ pub(super) fn page(body: &str) -> (String, String) { .map(|n| clean(&n.text().collect::>().join(" "))) .unwrap_or_default(); let root = doc - .select(&selector("article,main,[role=main]")) + .select(&selector("main,[role=main]")) .next() + .or_else(|| doc.select(&selector("article")).next()) + .or_else(|| doc.select(&selector("body")).next()) .unwrap_or_else(|| doc.root_element()); - let text = root - .text() + // Text nodes are not paragraphs: inline , and elements must + // not split a sentence into unrelated lines. Use an explicit stack so a + // deeply nested public page cannot overflow the call stack. + let mut text = String::new(); + let mut stack: Vec<_> = root.children().rev().map(|node| (node, false)).collect(); + while let Some((node, closing)) = stack.pop() { + if let Some(element) = node.value().as_element() { + let block = matches!( + element.name(), + "p" | "div" + | "section" + | "article" + | "main" + | "li" + | "ul" + | "ol" + | "pre" + | "blockquote" + | "h1" + | "h2" + | "h3" + | "h4" + | "h5" + | "h6" + | "br" + | "tr" + | "dt" + | "dd" + ); + if block && !text.ends_with('\n') { + text.push('\n'); + } + if closing && matches!(element.name(), "td" | "th") { + text.push(' '); + } + if !closing { + stack.push((node, true)); + stack.extend(node.children().rev().map(|child| (child, false))); + } + } else if let Some(value) = node.value().as_text() { + text.push_str(value); + } + } + let text = text + .lines() .map(clean) .filter(|s| !s.is_empty()) .collect::>() .join("\n"); - (title.chars().take(500).collect(), text) + if text.is_empty() { + return Err(error( + "page contained no readable text; it may require JavaScript", + )); + } + Ok((title.chars().take(500).collect(), text)) } pub(super) fn excerpt(text: &str, query: &str, limit: usize) -> String { @@ -199,19 +134,20 @@ pub(super) fn excerpt(text: &str, query: &str, limit: usize) -> String { .filter(|s| s.chars().count() > 2) .map(str::to_lowercase) .collect(); - let mut lines: Vec<_> = text.lines().enumerate().collect(); - lines.sort_by_key(|(i, line)| { + let lines: Vec<_> = text.lines().collect(); + let score = |line: &str| { let l = line.to_lowercase(); - ( - std::cmp::Reverse(words.iter().filter(|w| l.contains(w.as_str())).count()), - *i, - ) - }); - lines + words.iter().filter(|w| l.contains(w.as_str())).count() + }; + let best = lines .iter() - .take(12) - .map(|(_, s)| *s) - .collect::>() + .enumerate() + .max_by_key(|(i, line)| (score(line), std::cmp::Reverse(*i))) + .map(|(i, _)| i) + .unwrap_or(0); + // Preserve source order and surrounding context instead of sorting sentences + // by keyword count, which can change what the page actually says. + lines[best.saturating_sub(1)..lines.len().min(best + 11)] .join("\n") .chars() .take(limit) @@ -244,38 +180,52 @@ pub(super) fn find(text: &str, pattern: &str, limit: usize) -> String { mod tests { use super::*; #[test] - fn parses_search_sources_and_rejects_challenges() { - let r = search(SearchEngine::Bing, "Releasehttps://example.org/?utm_source=xVersion 3").unwrap(); - assert_eq!(r[0].url, "https://example.org/"); - let html = r#""#; - assert_eq!( - search(SearchEngine::Duckduckgo, html).unwrap()[0].text, - "Version 3" - ); - assert!(search(SearchEngine::Duckduckgo, "
").is_err()); - assert!(search(SearchEngine::Bing, "Access denied").is_err()); - } - #[test] fn extracts_readable_text_and_finds_unicode_context() { let (_, text) = page( "Doc

你好

Version 3

Next

", - ); + ).unwrap(); assert!(!text.contains("secret")); assert!(!text.contains("menu")); let found = find(&text, "VERSION", 1000); assert!(found.contains("你好")); assert!(found.contains("Version 3")); } -} -#[cfg(test)] -mod entity_tests { - use super::*; #[test] - fn rss_preserves_entities_cdata_and_query_parameters() { - let result = search(SearchEngine::Bing, r#"Rust & Web 中文https://example.org/?a=1&b=2A & B

]]>
"#).unwrap(); - assert_eq!(result[0].title, "Rust & Web 中文"); - assert_eq!(result[0].url, "https://example.org/?a=1&b=2"); - assert_eq!(result[0].text, "A & B"); + fn inline_markup_does_not_break_sentences_or_phrase_searches() { + let (_, text) = page( + "Doc

Use asyncio.TaskGroup to await all tasks.

Errors cancel the remaining tasks.

", + ).unwrap(); + assert_eq!( + text, + "Use asyncio.TaskGroup to await all tasks.\nErrors cancel the remaining tasks." + ); + assert!(find(&text, "await all tasks", 1000).contains("Use asyncio.TaskGroup")); + } + + #[test] + fn excerpts_keep_context_in_the_original_order() { + let text = "Introduction\nDo not use this on older versions.\nTaskGroup first appeared in Python 3.11.\nIn earlier versions, use gather.\nTaskGroup handles child tasks together."; + let extracted = excerpt(text, "TaskGroup Python", 1000); + assert!(extracted.starts_with("Do not use this on older versions.")); + assert!( + extracted.find("first appeared").unwrap() + < extracted.find("In earlier versions").unwrap() + ); + assert!( + extracted.find("In earlier versions").unwrap() + < extracted.find("handles child").unwrap() + ); + assert_eq!(excerpt("", "anything", 100), ""); + } + + #[test] + fn empty_pages_and_verification_are_not_returned_as_content() { + assert!(page("").is_err()); + assert!(page("Verify you are human
").is_err()); + // Cloudflare also injects scripts into successfully loaded normal pages. + assert!(page("
Article text
").is_ok()); + let (_, text) = page("
Advertisement
Version3.11
").unwrap(); + assert_eq!(text, "Version 3.11"); } } diff --git a/crates/tw-gateway/src/web_tools/mod.rs b/crates/tw-gateway/src/web_tools/mod.rs index 379545e2..2430ba72 100644 --- a/crates/tw-gateway/src/web_tools/mod.rs +++ b/crates/tw-gateway/src/web_tools/mod.rs @@ -1,38 +1,107 @@ //! Search and public-page tools, shared by the gateway and the control-plane tester. +mod cache; mod collect; mod history; -mod locale; pub(crate) mod protocol; +mod providers; mod public; +mod quality; +mod session; pub(crate) mod wire; use std::collections::HashMap; -use std::sync::{Arc, Mutex}; +use std::sync::Arc; use std::time::{Duration, Instant}; use tokio::sync::{Mutex as AsyncMutex, Semaphore}; use tw_api::{SearchEngine, WebToolRequest, WebToolResult, WebToolsConfig}; use tw_types::{Msg, msg}; -type PageCached = Arc>>; - -type Cached = Arc>>; - pub struct WebTools { pub(crate) history: Arc, config: WebToolsConfig, client: public::ClientSettings, slots: Semaphore, - cache: Mutex>, - pages: Mutex>, + cache: cache::Cache, + pages: cache::Cache<(String, String)>, engines: AsyncMutex>, + sessions: session::Pool, } #[derive(Default)] struct EngineState { next: Option, failed_until: Option, + last_failure: Option, +} + +/// Capabilities come from the same registry that executes requests. Desktop +/// clients can offer new engines without hard-coding an exhaustive enum list. +pub fn available_engines() -> Vec { + providers::AUTOMATIC + .iter() + .map(|&engine| { + let spec = providers::get(engine).spec(); + tw_api::WebSearchEngine { + id: engine, + name: spec.name.into(), + min_interval_ms: spec.min_interval_ms, + } + }) + .collect() +} + +fn engine_name(engine: SearchEngine) -> &'static str { + providers::get(engine).spec().name +} + +fn failure_detail(e: &Msg) -> &str { + if e.code == "gw.web_tools.failed" { + e.args.get("detail").map(String::as_str).unwrap_or(&e.text) + } else { + &e.text + } +} + +fn http_failure(status: reqwest::StatusCode, headers: &http::HeaderMap) -> Msg { + let mut failure = error(format!("source returned HTTP {}", status.as_u16())); + failure + .args + .insert("http_status".into(), status.as_u16().to_string()); + if matches!(status.as_u16(), 429 | 503) + && let Some(value) = headers.get("retry-after").and_then(|v| v.to_str().ok()) + { + let seconds = value.trim().parse::().ok().or_else(|| { + chrono::DateTime::parse_from_rfc2822(value) + .ok() + .map(|at| (at.timestamp() - chrono::Utc::now().timestamp()).max(0) as u64) + }); + if let Some(seconds) = seconds { + failure + .args + .insert("retry_after_secs".into(), seconds.to_string()); + } + } + failure +} + +impl EngineState { + fn cooling_down(&self, engine: SearchEngine) -> Msg { + let seconds = self + .failed_until + .map(|until| until.saturating_duration_since(Instant::now()).as_secs() + 1) + .unwrap_or(0); + let cause = self + .last_failure + .as_ref() + .map(failure_detail) + .unwrap_or("previous search failed"); + error(format!( + "{} is cooling down; retry in {seconds}s. Last failure: {cause}", + engine_name(engine) + )) + } } pub(crate) fn error(detail: impl Into) -> Msg { @@ -113,10 +182,7 @@ impl WebTools { result: WebToolResult, ) { let key = Self::cache_key(identity, request, &self.locale()).unwrap(); - self.cache.lock().unwrap().insert( - key, - Arc::new(AsyncMutex::new(Some((Instant::now(), result)))), - ); + *self.cache.get(key).unwrap().lock().await = Some((Instant::now(), result)); } pub fn new(config: &tw_config::Config) -> Result { Ok(Self { @@ -124,9 +190,10 @@ impl WebTools { config: config.web_tools.clone(), client: public::ClientSettings::new(config)?, slots: Semaphore::new(config.web_tools.max_concurrent as usize), - cache: Mutex::new(HashMap::new()), - pages: Mutex::new(HashMap::new()), + cache: cache::Cache::new(256), + pages: cache::Cache::new(32), engines: AsyncMutex::new(HashMap::new()), + sessions: session::Pool::default(), }) } @@ -155,19 +222,7 @@ impl WebTools { } else { format!("{key}\n{}", serde_json::json!([allowed, blocked])) }; - let entry = { - let mut cache = self.cache.lock().unwrap_or_else(|e| e.into_inner()); - if cache.len() >= 256 { - cache.retain(|_, e| Arc::strong_count(e) > 1); - } - if cache.len() >= 256 && !cache.contains_key(&key) { - return Err(error("too many pending web operations")); - } - cache - .entry(key) - .or_insert_with(|| Arc::new(AsyncMutex::new(None))) - .clone() - }; + let entry = self.cache.get(key)?; // Holding this lock coalesces concurrent requests without leaving a background // operation behind when the last caller is cancelled. let mut cached = entry.lock().await; @@ -229,7 +284,7 @@ impl WebTools { public::validate_domains(allowed_domains)?; public::validate_domains(blocked_domains)?; let engines = if self.config.engines.is_empty() { - vec![SearchEngine::Bing, SearchEngine::Duckduckgo] + providers::AUTOMATIC.to_vec() } else { self.config.engines.clone() }; @@ -243,49 +298,102 @@ impl WebTools { .saturating_sub(Duration::from_millis(20)); let budget = remaining / (engine_count - index + usize::from(self.config.read_pages > 0)) as u32; + let mut searching = false; let attempt = tokio::time::timeout(budget, async { if !self.reserve(engine).await { return Ok(None); } - self.search(engine, query, allowed_domains, blocked_domains, locale) - .await - .map(Some) + searching = true; + self.search( + identity, + engine, + query, + allowed_domains, + blocked_domains, + locale, + ) + .await + .map(Some) }) .await - .unwrap_or_else(|_| Err(error("search engine timed out"))); + .unwrap_or_else(|_| { + Err(error(if searching { + "search engine timed out" + } else { + "search rate-limit wait timed out" + })) + }); match attempt { - Ok(None) => continue, + Ok(None) => { + let engines = self.engines.lock().await; + if let Some(state) = engines.get(&engine) { + result.warnings.push(state.cooling_down(engine)); + } + } Ok(Some(sources)) => { + if let Some(state) = self.engines.lock().await.get_mut(&engine) { + state.failed_until = None; + state.last_failure = None; + } succeeded = true; for source in sources { if !result.sources.iter().any(|s| s.url == source.url) { result.sources.push(source); } } - if result.sources.len() >= self.config.max_results as usize { + // Engine order is failover order. Filling a healthy + // engine's shorter result set from a worse fallback + // can reintroduce unrelated pages. + if !result.sources.is_empty() { break; } } Err(e) => { - self.engines - .lock() - .await - .entry(engine) - .or_default() - .failed_until = Some(Instant::now() + Duration::from_secs(60)); + let query_mismatch = e.code == "gw.web_tools.query_mismatch"; + let cooldown = e + .args + .get("retry_after_secs") + .and_then(|v| v.parse::().ok()) + .and_then(|seconds| { + Instant::now().checked_add(Duration::from_secs(seconds.max(1))) + }) + .unwrap_or_else(|| Instant::now() + Duration::from_secs(60)); + let e = + error(format!("{}: {}", engine_name(engine), failure_detail(&e))); + // A poor match for one query does not make the engine + // unavailable for every other caller for a minute. + if searching && !query_mismatch { + let mut engines = self.engines.lock().await; + let state = engines.entry(engine).or_default(); + state.failed_until = Some(cooldown); + state.last_failure = Some(e.clone()); + } result.warnings.push(e); } } } if !succeeded { - return Err(error( - "all configured search engines failed or are cooling down", - )); + // Do not discard the engine errors: a challenge, a broken proxy, + // and a timeout need different remedies. Retries during cooldown + // retain the original cause and say when another attempt is due. + let reasons = result + .warnings + .iter() + .map(failure_detail) + .collect::>() + .join("; "); + return Err(error(format!( + "all configured search engines failed or are cooling down. {reasons}" + ))); } result.sources.truncate(self.config.max_results as usize); let mut read = 0; for source in &mut result.sources { - if read >= self.config.read_pages || source.text.chars().count() >= 300 { + if read >= self.config.read_pages + || source.text.chars().count() >= 300 + || reqwest::Url::parse(&source.url) + .is_ok_and(|url| url.path().to_ascii_lowercase().ends_with(".pdf")) + { continue; } read += 1; @@ -303,7 +411,19 @@ impl WebTools { .await .unwrap_or_else(|_| Err(error("optional page reading timed out"))) { - Ok((_, text)) => source.text = collect::excerpt(&text, query, 3000), + Ok((_, text)) => { + let excerpt = collect::excerpt(&text, query, 3000); + if !excerpt.is_empty() && !source.text.contains(&excerpt) { + // The engine snippet may be more focused than the + // landing page. Keep it instead of replacing it + // with navigation text or unrelated paragraphs. + source.text = format!("{}\n\n{excerpt}", source.text) + .trim() + .chars() + .take(4000) + .collect(); + } + } Err(e) => result.warnings.push(e), } } @@ -344,62 +464,101 @@ impl WebTools { } async fn reserve(&self, engine: SearchEngine) -> bool { - let delay = { - let mut engines = self.engines.lock().await; - let state = engines.entry(engine).or_default(); - let now = Instant::now(); - if state.failed_until.is_some_and(|until| until > now) { - return false; - } - let at = state.next.unwrap_or(now).max(now); - state.next = Some(at + Duration::from_millis(u64::from(self.config.interval_ms))); - at.saturating_duration_since(now) - }; - tokio::time::sleep(delay).await; - true + loop { + let delay = { + let mut engines = self.engines.lock().await; + let state = engines.entry(engine).or_default(); + let now = Instant::now(); + if state.failed_until.is_some_and(|until| until > now) { + return false; + } + let at = state.next.unwrap_or(now); + if at <= now { + state.next = Some( + now + Duration::from_millis(u64::from( + self.config + .interval_ms + .max(providers::get(engine).spec().min_interval_ms), + )), + ); + return true; + } + at.saturating_duration_since(now) + }; + // Reserve only when a request can actually start. Cancelled waiters + // must not accumulate future slots and delay unrelated searches. + tokio::time::sleep(delay).await; + } } async fn search( &self, + identity: &str, engine: SearchEngine, query: &str, allowed: &[String], blocked: &[String], locale: &tw_api::WebSearchLocale, ) -> Result, Msg> { - let mut q = query.to_string(); - if !allowed.is_empty() { - q.push_str(&format!( - " ({})", - allowed - .iter() - .map(|d| format!("site:{d}")) - .collect::>() - .join(" OR ") - )); - } else if !self.config.allowed_domains.is_empty() { - q.push_str(&format!( - " ({})", - self.config - .allowed_domains - .iter() - .map(|d| format!("site:{d}")) - .collect::>() - .join(" OR ") - )); + let provider = providers::get(engine); + let spec = provider.spec(); + let session = self.sessions.get(identity, spec, locale)?; + let context = session::Request { + spec, + kind: providers::Kind::Document, + session: session.as_ref().map(|s| &s.session), + locale, + }; + if let Some(session) = &session { + session + .session + .ready + .get_or_try_init(|| async { + let home = spec + .home + .expect("session requires a homepage") + .parse() + .unwrap(); + let (_, body) = self.fetch(home, None, Some(context), None).await?; + provider.validate_home(&body) + }) + .await?; } - let mut url = reqwest::Url::parse(match engine { - SearchEngine::Bing => "https://www.bing.com/search", - SearchEngine::Duckduckgo => "https://html.duckduckgo.com/html/", + let query_context = providers::Query { + text: query.trim(), + allowed: if allowed.is_empty() { + &self.config.allowed_domains + } else { + allowed + }, + locale, + }; + let mut requests = 0; + let mut sources = providers::run(provider, &query_context, |request| { + let retry = requests > 0; + requests += 1; + async move { + // The first reservation is made by the scheduler. Further exchange + // steps with a source minimum must obey that same engine-wide pace. + if retry && spec.min_interval_ms > 0 && !self.reserve(engine).await { + return Err(error("search engine entered cooldown")); + } + let context = session::Request { + kind: request.kind, + ..context + }; + self.fetch(request.url, None, Some(context), request.form.as_deref()) + .await + .map(|(_, body)| body) + } }) - .expect("built-in URL"); - url.query_pairs_mut().append_pair("q", &q); - if engine == SearchEngine::Bing { - url.query_pairs_mut().append_pair("format", "rss"); + .await?; + let had_results = !sources.is_empty(); + let usable = quality::filter(query, &mut sources); + if had_results && !usable { + return Err(msg!("gw.web_tools.query_mismatch" => + "The search engine returned results that did not match the query.")); } - locale::apply(&mut url, engine, locale); - let (_, body) = self.fetch(url, None).await?; - let mut sources = collect::search(engine, &body)?; sources.retain(|s| { reqwest::Url::parse(&s.url).ok().is_some_and(|url| { self.allowed(&url) && public::domains_allow(&url, allowed, blocked) @@ -427,19 +586,7 @@ impl WebTools { "{identity}\n{url}\n{}", serde_json::json!([allowed, blocked]) ); - let entry = { - let mut pages = self.pages.lock().unwrap_or_else(|e| e.into_inner()); - if pages.len() >= 32 { - pages.retain(|_, p| Arc::strong_count(p) > 1); - } - if pages.len() >= 32 && !pages.contains_key(&key) { - return Err(error("too many pending page reads")); - } - pages - .entry(key) - .or_insert_with(|| Arc::new(AsyncMutex::new(None))) - .clone() - }; + let entry = self.pages.get(key)?; let mut cache = entry.lock().await; if let Some((at, page)) = &*cache && at.elapsed() < Duration::from_secs(u64::from(self.config.cache_secs)) @@ -463,9 +610,11 @@ impl WebTools { if !self.allowed(&url) { return Err(error("page is excluded by the configured domain rules")); } - let (kind, body) = self.fetch(url, Some((allowed, blocked))).await?; + let (kind, body) = self + .fetch(url, Some((allowed, blocked)), None, None) + .await?; if kind.contains("html") { - Ok(collect::page(&body)) + collect::page(&body) } else if kind.starts_with("text/") { Ok((String::new(), body)) } else { @@ -477,17 +626,36 @@ impl WebTools { &self, mut url: reqwest::Url, filters: Option<(&[String], &[String])>, + search: Option>, + form: Option<&str>, ) -> Result<(String, String), Msg> { use futures::StreamExt; for _ in 0..6 { - let addresses = public::check(&url).await?; if let Some((allowed, blocked)) = filters && (!self.allowed(&url) || !public::domains_allow(&url, allowed, blocked)) { return Err(error("redirect is excluded by the configured domain rules")); } - let response = self.client.get(&url, &addresses).await?; + let response = match search { + Some(context) => { + let response = self + .client + .get_search(context.spec, &url, context.headers(&url)?, form) + .await?; + context.remember(&url, response.headers())?; + response + } + None => { + let addresses = public::check(&url).await?; + self.client.get(&url, &addresses).await? + } + }; if response.status().is_redirection() { + // Form queries never follow redirects: do not replay their body + // into a login, verification, or different document endpoint. + if form.is_some() { + return Err(http_failure(response.status(), response.headers())); + } let to = response .headers() .get("location") @@ -501,10 +669,7 @@ impl WebTools { continue; } if !response.status().is_success() { - return Err(error(format!( - "source returned HTTP {}", - response.status().as_u16() - ))); + return Err(http_failure(response.status(), response.headers())); } let kind = response .headers() @@ -512,6 +677,10 @@ impl WebTools { .and_then(|v| v.to_str().ok()) .unwrap_or("application/octet-stream") .to_lowercase(); + // Reject binary documents before trying to decode them as UTF-8. + if search.is_none() && !kind.contains("html") && !kind.starts_with("text/") { + return Err(error("page must be HTML or plain text")); + } let mut bytes = Vec::new(); let mut stream = response.bytes_stream(); while let Some(chunk) = stream.next().await { @@ -532,11 +701,358 @@ impl WebTools { #[cfg(test)] mod tests { use super::*; + + #[test] + fn rate_limits_keep_server_retry_after_seconds_and_dates() { + let mut headers = http::HeaderMap::new(); + headers.insert("retry-after", "120".parse().unwrap()); + assert_eq!( + http_failure(reqwest::StatusCode::TOO_MANY_REQUESTS, &headers).arg("retry_after_secs"), + "120" + ); + let future = chrono::Utc::now() + chrono::Duration::minutes(2); + headers.insert("retry-after", future.to_rfc2822().parse().unwrap()); + let seconds: u64 = http_failure(reqwest::StatusCode::SERVICE_UNAVAILABLE, &headers) + .arg("retry_after_secs") + .parse() + .unwrap(); + assert!((118..=120).contains(&seconds)); + headers.insert("retry-after", "invalid".parse().unwrap()); + assert!( + http_failure(reqwest::StatusCode::TOO_MANY_REQUESTS, &headers) + .arg("retry_after_secs") + .is_empty() + ); + } + + #[tokio::test] + #[ignore = "live traditional-engine relevance, filters, and concurrent cache checks"] + async fn live_direct_search_matrix() { + live_engine_matrix(&[SearchEngine::Duckduckgo]).await; + } + + #[tokio::test] + #[ignore = "live Bing relevance with a configured Chinese locale, filters, and cache"] + async fn live_bing_search_matrix() { + live_engine_matrix(&[SearchEngine::Bing]).await; + } + + #[tokio::test] + #[ignore = "live Bing-first search with DuckDuckGo failover, filters, and cache"] + async fn live_bing_failover_matrix() { + live_engine_matrix(&[SearchEngine::Bing, SearchEngine::Duckduckgo]).await; + } + + #[tokio::test] + #[ignore = "live Google Docs relevance, domain filters, and concurrent cache checks"] + async fn live_google_docs_search_matrix() { + live_engine_matrix(&[SearchEngine::GoogleDocs]).await; + } + + #[tokio::test] + #[ignore = "live automatic search with Google Docs and traditional-engine failover"] + async fn live_automatic_search_matrix() { + live_engine_matrix(providers::AUTOMATIC).await; + } + + async fn live_engine_matrix(engines: &[SearchEngine]) { + let mut config = tw_config::Config::default(); + config.web_tools.engines = engines.to_vec(); + config.web_tools.language = "zh-Hans".into(); + config.web_tools.region = "CN".into(); + config.web_tools.read_pages = 0; + let tools = WebTools::new(&config).unwrap(); + let request = |query: &str| WebToolRequest::Search { + query: query.into(), + allowed_domains: vec![], + blocked_domains: vec![], + }; + let mut failures = Vec::new(); + for (query, domain, path) in [ + ( + "Python asyncio TaskGroup documentation", + "python.org", + "asyncio-task", + ), + ("Rust OnceLock documentation", "rust-lang.org", "OnceLock"), + ("SQLite release notes", "sqlite.org", "changes"), + ("劳动合同法 第三十八条", "gov.cn", ""), + ( + "2025 Nobel physics prize winners", + "nobelprize.org", + "physics/2025", + ), + ( + "Apple Mac mini M4 technical specifications", + "apple.com", + "121555", + ), + ("中华人民共和国民法典 第一千零六十二条", "gov.cn", ""), + ("OpenAI Responses API reference", "openai.com", "responses"), + ] { + let result = match tools.execute("matrix", &request(query)).await { + Ok(result) => result, + Err(error) => { + failures.push(format!("{query}: {error}")); + continue; + } + }; + eprintln!( + "{query}: {} sources, {} ms, {:?}, {:?}", + result.sources.len(), + result.elapsed_ms, + result.sources.first().and_then(|s| s.engine), + result + .sources + .iter() + .take(3) + .map(|s| &s.url) + .collect::>() + ); + if engines.len() == 1 { + assert!(result.warnings.is_empty(), "{query}: {:?}", result.warnings); + } + if !result.sources.iter().any(|source| { + reqwest::Url::parse(&source.url).is_ok_and(|url| { + public::domains_allow(&url, &[domain.into()], &[]) && url.path().contains(path) + }) + }) { + failures.push(format!("{query}: missing {domain}/{path}")); + } + assert!( + result + .sources + .iter() + .all(|s| s.engine.is_some_and(|e| engines.contains(&e))) + ); + } + assert!( + failures.is_empty(), + "{engines:?} failed relevance checks: {failures:?}" + ); + let req = request("PostgreSQL transaction isolation documentation"); + let results = + futures::future::join_all((0..4).map(|_| tools.execute("matrix", &req))).await; + let results: Vec<_> = results.into_iter().collect::>().unwrap(); + assert_eq!(results.iter().filter(|r| r.cached).count(), 3); + assert!(results.iter().all(|r| !r.sources.is_empty())); + for (allowed_domains, blocked_domains) in [ + (vec!["sqlite.org".into()], vec![]), + (vec![], vec!["sqlite.org".into()]), + ] { + let result = tools + .execute( + "matrix", + &WebToolRequest::Search { + query: "SQLite release notes".into(), + allowed_domains: allowed_domains.clone(), + blocked_domains: blocked_domains.clone(), + }, + ) + .await + .unwrap(); + assert!(!result.sources.is_empty()); + assert!(result.sources.iter().all(|s| public::domains_allow( + &reqwest::Url::parse(&s.url).unwrap(), + &allowed_domains, + &blocked_domains + ))); + } + } + + #[tokio::test] + async fn source_minimum_interval_and_capabilities_share_the_registry() { + let mut config = tw_config::Config::default(); + config.web_tools.interval_ms = 250; + let tools = WebTools::new(&config).unwrap(); + assert!(tools.reserve(SearchEngine::GoogleDocs).await); + let next = tools.engines.lock().await[&SearchEngine::GoogleDocs] + .next + .unwrap(); + assert!(next.saturating_duration_since(Instant::now()) > Duration::from_millis(5900)); + assert!( + tokio::time::timeout( + Duration::from_millis(10), + tools.reserve(SearchEngine::GoogleDocs) + ) + .await + .is_err() + ); + assert_eq!( + tools.engines.lock().await[&SearchEngine::GoogleDocs].next, + Some(next) + ); + let available = available_engines(); + assert_eq!(available[0].id, SearchEngine::GoogleDocs); + assert_eq!(available[0].min_interval_ms, 6000); + } + + #[tokio::test] + async fn cancelled_waiters_do_not_reserve_more_engine_slots() { + let tools = WebTools::new(&tw_config::Config::default()).unwrap(); + assert!(tools.reserve(SearchEngine::Bing).await); + let next = tools.engines.lock().await[&SearchEngine::Bing].next; + for _ in 0..3 { + assert!( + tokio::time::timeout(Duration::from_millis(1), tools.reserve(SearchEngine::Bing)) + .await + .is_err() + ); + assert_eq!(tools.engines.lock().await[&SearchEngine::Bing].next, next); + } + } + + #[tokio::test] + async fn rate_limit_wait_timeout_does_not_mark_the_engine_unavailable() { + let mut config = tw_config::Config::default(); + config.web_tools.timeout_secs = 1; + config.web_tools.read_pages = 0; + config.web_tools.engines = vec![SearchEngine::Bing]; + let tools = WebTools::new(&config).unwrap(); + tools.engines.lock().await.insert( + SearchEngine::Bing, + EngineState { + next: Some(Instant::now() + Duration::from_secs(60)), + ..Default::default() + }, + ); + let failure = tools + .execute( + "test", + &WebToolRequest::Search { + query: "SQLite release notes".into(), + allowed_domains: vec![], + blocked_domains: vec![], + }, + ) + .await + .unwrap_err(); + assert!(failure.text.contains("rate-limit wait"), "{failure}"); + assert!( + tools.engines.lock().await[&SearchEngine::Bing] + .failed_until + .is_none() + ); + } + + #[tokio::test] + async fn unavailable_engines_keep_their_causes_and_retry_deadline() { + let mut config = tw_config::Config::default(); + config.web_tools.engines = vec![SearchEngine::Bing, SearchEngine::Duckduckgo]; + let tools = WebTools::new(&config).unwrap(); + let until = Instant::now() + Duration::from_secs(60); + for (engine, reason) in [ + (SearchEngine::Bing, "source returned HTTP 429"), + ( + SearchEngine::Duckduckgo, + "search engine requires a verification challenge", + ), + ] { + tools.engines.lock().await.insert( + engine, + EngineState { + next: None, + failed_until: Some(until), + last_failure: Some(error(format!("{}: {reason}", engine_name(engine)))), + }, + ); + } + for query in ["SQLite release notes", "Rust documentation"] { + let failure = tools + .execute( + "test", + &WebToolRequest::Search { + query: query.into(), + allowed_domains: vec![], + blocked_domains: vec![], + }, + ) + .await + .unwrap_err(); + assert!(failure.text.contains("Bing"), "{failure}"); + assert!(failure.text.contains("429"), "{failure}"); + assert!(failure.text.contains("DuckDuckGo"), "{failure}"); + assert!(failure.text.contains("verification challenge"), "{failure}"); + assert!(failure.text.contains("retry in"), "{failure}"); + assert_eq!(failure.text.matches("Web tools:").count(), 1, "{failure}"); + for state in tools.engines.lock().await.values() { + assert_eq!( + state.failed_until, + Some(until), + "retry must not extend cooldown" + ); + } + } + } + + #[tokio::test] + #[ignore = "uses the public DuckDuckGo search engine"] + async fn live_search_duckduckgo() { + live_duckduckgo_after_cooldown(false).await; + } + + #[tokio::test] + #[ignore = "live DuckDuckGo fallback while Google Docs is cooling down"] + async fn live_search_google_cooldown_falls_back() { + live_duckduckgo_after_cooldown(true).await; + } + + async fn live_duckduckgo_after_cooldown(google_failed: bool) { + let mut config = tw_config::Config::default(); + config.web_tools.engines = vec![SearchEngine::Duckduckgo]; + if google_failed { + config.web_tools.engines.insert(0, SearchEngine::GoogleDocs); + } + config.web_tools.read_pages = 0; + let tools = WebTools::new(&config).unwrap(); + if google_failed { + tools.engines.lock().await.insert( + SearchEngine::GoogleDocs, + EngineState { + failed_until: Some(Instant::now() + Duration::from_secs(60)), + last_failure: Some(error("Google (Docs): source returned HTTP 500")), + ..Default::default() + }, + ); + } + let result = tools + .execute( + "live-test", + &WebToolRequest::Search { + query: "SQLite release notes".into(), + allowed_domains: vec![], + blocked_domains: vec![], + }, + ) + .await + .unwrap(); + if google_failed { + assert!(result.warnings.iter().any(|w| w.text.contains("500"))); + } + assert!( + result.sources.iter().any(|s| { + reqwest::Url::parse(&s.url).is_ok_and(|url| { + url.host_str() + .is_some_and(|host| host == "sqlite.org" || host.ends_with(".sqlite.org")) + && url.path().contains("changes") + }) + }), + "release notes query must find SQLite's release history: {:?}", + result.sources + ); + assert!( + result + .sources + .iter() + .all(|s| s.engine == Some(SearchEngine::Duckduckgo)) + ); + } + #[tokio::test] #[ignore = "uses public search engines and public webpages"] async fn live_search_read_and_find() { let mut config = tw_config::Config::default(); - config.web_tools.engines = vec![SearchEngine::Bing]; + config.web_tools.engines = vec![SearchEngine::Duckduckgo]; config.web_tools.read_pages = 0; let tools = WebTools::new(&config).unwrap(); let search = tools @@ -550,7 +1066,17 @@ mod tests { ) .await .unwrap(); - assert!(!search.sources.is_empty()); + assert!( + search.sources.iter().any(|s| { + reqwest::Url::parse(&s.url).is_ok_and(|url| { + url.host_str() + .is_some_and(|host| host == "sqlite.org" || host.ends_with(".sqlite.org")) + && url.path().contains("changes") + }) + }), + "release notes query must find SQLite's release history: {:?}", + search.sources + ); let page = tools .execute( "live-test", diff --git a/crates/tw-gateway/src/web_tools/providers/bing.rs b/crates/tw-gateway/src/web_tools/providers/bing.rs new file mode 100644 index 00000000..c22c042d --- /dev/null +++ b/crates/tw-gateway/src/web_tools/providers/bing.rs @@ -0,0 +1,282 @@ +use super::super::{collect::source, error}; +use scraper::{Html, Selector}; +use std::sync::LazyLock; +use tw_api::{SearchEngine, WebSource}; +use tw_types::Msg; + +static CHALLENGE: LazyLock = LazyLock::new(|| { + Selector::parse("#challenge-form, #anomaly-form, #b_captcha, .b_captcha, #turnstile-widget") + .unwrap() +}); +static ROWS: LazyLock = LazyLock::new(|| Selector::parse("#b_results .b_algo").unwrap()); +static LINK: LazyLock = LazyLock::new(|| Selector::parse("h2 a[href]").unwrap()); +static SNIPPET: LazyLock = + LazyLock::new(|| Selector::parse(".b_caption p, .b_snippet").unwrap()); +static EMPTY: LazyLock = LazyLock::new(|| Selector::parse("#b_results .b_no").unwrap()); +fn clean(s: &str) -> String { + s.split_whitespace().collect::>().join(" ") +} + +fn verification(doc: &Html) -> Result<(), Msg> { + if doc.select(&CHALLENGE).next().is_some() { + Err(error("search engine requires a verification challenge")) + } else { + Ok(()) + } +} + +fn html_results(body: &str) -> Result, Msg> { + use base64::Engine; + let doc = Html::parse_document(body); + verification(&doc)?; + let mut sources = Vec::new(); + let base = reqwest::Url::parse(URL).unwrap(); + for row in doc.select(&ROWS) { + let Some(link) = row.select(&LINK).next() else { + continue; + }; + let Some(href) = link.attr("href") else { + continue; + }; + let Ok(url) = base.join(href) else { continue }; + let destination = if matches!( + url.host_str(), + Some("www.bing.com" | "cn.bing.com" | "bing.com") + ) { + if !matches!(url.path(), "/ck/a" | "/ck/a/") { + continue; + } + let Some(encoded) = url + .query_pairs() + .find_map(|(key, value)| (key == "u").then(|| value.into_owned())) + else { + continue; + }; + let Some(encoded) = encoded.strip_prefix("a1") else { + continue; + }; + let Ok(bytes) = base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(encoded.trim_end_matches('=')) + else { + continue; + }; + let Ok(destination) = String::from_utf8(bytes) else { + continue; + }; + destination + } else { + url.to_string() + }; + let title = link.text().collect::(); + let snippet = row + .select(&SNIPPET) + .next() + .map(|node| node.text().collect::()) + .unwrap_or_default(); + if let Some(source) = source(SearchEngine::Bing, &destination, &title, &snippet) { + sources.push(source); + } + } + if sources.is_empty() && doc.select(&EMPTY).next().is_none() { + return Err(error("Bing search page layout was not recognized")); + } + Ok(sources) +} + +fn rss(body: &str) -> Result, Msg> { + use quick_xml::events::Event; + let mut reader = quick_xml::Reader::from_str(body); + reader.config_mut().trim_text(false); + let mut items = Vec::new(); + let mut in_item = false; + let mut rss = false; + let (mut tag, mut title, mut url, mut text) = + (Vec::new(), String::new(), String::new(), String::new()); + loop { + match reader.read_event() { + Ok(Event::Start(e)) => { + tag = e.name().as_ref().as_bytes().to_vec(); + if tag == b"rss" { + rss = true; + } + if tag == b"item" { + in_item = true; + title.clear(); + url.clear(); + text.clear(); + } + } + Ok(event @ (Event::Text(_) | Event::CData(_) | Event::GeneralRef(_))) if in_item => { + let value = match &event { + Event::Text(e) => e.as_ref().to_string(), + Event::CData(e) => e.as_ref().to_string(), + Event::GeneralRef(e) => { + quick_xml::escape::unescape(&format!("&{};", e.as_ref())) + .map_err(|_| error("invalid search XML entity"))? + .into_owned() + } + _ => unreachable!(), + }; + match tag.as_slice() { + b"title" => title.push_str(&value), + b"link" => url.push_str(&value), + b"description" => text.push_str(&value), + _ => {} + } + } + Ok(Event::End(e)) if e.name().as_ref() == "item" => { + if let Some(s) = source( + SearchEngine::Bing, + &url, + &title, + &clean( + &Html::parse_fragment(&text) + .root_element() + .text() + .collect::>() + .join(" "), + ), + ) { + // Bing RSS dates describe the search item, not a verified + // publication date of the linked page. Do not invent recency. + items.push(s); + } + in_item = false; + } + Ok(Event::Eof) => break, + Err(_) => return Err(error("invalid search XML")), + _ => {} + } + } + if !rss { + return Err(error("search engine did not return RSS results")); + } + Ok(items) +} + +fn search(body: &str) -> Result, Msg> { + if body.trim_start().starts_with(" &'static Spec { + &SPEC + } + fn start(&self, query: &Query<'_>) -> Result { + let mut q = query.text.to_owned(); + if !query.allowed.is_empty() { + q.push_str(&format!( + " ({})", + query + .allowed + .iter() + .map(|d| format!("site:{d}")) + .collect::>() + .join(" OR ") + )); + } + // Language is supplied by the browser headers. Forced cc/setlang URL + // parameters can select a regional backend that broadens the query. + let mut url = reqwest::Url::parse(URL).unwrap(); + url.query_pairs_mut().append_pair("q", &q); + Ok(Request { + url, + kind: Kind::Document, + form: None, + }) + } + fn parse(&self, _: &Request, body: &str, _: &Query<'_>) -> Result { + search(body).map(Step::Done) + } + fn validate_home(&self, body: &str) -> Result<(), Msg> { + verification(&Html::parse_document(body)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn bing_web_results_exclude_ads_and_decode_only_known_tracking_links() { + use base64::Engine; + let encoded = base64::engine::general_purpose::URL_SAFE_NO_PAD + .encode("https://docs.example.org/guide?a=1&b=2#section"); + let body = format!( + r#"
    +
  1. Advertisement

  2. +
  3. The official guide

    Use TaskGroup & await tasks.

  4. +
  5. Direct source

    Plain snippet
  6. +
  7. Broken tracker

  8. +
  9. Related search

  10. +
"# + ); + let rows = search(&body).unwrap(); + assert_eq!(rows.len(), 2); + assert_eq!(rows[0].url, "https://docs.example.org/guide?a=1&b=2"); + assert_eq!(rows[0].title, "The official guide"); + assert_eq!(rows[0].text, "Use TaskGroup & await tasks."); + assert!(rows[0].published_at.is_none()); + assert_eq!(rows[1].text, "Plain snippet"); + assert!( + search(r#"
  1. No results
"#) + .unwrap() + .is_empty() + ); + assert!(search(r#"
Verify
"#).is_err()); + assert!( + search(r#"
"#) + .unwrap_err() + .text + .contains("verification") + ); + // Merely mentioning a challenge in scripts or an article is not one. + assert!( + verification(&Html::parse_document( + r#""# + )) + .is_ok() + ); + assert!(search("
Unexpected layout
").is_err()); + } +} +#[cfg(test)] +mod entity_tests { + use super::*; + #[test] + fn rss_preserves_entities_cdata_and_query_parameters() { + let result = search( r#"Rust & Web 中文https://example.org/?a=1&b=2A & B

]]>
"#).unwrap(); + assert_eq!(result[0].title, "Rust & Web 中文"); + assert_eq!(result[0].url, "https://example.org/?a=1&b=2"); + assert_eq!(result[0].text, "A & B"); + } + + #[test] + fn search_item_dates_are_not_page_publication_dates() { + let result = search( "Old pagehttps://example.org/Mon, 16 Nov 2026 00:00:00 GMT").unwrap(); + assert!(result[0].published_at.is_none()); + } +} diff --git a/crates/tw-gateway/src/web_tools/providers/duckduckgo.rs b/crates/tw-gateway/src/web_tools/providers/duckduckgo.rs new file mode 100644 index 00000000..3214f3c8 --- /dev/null +++ b/crates/tw-gateway/src/web_tools/providers/duckduckgo.rs @@ -0,0 +1,257 @@ +//! Read the same first-party organic results used by DuckDuckGo's regular site. +//! The HTML endpoint frequently challenges even when this normal flow works. +//! Never execute returned JavaScript or consume ads / generated answer text. + +use scraper::{Html, Selector}; +use serde::Deserialize; +use std::sync::LazyLock; +use tw_api::{SearchEngine, WebSource}; +use tw_types::Msg; + +use super::super::{collect, error}; +use super::{Endpoint, Kind, Profile, Provider, Query, Request, Spec, Step}; + +static CHALLENGE: LazyLock = LazyLock::new(|| { + Selector::parse("#challenge-form, #anomaly-form, script[src*='anomaly.js']").unwrap() +}); +static PRELOAD: LazyLock = LazyLock::new(|| { + Selector::parse("script#deep_preload_script[src], link#deep_preload_link[href]").unwrap() +}); + +pub(super) fn is_data_url(url: &reqwest::Url) -> bool { + url.scheme() == "https" + && matches!( + url.host_str(), + Some("links.duckduckgo.com" | "duckduckgo.com") + ) + && url.port_or_known_default() == Some(443) + && url.path() == "/d.js" + && url.username().is_empty() + && url.password().is_none() + && url.fragment().is_none() +} + +pub(super) fn data_url(body: &str, query: &str) -> Result { + let doc = Html::parse_document(body); + if doc.select(&CHALLENGE).next().is_some() { + return Err(error("search engine requires a verification challenge")); + } + let link = doc + .select(&PRELOAD) + .next() + .and_then(|node| node.attr("src").or_else(|| node.attr("href"))) + .ok_or_else(|| error("DuckDuckGo search page layout was not recognized"))?; + let base = reqwest::Url::parse(URL).unwrap(); + let url = base + .join(link) + .map_err(|_| error("invalid search data URL"))?; + // This URL comes from untrusted HTML. Do not grant its host proxy remote DNS + // unless it is exactly a known first-party data endpoint for this query. + let queries: Vec<_> = url.query_pairs().filter(|(key, _)| key == "q").collect(); + if !is_data_url(&url) || queries.len() != 1 || queries[0].1 != query { + return Err(error("DuckDuckGo returned an unexpected search data URL")); + } + Ok(url) +} + +#[derive(Deserialize)] +struct Row { + #[serde(default)] + u: Option, + #[serde(default)] + t: Option, + #[serde(default)] + a: Option, + #[serde(default)] + n: Option, +} + +fn text(html: &str) -> String { + Html::parse_fragment(html).root_element().text().collect() +} + +pub(super) fn results(body: &str) -> Result, Msg> { + // Only the 'd' (organic) array. The 'a' array and deepPayload can contain + // advertisements and AI answers. Parse JSON as data, never evaluate code. + let data = ["DDG.pageLayout.load('d',", "DDG.pageLayout.load(\"d\","] + .iter() + .find_map(|marker| body.split_once(marker).map(|(_, tail)| tail)) + .ok_or_else(|| { + error( + if body.contains("isJsaChallenge") || body.contains("challenge-form") { + "search engine requires browser verification" + } else { + "DuckDuckGo did not return organic search results" + }, + ) + })?; + let mut stream = serde_json::Deserializer::from_str(data).into_iter::>(); + let rows = stream + .next() + .ok_or_else(|| error("DuckDuckGo search results are missing"))? + .map_err(|_| error("invalid DuckDuckGo search results"))?; + if !data[stream.byte_offset()..].trim_start().starts_with(')') { + return Err(error("invalid DuckDuckGo search result wrapper")); + } + let mut sources = Vec::new(); + let mut malformed = false; + for row in rows { + match (row.u, row.t) { + (Some(url), Some(title)) if !title.trim().is_empty() => { + if let Some(source) = collect::source( + SearchEngine::Duckduckgo, + &url, + &text(&title), + &text(&row.a.unwrap_or_default()), + ) { + sources.push(source); + } else { + malformed = true; + } + } + // A final pagination object is not a result; never follow it here. + (None, None) if row.n.is_some() => {} + _ => malformed = true, + } + } + if malformed && sources.is_empty() { + return Err(error("DuckDuckGo search result layout was not recognized")); + } + Ok(sources) +} + +const URL: &str = "https://duckduckgo.com/"; +static SPEC: Spec = Spec { + id: SearchEngine::Duckduckgo, + name: "DuckDuckGo", + profile: Profile::Basic, + min_interval_ms: 0, + home: None, + endpoints: &[ + Endpoint { + host: "duckduckgo.com", + paths: &["/", "/d.js"], + }, + Endpoint { + host: "links.duckduckgo.com", + paths: &["/d.js"], + }, + ], +}; +pub(super) struct Duckduckgo; +impl Provider for Duckduckgo { + fn spec(&self) -> &'static Spec { + &SPEC + } + fn start(&self, query: &Query<'_>) -> Result { + let mut url = reqwest::Url::parse(URL).unwrap(); + url.query_pairs_mut() + .append_pair("q", query.text) + .append_pair("ia", "web"); + if let Some(market) = super::locale::duckduckgo_market(query.locale) { + url.query_pairs_mut().append_pair("kl", market); + } + Ok(Request { + url, + kind: Kind::Document, + form: None, + }) + } + fn parse(&self, request: &Request, body: &str, query: &Query<'_>) -> Result { + match request.kind { + Kind::Document => data_url(body, query.text).map(|url| { + Step::Fetch(Request { + url, + kind: Kind::Data, + form: None, + }) + }), + Kind::Data => results(body).map(Step::Done), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn discovers_first_party_data_without_executing_scripts() { + let url = data_url( + r#""#, + "Rust OnceLock", + ).unwrap(); + assert_eq!(url.query_pairs().count(), 2); + assert!(is_data_url(&url)); + assert!( + data_url( + r#""#, + "中文" + ) + .is_ok() + ); + } + + #[test] + fn rejects_changed_queries_and_untrusted_data_destinations() { + for url in [ + "https://evil.example/d.js?q=Rust", + "https://links.duckduckgo.com.evil.example/d.js?q=Rust", + "https://127.0.0.1/d.js?q=Rust", + "http://links.duckduckgo.com/d.js?q=Rust", + "https://links.duckduckgo.com:8443/d.js?q=Rust", + "https://user@links.duckduckgo.com/d.js?q=Rust", + "https://links.duckduckgo.com/redirect?q=Rust", + "https://links.duckduckgo.com/d.js?q=Python", + "https://links.duckduckgo.com/d.js?q=Rust&q=Python", + ] { + assert!( + data_url( + &format!(r#""#), + "Rust" + ) + .is_err(), + "{url}" + ); + } + assert!( + data_url("
", "Rust") + .unwrap_err() + .text + .contains("verification") + ); + } + + #[test] + fn reads_only_organic_json_and_preserves_markup_text_and_escaped_punctuation() { + let body = r#"window.execDeep = function() { DDG.pageLayout.load('a',[{"u":"https://ads.example/","t":"Ad"}]); DDG.deep.deepPayload={"answer":"invented answer"}; DDG.pageLayout.load('d',[{"u":"https://docs.example/a?x=1&y=2&utm_source=search#top","t":"Rust & 中文","a":"Use OnceLock with \"quoted ], text\"."},{"n":"/d.js?next=1"}]); }"#; + let found = results(body).unwrap(); + assert_eq!(found.len(), 1); + assert_eq!(found[0].url, "https://docs.example/a?x=1&y=2"); + assert_eq!(found[0].title, "Rust & 中文"); + assert_eq!(found[0].text, "Use OnceLock with \"quoted ], text\"."); + assert!(found[0].published_at.is_none()); + assert!(results("DDG.pageLayout.load('d',[]);").unwrap().is_empty()); + } + + #[test] + fn malformed_data_and_challenges_are_failures_not_empty_successes() { + for body in [ + "DDG.pageLayout.load('a',[]);", + "DDG.pageLayout.load('d',[{bad}]);", + "DDG.pageLayout.load('d',[{\"new_field\":1}]);", + "DDG.pageLayout.load('d',[] + injected());", + "
", + ] { + assert!(results(body).is_err(), "{body}"); + } + assert!( + results("return {isJsaChallenge: true};") + .unwrap_err() + .text + .contains("browser verification") + ); + // A normal result may discuss a challenge; it is still a valid result. + assert!(results(r#"DDG.pageLayout.load('d',[{"u":"https://example.org/","t":"isJsaChallenge and challenge-form"}]);"#).is_ok()); + } +} diff --git a/crates/tw-gateway/src/web_tools/providers/google_docs.rs b/crates/tw-gateway/src/web_tools/providers/google_docs.rs new file mode 100644 index 00000000..170abd3c --- /dev/null +++ b/crates/tw-gateway/src/web_tools/providers/google_docs.rs @@ -0,0 +1,315 @@ +//! Google's legacy Docs research endpoint returns web results as JSON. Document +//! IDs are public request contexts, not credentials or result filters. Keep the +//! source identity distinct from the ordinary Google SERP. + +use std::sync::{LazyLock, Mutex}; +use std::time::{Duration, Instant}; + +use scraper::Html; +use serde_json::{Value, json}; +use tw_api::{SearchEngine, WebSource}; +use tw_types::Msg; + +use super::super::{collect, error}; +use super::{Endpoint, Kind, Profile, Provider, Query, Request, Spec, Step}; + +// Public Kubernetes community agendas, linked from: +// https://github.com/kubernetes/community/tree/master/sig-autoscaling +// https://github.com/kubernetes/community/tree/master/sig-architecture +// Search reads the web; it does not read or modify these documents. Eligibility +// can change independently of public readability, so failed contexts cool down. +const PATHS: &[&str] = &[ + "/document/d/1RvhQAEIrVLHbyNnuaT99-6u9ZUMp7BfkPupT2LAZK7w/explore/search", + "/document/d/10QkXwiZfBL7wBKHHFslooXvGDKHA75jkkgiIXuKFryM/explore/search", + "/document/d/1HtTI0rJEGP_MSf6eO87aCmx_tzpovPAAg7U2Zxwm8FE/explore/search", +]; +static SPEC: Spec = Spec { + id: SearchEngine::GoogleDocs, + name: "Google (Docs)", + profile: Profile::BrowserHttp1, + min_interval_ms: 6000, + home: None, + endpoints: &[Endpoint { + host: "docs.google.com", + paths: PATHS, + }], +}; + +#[derive(Default)] +struct State { + preferred: usize, + unavailable: [Option; 3], +} + +#[derive(Default)] +pub(super) struct GoogleDocs { + state: Mutex, +} + +pub(super) static GOOGLE_DOCS: LazyLock = LazyLock::new(GoogleDocs::default); + +impl GoogleDocs { + fn choose(&self, query: &Query<'_>) -> Option { + let state = self.state.lock().unwrap_or_else(|e| e.into_inner()); + let now = Instant::now(); + (0..PATHS.len()) + .map(|offset| (state.preferred + offset) % PATHS.len()) + .find(|&index| state.unavailable[index].is_none_or(|until| until <= now)) + .map(|index| request(index, query)) + } +} + +impl Provider for GoogleDocs { + fn spec(&self) -> &'static Spec { + &SPEC + } + + fn start(&self, query: &Query<'_>) -> Result { + self.choose(query) + .ok_or_else(|| error("all Google Docs search contexts are cooling down")) + } + + fn recover(&self, request: &Request, failure: &Msg, query: &Query<'_>) -> Option { + let status = failure.arg("http_status").parse::().ok()?; + // Never switch documents to retry an engine-wide throttle, server error, + // network failure, or unknown response. Those go to normal failover. + if !matches!(status, 300..=399 | 401 | 403 | 404) { + return None; + } + let index = PATHS.iter().position(|path| *path == request.url.path())?; + { + let mut state = self.state.lock().unwrap_or_else(|e| e.into_inner()); + state.unavailable[index] = Some(Instant::now() + Duration::from_secs(900)); + } + self.choose(query) + } + + fn parse(&self, request: &Request, body: &str, _: &Query<'_>) -> Result { + let sources = results(body)?; + if let Some(index) = PATHS.iter().position(|path| *path == request.url.path()) { + let mut state = self.state.lock().unwrap_or_else(|e| e.into_inner()); + state.preferred = index; + state.unavailable[index] = None; + } + Ok(Step::Done(sources)) + } +} + +fn request(index: usize, query: &Query<'_>) -> Request { + let language = match query.locale.language.as_str() { + "" => "en", + lang if lang.starts_with("zh-Hans") => "zh-CN", + lang if lang.starts_with("zh-Hant") => "zh-TW", + lang => lang, + }; + // Twenty is the UI's maximum. One page avoids multiplying requests when + // domain/relevance filtering removes some entries. Region is not supported + // by this legacy protocol; do not silently add a country restriction. + let payload = json!([ + "documentsuggest.search.search_request", + query.text, + [20, "CAA="], + [language, language], + 1 + ]); + Request { + url: format!("https://docs.google.com{}", PATHS[index]) + .parse() + .unwrap(), + kind: Kind::Data, + form: Some( + url::form_urlencoded::Serializer::new(String::new()) + .append_pair("request", &payload.to_string()) + .finish(), + ), + } +} + +fn text(value: &str) -> String { + Html::parse_fragment(value) + .root_element() + .text() + .collect::() +} + +fn results(body: &str) -> Result, Msg> { + let invalid = || error("Google Docs returned an unrecognized search response"); + let body = body.trim_start(); + let body = if body.starts_with(")]}'") { + body.split_once('\n').ok_or_else(invalid)?.1 + } else { + body + }; + let root: Value = serde_json::from_str(body).map_err(|_| invalid())?; + let message = root.get(0).and_then(Value::as_array).ok_or_else(invalid)?; + if message.first().and_then(Value::as_str) != Some("documentsuggest.search.search_response") { + return Err(invalid()); + } + let records = match message.get(1) { + None | Some(Value::Null) => return Ok(Vec::new()), + Some(Value::Array(records)) => records, + _ => return Err(invalid()), + }; + let mut sources = Vec::new(); + for record in records { + let row = record.as_array().ok_or_else(invalid)?; + let organic = match row.first() { + Some(Value::Null) => continue, // Knowledge card, not an organic result. + Some(Value::Array(organic)) => organic, + _ => return Err(invalid()), + }; + let url = organic + .first() + .and_then(Value::as_str) + .ok_or_else(invalid)?; + let title = organic.get(1).and_then(Value::as_str).ok_or_else(invalid)?; + let snippet = organic.get(2).and_then(Value::as_str).ok_or_else(invalid)?; + let title = text(title); + if title.trim().is_empty() { + continue; + } + if let Some(source) = collect::source(SearchEngine::GoogleDocs, url, &title, &text(snippet)) + && !sources.iter().any(|s: &WebSource| s.url == source.url) + { + sources.push(source); + } + } + Ok(sources) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_fixed_google_contexts_are_allowed() { + for path in PATHS { + assert!(SPEC.allows(&format!("https://docs.google.com{path}").parse().unwrap())); + } + for url in [ + "https://docs.google.com/document/d/arbitrary/explore/search", + "https://docs.google.com/document/d/private/edit", + "https://accounts.google.com/ServiceLogin", + "https://127.0.0.1/", + "http://docs.google.com/", + ] { + assert!(!SPEC.allows(&url.parse().unwrap()), "{url}"); + } + } + + #[test] + fn parses_organic_json_without_turning_invalid_responses_into_empty_success() { + let body = json!([[ + "documentsuggest.search.search_response", + [ + [ + [ + "https://docs.example.org/task?x=1&utm_source=google#part", + "TaskGroup & API", + "Use TaskGroup & await." + ], + null + ], + [null, "knowledge card"], + [[ + "https://docs.example.org/task?x=1#other", + "duplicate", + "same URL" + ]], + [["javascript:alert(1)", "invalid URL", "ignore"]] + ], + "CAo=" + ]]); + let found = results(&format!(")]}}'\n{body}")).unwrap(); + assert_eq!(found.len(), 1); + assert_eq!(found[0].title, "TaskGroup & API"); + assert_eq!(found[0].text, "Use TaskGroup & await."); + assert_eq!(found[0].url, "https://docs.example.org/task?x=1"); + assert_eq!(found[0].engine, Some(SearchEngine::GoogleDocs)); + for empty in [ + r#"[["documentsuggest.search.search_response", null]]"#, + r#"[["documentsuggest.search.search_response", []]]"#, + ] { + assert!(results(empty).unwrap().is_empty()); + } + for invalid in [ + "Sign in", + "[]", + "{}", + ")]}'", + r#"[["error", null]]"#, + r#"[["documentsuggest.search.search_response", {}]]"#, + r#"[["documentsuggest.search.search_response", [[]]]]"#, + r#"[["documentsuggest.search.search_response", [[[123,"title","text"]]]]]"#, + ] { + assert!(results(invalid).is_err(), "{invalid}"); + } + } + + #[tokio::test] + async fn context_failures_are_bounded_and_remembered_but_throttles_never_rotate() { + let provider = GoogleDocs::default(); + let locale = tw_api::WebSearchLocale { + language: "zh-Hans".into(), + region: "CN".into(), + }; + let query = Query { + text: "Rust & 中文 + ?", + allowed: &[], + locale: &locale, + }; + let first = provider.start(&query).unwrap(); + let fields: Vec<_> = + url::form_urlencoded::parse(first.form.as_ref().unwrap().as_bytes()).collect(); + let body: Value = serde_json::from_str(&fields[0].1).unwrap(); + assert_eq!(body[1], query.text); + assert_eq!(body[3], json!(["zh-CN", "zh-CN"])); + assert!( + first.url.query().is_none(), + "query belongs only in the POST body" + ); + let mut calls = Vec::new(); + super::super::run(&provider, &query, |request| { + calls.push(request.url.path().to_string()); + std::future::ready(if calls.len() == 1 { + Err(super::super::super::http_failure( + reqwest::StatusCode::FORBIDDEN, + &Default::default(), + )) + } else { + Ok(r#"[["documentsuggest.search.search_response", []]]"#.into()) + }) + }) + .await + .unwrap(); + assert_eq!(calls, [PATHS[0], PATHS[1]]); + assert_eq!(provider.start(&query).unwrap().url.path(), PATHS[1]); + for status in [429, 500, 503] { + let mut calls = 0; + let result = super::super::run(&provider, &query, |_| { + calls += 1; + std::future::ready(Err(super::super::super::http_failure( + reqwest::StatusCode::from_u16(status).unwrap(), + &Default::default(), + ))) + }) + .await; + assert!(result.is_err()); + assert_eq!(calls, 1); + } + let mut calls = 0; + assert!( + super::super::run(&provider, &query, |_| { + calls += 1; + std::future::ready(Err(super::super::super::http_failure( + reqwest::StatusCode::NOT_FOUND, + &Default::default(), + ))) + }) + .await + .is_err() + ); + assert_eq!(calls, 2, "first document is already cooling down"); + assert!(provider.start(&query).is_err()); + } +} diff --git a/crates/tw-gateway/src/web_tools/locale.rs b/crates/tw-gateway/src/web_tools/providers/locale.rs similarity index 62% rename from crates/tw-gateway/src/web_tools/locale.rs rename to crates/tw-gateway/src/web_tools/providers/locale.rs index 41771ba5..a186a8ec 100644 --- a/crates/tw-gateway/src/web_tools/locale.rs +++ b/crates/tw-gateway/src/web_tools/providers/locale.rs @@ -1,23 +1,4 @@ -use tw_api::{SearchEngine, WebSearchLocale}; - -pub(super) fn apply(url: &mut reqwest::Url, engine: SearchEngine, locale: &WebSearchLocale) { - let mut params = url.query_pairs_mut(); - match engine { - SearchEngine::Bing => { - if !locale.language.is_empty() { - params.append_pair("setlang", &locale.language); - } - if !locale.region.is_empty() { - params.append_pair("cc", &locale.region); - } - } - SearchEngine::Duckduckgo => { - if let Some(market) = duckduckgo_market(locale) { - params.append_pair("kl", market); - } - } - } -} +use tw_api::WebSearchLocale; // DuckDuckGo has a fixed set of regional markets, not arbitrary BCP 47 pairs. // https://duckduckgo.com/duckduckgo-help-pages/settings/params @@ -25,7 +6,7 @@ pub(super) fn apply(url: &mut reqwest::Url, engine: SearchEngine, locale: &WebSe // supported default. Unsupported regions keep the engine's default. const DDG_MARKETS: &str = "ar-es au-en at-de be-fr be-nl br-pt bg-bg ca-en ca-fr ct-ca cl-es cn-zh co-es hr-hr cz-cs dk-da ee-et fi-fi fr-fr de-de gr-el hk-tzh hu-hu is-is in-en id-en ie-en il-he it-it jp-jp kr-kr lv-lv lt-lt my-en mx-es nl-nl nz-en no-no pk-en pe-es ph-en pl-pl pt-pt ro-ro ru-ru sa-ar sg-en sk-sk sl-sl za-en es-es se-sv ch-de ch-fr ch-it tw-tzh th-en tr-tr us-en us-es ua-uk uk-en vn-en"; -fn duckduckgo_market(locale: &WebSearchLocale) -> Option<&'static str> { +pub(super) fn duckduckgo_market(locale: &WebSearchLocale) -> Option<&'static str> { let region = match locale.region.to_ascii_lowercase().as_str() { "gb" => "uk".into(), "si" => "sl".into(), @@ -54,7 +35,7 @@ mod tests { use super::*; #[test] - fn engine_parameters_use_supported_markets_without_duplicating_bcp47_regions() { + fn duckduckgo_markets_match_language_and_region() { for (language, region, market) in [ ("en-US", "US", Some("us-en")), ("zh-Hant", "TW", Some("tw-tzh")), @@ -73,18 +54,6 @@ mod tests { region: region.into(), }; assert_eq!(duckduckgo_market(&locale), market); - let mut url = reqwest::Url::parse("https://www.bing.com/search?q=test").unwrap(); - apply(&mut url, SearchEngine::Bing, &locale); - assert!( - url.query_pairs() - .any(|(k, v)| k == "setlang" && v == language) - ); - assert_eq!( - url.query_pairs() - .find(|(k, _)| k == "cc") - .map(|(_, v)| v.into_owned()), - (!region.is_empty()).then(|| region.to_string()) - ); } } } diff --git a/crates/tw-gateway/src/web_tools/providers/mod.rs b/crates/tw-gateway/src/web_tools/providers/mod.rs new file mode 100644 index 00000000..e1fff086 --- /dev/null +++ b/crates/tw-gateway/src/web_tools/providers/mod.rs @@ -0,0 +1,286 @@ +//! Source adapters describe requests and parse responses; only the executor does +//! I/O. New sources share the same URL checks, budgets, sessions, and failover. + +mod bing; +mod duckduckgo; +mod google_docs; +mod locale; + +use reqwest::Url; +use tw_api::{SearchEngine, WebSearchLocale, WebSource}; +use tw_types::Msg; + +pub(super) const AUTOMATIC: &[SearchEngine] = &[ + SearchEngine::GoogleDocs, + SearchEngine::Duckduckgo, + SearchEngine::Bing, +]; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Profile { + /// The existing uncompressed HTTP/1 tunnel, also used for page reads. + Basic, + /// Browser navigation headers and decoded, pooled HTTP/1 connections. + BrowserHttp1, +} + +pub(super) struct Endpoint { + pub host: &'static str, + pub paths: &'static [&'static str], +} + +pub(super) struct Spec { + pub id: SearchEngine, + pub name: &'static str, + pub profile: Profile, + pub min_interval_ms: u32, + pub home: Option<&'static str>, + pub endpoints: &'static [Endpoint], +} + +impl Spec { + pub fn allows(&self, url: &Url) -> bool { + url.scheme() == "https" + && url.port_or_known_default() == Some(443) + && url.username().is_empty() + && url.password().is_none() + && url.fragment().is_none() + && self + .endpoints + .iter() + .any(|e| url.host_str() == Some(e.host) && e.paths.contains(&url.path())) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Kind { + Document, + Data, +} + +#[derive(Clone)] +pub(super) struct Request { + pub url: Url, + pub kind: Kind, + /// Encoded form body selects POST. None preserves the existing GET flow. + pub form: Option, +} + +pub(super) struct Query<'a> { + pub text: &'a str, + pub allowed: &'a [String], + pub locale: &'a WebSearchLocale, +} + +pub(super) enum Step { + Fetch(Request), + Done(Vec), +} + +pub(super) trait Provider: Sync { + fn spec(&self) -> &'static Spec; + fn start(&self, query: &Query<'_>) -> Result; + fn recover(&self, _request: &Request, _failure: &Msg, _query: &Query<'_>) -> Option { + None + } + fn parse(&self, request: &Request, body: &str, query: &Query<'_>) -> Result; + fn validate_home(&self, _body: &str) -> Result<(), Msg> { + Ok(()) + } +} + +pub(super) fn get(engine: SearchEngine) -> &'static dyn Provider { + match engine { + SearchEngine::GoogleDocs => &*google_docs::GOOGLE_DOCS, + SearchEngine::Bing => &bing::Bing, + SearchEngine::Duckduckgo => &duckduckgo::Duckduckgo, + } +} + +/// A bounded multi-step exchange shared by all adapters. No adapter can fetch a +/// discovered URL outside its declared first-party endpoints, even in tests. +pub(super) async fn run( + provider: &dyn Provider, + query: &Query<'_>, + mut fetch: F, +) -> Result, Msg> +where + F: FnMut(Request) -> Fut, + Fut: std::future::Future>, +{ + let mut request = provider.start(query)?; + for _ in 0..4 { + if !provider.spec().allows(&request.url) { + return Err(super::error( + "search requested an unsupported first-party endpoint", + )); + } + let body = match fetch(request.clone()).await { + Ok(body) => body, + Err(failure) => match provider.recover(&request, &failure, query) { + Some(next) => { + request = next; + continue; + } + None => return Err(failure), + }, + }; + match provider.parse(&request, &body, query)? { + Step::Fetch(next) => request = next, + Step::Done(sources) => return Ok(sources), + } + } + Err(super::error("search exceeded its request-step limit")) +} + +#[cfg(test)] +mod tests { + use super::*; + + struct AdditionalSource; + static ADDITIONAL: Spec = Spec { + id: SearchEngine::Bing, + name: "Test adapter", + profile: Profile::Basic, + min_interval_ms: 0, + home: None, + endpoints: &[Endpoint { + host: "source.test", + paths: &["/search", "/data"], + }], + }; + impl Provider for AdditionalSource { + fn spec(&self) -> &'static Spec { + &ADDITIONAL + } + fn start(&self, _: &Query<'_>) -> Result { + Ok(Request { + url: "https://source.test/search".parse().unwrap(), + kind: Kind::Document, + form: None, + }) + } + fn parse(&self, _: &Request, body: &str, _: &Query<'_>) -> Result { + if body == "done" { + Ok(Step::Done(vec![])) + } else { + Ok(Step::Fetch(Request { + url: body.parse().unwrap(), + kind: Kind::Data, + form: None, + })) + } + } + } + + #[tokio::test] + async fn an_additional_adapter_runs_without_engine_branches_and_cannot_escape_or_loop() { + let locale = WebSearchLocale::default(); + let query = Query { + text: "sample", + allowed: &[], + locale: &locale, + }; + let mut calls = Vec::new(); + let found = run(&AdditionalSource, &query, |request| { + calls.push(request.kind); + std::future::ready(Ok(if request.kind == Kind::Document { + "https://source.test/data".into() + } else { + "done".into() + })) + }) + .await + .unwrap(); + assert!(found.is_empty()); + assert_eq!(calls, [Kind::Document, Kind::Data]); + + let mut calls = 0; + let result = run(&AdditionalSource, &query, |_| { + calls += 1; + std::future::ready(Ok("https://127.0.0.1/data".into())) + }) + .await; + assert!(result.unwrap_err().text.contains("unsupported")); + assert_eq!( + calls, 1, + "untrusted discovered URLs must fail before transport" + ); + + let mut calls = 0; + let result = run(&AdditionalSource, &query, |_| { + calls += 1; + std::future::ready(Ok(format!("https://source.test/data?page={calls}"))) + }) + .await; + assert!(result.unwrap_err().text.contains("request-step limit")); + assert_eq!(calls, 4); + } + + #[test] + fn adapters_keep_their_query_and_locale_semantics() { + let locale = WebSearchLocale { + language: "zh-Hans".into(), + region: "CN".into(), + }; + let allowed = vec!["sqlite.org".into()]; + let query = Query { + text: "SQLite release notes", + allowed: &allowed, + locale: &locale, + }; + for &engine in AUTOMATIC { + let provider = get(engine); + assert_eq!(provider.spec().id, engine); + let request = provider.start(&query).unwrap(); + assert!(provider.spec().allows(&request.url)); + if let Some(home) = provider.spec().home { + assert!(provider.spec().allows(&home.parse().unwrap())); + } + } + let bing = get(SearchEngine::Bing).start(&query).unwrap().url; + assert_eq!( + bing.query_pairs().find(|(k, _)| k == "q").unwrap().1, + "SQLite release notes (site:sqlite.org)" + ); + assert_eq!(bing.query_pairs().count(), 1); + let ddg = get(SearchEngine::Duckduckgo).start(&query).unwrap().url; + assert_eq!( + ddg.query_pairs().find(|(k, _)| k == "q").unwrap().1, + query.text + ); + assert_eq!( + ddg.query_pairs().find(|(k, _)| k == "kl").unwrap().1, + "cn-zh" + ); + } + #[test] + fn remote_dns_exception_only_covers_fixed_https_search_endpoints() { + assert!( + get(SearchEngine::Bing).spec().allows( + &super::super::public::parse("https://www.bing.com/search?q=Rust").unwrap() + ) + ); + assert!( + get(SearchEngine::Duckduckgo) + .spec() + .allows(&super::super::public::parse("https://duckduckgo.com/?q=Rust").unwrap()) + ); + for value in [ + "http://www.bing.com/search?q=Rust", + "https://www.bing.com/redirect", + "https://www.bing.com.example.org/search", + "https://example.org/search", + "https://127.0.0.1/search", + "https://www.bing.com:8443/search", + "https://user:password@www.bing.com/search", + ] { + assert!( + !get(SearchEngine::Bing) + .spec() + .allows(&reqwest::Url::parse(value).unwrap()), + "{value}" + ); + } + } +} diff --git a/crates/tw-gateway/src/web_tools/public.rs b/crates/tw-gateway/src/web_tools/public.rs index ca759e34..3e2706d3 100644 --- a/crates/tw-gateway/src/web_tools/public.rs +++ b/crates/tw-gateway/src/web_tools/public.rs @@ -1,22 +1,42 @@ use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; +use super::providers::{Profile, Spec}; +use std::sync::Mutex; +use std::time::{Duration, Instant}; use tw_types::Msg; use super::error; -/// Pin the destination through direct connections AND proxy tunnels. Letting an -/// HTTP CONNECT/SOCKS proxy resolve the original hostname would reopen the DNS -/// rebinding window even though the local preflight check passed. +/// Pin user-selected destinations through direct connections AND proxy tunnels. +/// The fixed HTTPS search endpoints separately support the proxy's remote DNS; +/// that exception never applies to result pages or arbitrary redirect targets. pub(super) struct ClientSettings { proxy: hyper_util::client::proxy::matcher::Matcher, timeout: std::time::Duration, + search: reqwest::Client, + direct: Mutex>, +} + +// A pooled direct connection is reusable only for the same origin, profile, +// and freshly checked DNS addresses. A DNS change cannot reuse an older pin. +struct DirectClient { + origin: String, + addresses: Vec, + profile: Profile, + used: Instant, + client: reqwest::Client, } impl ClientSettings { pub fn new(config: &tw_config::Config) -> Result { use hyper_util::client::proxy::matcher::Matcher; + let timeout = std::time::Duration::from_secs(u64::from(config.web_tools.timeout_secs)); + let mut search = search_builder().timeout(timeout); let proxy = match config.web_tools.proxy.as_str() { - tw_config::DIRECT => Matcher::builder().build(), + tw_config::DIRECT => { + search = search.no_proxy(); + Matcher::builder().build() + } tw_config::SYSTEM => Matcher::from_system(), name => { let url = config @@ -32,41 +52,152 @@ impl ClientSettings { "search proxy credential could not be read", )) })?; + search = search.proxy( + reqwest::Proxy::all(&url) + .map_err(|_| crate::GatewayError::config(error("invalid search proxy")))?, + ); Matcher::builder().all(url).build() } }; Ok(Self { proxy, - timeout: std::time::Duration::from_secs(u64::from(config.web_tools.timeout_secs)), + timeout, + direct: Mutex::new(Vec::new()), + search: search.build().map_err(|_| { + crate::GatewayError::config(error("could not create search client")) + })?, }) } - pub async fn get( + /// Only the built-in HTTPS search endpoint may use the proxy's DNS. Its host + /// is chosen by Core, never by tool input. Arbitrary pages and redirects still + /// resolve locally, reject private addresses, and pin the checked IP below. + /// Otherwise a working remote-DNS proxy inherits poisoned local DNS answers. + pub async fn get_search( &self, + spec: &Spec, url: &reqwest::Url, - addresses: &[SocketAddr], + headers: http::HeaderMap, + form: Option<&str>, ) -> Result { + parse(url.as_str())?; + if !spec.allows(url) { + return Err(error( + "search redirected outside its supported first-party pages", + )); + } let uri: http::Uri = url .as_str() .parse() .map_err(|_| error("invalid source URI"))?; - if let Some(proxy) = self.proxy.intercept(&uri) { - return tokio::time::timeout(self.timeout, self.proxied(url, addresses, proxy)) + if let Some(proxy) = self.proxy.intercept(&uri) + // SOCKS5 explicitly selects local DNS; SOCKS5h and CONNECT use remote DNS. + && proxy.uri().scheme_str() != Some("socks5") + { + if spec.profile == Profile::Basic { + return tokio::time::timeout( + self.timeout, + self.proxied(url, None, proxy, headers, form), + ) .await .map_err(|_| error("proxy connection timed out"))?; + } + // Reuse the first-party browser session connection. Result pages + // still use checked and pinned IPs. + return request(&self.search, url, form) + .headers(headers) + .send() + .await + .map_err(|e| error(e.without_url().to_string())); + } + let addresses = check(url).await?; + self.get_with_headers(url, &addresses, headers, spec.profile, form) + .await + } + + pub async fn get( + &self, + url: &reqwest::Url, + addresses: &[SocketAddr], + ) -> Result { + self.get_with_headers(url, addresses, request_headers(), Profile::Basic, None) + .await + } + + fn direct_client( + &self, + url: &reqwest::Url, + addresses: &[SocketAddr], + profile: Profile, + ) -> Result { + let origin = url.origin().ascii_serialization(); + let mut addresses = addresses.to_vec(); + addresses.sort_unstable(); + addresses.dedup(); + let mut clients = self.direct.lock().unwrap_or_else(|e| e.into_inner()); + clients.retain(|entry| entry.used.elapsed() < Duration::from_secs(90)); + if let Some(entry) = clients.iter_mut().find(|entry| { + entry.origin == origin && entry.addresses == addresses && entry.profile == profile + }) { + entry.used = Instant::now(); + return Ok(entry.client.clone()); } - let client = crate::outbound::base_client_builder() + let builder = if profile == Profile::BrowserHttp1 { + search_builder() + } else { + crate::outbound::base_client_builder() + }; + let client = builder .no_proxy() .resolve_to_addrs( url.host_str().ok_or_else(|| error("URL has no host"))?, - addresses, + &addresses, ) .timeout(self.timeout) .build() .map_err(|_| error("could not create web client"))?; - client - .get(url.clone()) - .headers(request_headers()) + if clients.len() >= 32 { + let oldest = clients + .iter() + .enumerate() + .min_by_key(|(_, entry)| entry.used) + .map(|(i, _)| i) + .unwrap(); + clients.swap_remove(oldest); + } + clients.push(DirectClient { + origin, + addresses, + profile, + used: Instant::now(), + client: client.clone(), + }); + Ok(client) + } + + async fn get_with_headers( + &self, + url: &reqwest::Url, + addresses: &[SocketAddr], + headers: http::HeaderMap, + profile: Profile, + form: Option<&str>, + ) -> Result { + let uri: http::Uri = url + .as_str() + .parse() + .map_err(|_| error("invalid source URI"))?; + if let Some(proxy) = self.proxy.intercept(&uri) { + return tokio::time::timeout( + self.timeout, + self.proxied(url, Some(addresses), proxy, headers, form), + ) + .await + .map_err(|_| error("proxy connection timed out"))?; + } + let client = self.direct_client(url, addresses, profile)?; + request(&client, url, form) + .headers(headers) .send() .await .map_err(|e| error(e.without_url().to_string())) @@ -75,8 +206,10 @@ impl ClientSettings { async fn proxied( &self, url: &reqwest::Url, - addresses: &[SocketAddr], + addresses: Option<&[SocketAddr]>, proxy: hyper_util::client::proxy::matcher::Intercept, + mut headers: http::HeaderMap, + form: Option<&str>, ) -> Result { use http_body_util::BodyExt; use tw_config::ProxyKind; @@ -98,11 +231,21 @@ impl ClientSettings { .raw_auth() .map(|(user, password)| (user.into(), password.into())), }; - let mut addresses = addresses.to_vec(); - addresses.sort_by_key(SocketAddr::is_ipv6); + let targets = match addresses { + Some(addresses) => { + let mut addresses = addresses.to_vec(); + addresses.sort_by_key(SocketAddr::is_ipv6); + addresses.into_iter().take(4).map(Some).collect::>() + } + None => vec![None], + }; let mut last = error("source has no public address"); let mut connected = None; - for target in addresses.into_iter().take(4) { + for target in targets { + let target_host = target + .map(|a| a.ip().to_string()) + .unwrap_or_else(|| url.host_str().expect("validated search URL").to_string()); + let target_port = target.map(|a| a.port()).unwrap_or(443); let connect = async { let tcp = tokio::net::TcpStream::connect((host.trim_matches(['[', ']']), port)) .await @@ -113,24 +256,10 @@ impl ClientSettings { } match kind { ProxyKind::Socks5 | ProxyKind::Socks5h => { - crate::l1::socks5_connect( - &mut io, - &hop, - &target.ip().to_string(), - target.port(), - Some(target), - ) - .await? - } - _ => { - crate::l1::http_connect( - &mut io, - &hop, - &target.ip().to_string(), - target.port(), - ) - .await? + crate::l1::socks5_connect(&mut io, &hop, &target_host, target_port, target) + .await? } + _ => crate::l1::http_connect(&mut io, &hop, &target_host, target_port).await?, } if url.scheme() == "https" { io = crate::l1::tls_handshake_boxed( @@ -158,16 +287,31 @@ impl ClientSettings { let connection = Connection(tokio::spawn(async move { let _ = connection.await; })); + if form.is_some() { + headers.insert( + "content-type", + http::HeaderValue::from_static("application/x-www-form-urlencoded"), + ); + } let mut request = http::Request::builder() + .method(if form.is_some() { "POST" } else { "GET" }) .uri(format!( "{}{}", url.path(), url.query().map(|q| format!("?{q}")).unwrap_or_default() )) .header("host", url.authority()) - .body(http_body_util::Empty::::new()) + .body(http_body_util::Full::new(bytes::Bytes::copy_from_slice( + form.unwrap_or_default().as_bytes(), + ))) .map_err(|_| error("invalid web request"))?; - request.headers_mut().extend(request_headers()); + // This manually pinned HTTP/1 tunnel does not run reqwest's decoder. + // Do not advertise compression that this transport cannot decode. + headers.insert( + "accept-encoding", + http::HeaderValue::from_static("identity"), + ); + request.headers_mut().extend(headers); let response = sender .send_request(request) .await @@ -184,6 +328,31 @@ impl ClientSettings { } } +fn request( + client: &reqwest::Client, + url: &reqwest::Url, + form: Option<&str>, +) -> reqwest::RequestBuilder { + match form { + Some(body) => client + .post(url.clone()) + .header("content-type", "application/x-www-form-urlencoded") + .body(body.to_owned()), + None => client.get(url.clone()), + } +} + +fn search_builder() -> reqwest::ClientBuilder { + crate::outbound::base_client_builder() + // Match the HTTP/1 navigation flow on both direct and tunneled paths. + // Some search frontends return a different page when ALPN selects h2. + .http1_only() + .gzip(true) + .brotli(true) + .deflate(true) + .zstd(true) +} + struct Connection(tokio::task::JoinHandle<()>); impl Drop for Connection { fn drop(&mut self) { @@ -191,7 +360,7 @@ impl Drop for Connection { } } -fn request_headers() -> http::HeaderMap { +pub(super) fn request_headers() -> http::HeaderMap { let mut headers = http::HeaderMap::new(); for (name, value) in [ ( @@ -360,7 +529,222 @@ mod tests { #[cfg(test)] mod network_tests { + use super::super::providers; use super::*; + use tw_api::SearchEngine; + + #[tokio::test] + async fn checked_direct_connections_are_reused_but_a_changed_dns_pin_is_not() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let v4 = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let first = v4.local_addr().unwrap(); + let v6 = tokio::net::TcpListener::bind((Ipv6Addr::LOCALHOST, first.port())) + .await + .unwrap(); + let second = v6.local_addr().unwrap(); + let server = |listener: tokio::net::TcpListener, body: &'static str| { + tokio::spawn(async move { + // A second request must arrive on this same socket. If the client is + // rebuilt for each GET, the next accept is never served and times out. + let (mut socket, _) = listener.accept().await.unwrap(); + loop { + let mut head = Vec::new(); + while !head.ends_with(b"\r\n\r\n") { + let Ok(byte) = socket.read_u8().await else { + return; + }; + head.push(byte); + } + socket + .write_all( + format!("HTTP/1.1 200 OK\r\nContent-Length: 3\r\n\r\n{body}") + .as_bytes(), + ) + .await + .unwrap(); + } + }) + }; + let old = server(v4, "old"); + let new = server(v6, "new"); + let mut config = tw_config::Config::default(); + config.web_tools.proxy = tw_config::DIRECT.into(); + let settings = ClientSettings::new(&config).unwrap(); + // Call the pinned transport directly with test loopbacks. Production + // callers must pass public::check before reaching this layer. + let url: reqwest::Url = format!("http://pool.test:{}/first", first.port()) + .parse() + .unwrap(); + tokio::time::timeout(Duration::from_secs(5), async { + for address in [first, first, second] { + let body = settings + .get(&url, &[address]) + .await + .unwrap() + .text() + .await + .unwrap(); + assert_eq!(body, if address == first { "old" } else { "new" }); + } + }) + .await + .unwrap(); + assert_eq!(settings.direct.lock().unwrap().len(), 2); + settings + .direct_client(&url, &[second], Profile::BrowserHttp1) + .unwrap(); + assert_eq!( + settings.direct.lock().unwrap().len(), + 3, + "profiles have distinct pools" + ); + for i in 0..40 { + let url = format!("http://host-{i}.test/").parse().unwrap(); + settings + .direct_client(&url, &[first], Profile::Basic) + .unwrap(); + } + assert_eq!(settings.direct.lock().unwrap().len(), 32); + old.abort(); + new.abort(); + } + + #[tokio::test] + async fn only_search_transport_decodes_browser_compression() { + use std::io::Write; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let plain = b"compressed search result"; + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(plain).unwrap(); + let encoded = encoder.finish().unwrap(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let wire = encoded.clone(); + let server = tokio::spawn(async move { + for search in [true, false] { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut head = Vec::new(); + while !head.ends_with(b"\r\n\r\n") { + head.push(socket.read_u8().await.unwrap()); + } + let head = String::from_utf8(head).unwrap().to_ascii_lowercase(); + assert_eq!(head.contains("accept-encoding:"), search); + socket.write_all(format!("HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Encoding: gzip\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", wire.len()).as_bytes()).await.unwrap(); + socket.write_all(&wire).await.unwrap(); + } + }); + let url = format!("http://{address}/"); + let decoded = search_builder() + .no_proxy() + .build() + .unwrap() + .get(&url) + .send() + .await + .unwrap() + .bytes() + .await + .unwrap(); + assert_eq!(decoded.as_ref(), plain); + let untouched = crate::outbound::base_client_builder() + .no_proxy() + .build() + .unwrap() + .get(&url) + .send() + .await + .unwrap() + .bytes() + .await + .unwrap(); + assert_eq!(untouched.as_ref(), encoded); + server.await.unwrap(); + } + + #[tokio::test] + async fn search_uses_remote_dns_through_http_and_socks5h_proxies() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + for (scheme, engine) in [ + ("http", SearchEngine::Duckduckgo), + ("socks5h", SearchEngine::Duckduckgo), + ("http", SearchEngine::Bing), + ("socks5h", SearchEngine::Bing), + ] { + let provider = providers::get(engine); + let locale = tw_api::WebSearchLocale::default(); + let url = provider + .start(&providers::Query { + text: "test", + allowed: &[], + locale: &locale, + }) + .unwrap() + .url; + let hostname = url.host_str().unwrap().to_owned(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + if scheme == "http" { + let mut head = Vec::new(); + while !head.ends_with(b"\r\n\r\n") { + head.push(socket.read_u8().await.unwrap()); + } + assert!( + String::from_utf8(head) + .unwrap() + .starts_with(&format!("CONNECT {hostname}:443 HTTP/1.1")) + ); + socket + .write_all(b"HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\n\r\n") + .await + .unwrap(); + } else { + let mut hello = [0u8; 3]; + socket.read_exact(&mut hello).await.unwrap(); + assert_eq!(hello, [5, 1, 0]); + socket.write_all(&[5, 0]).await.unwrap(); + let mut head = [0u8; 5]; + socket.read_exact(&mut head).await.unwrap(); + assert_eq!(&head[..4], &[5, 1, 0, 3], "proxy must receive a hostname"); + let mut name = vec![0u8; head[4] as usize]; + socket.read_exact(&mut name).await.unwrap(); + assert_eq!(name, hostname.as_bytes()); + assert_eq!(socket.read_u16().await.unwrap(), 443); + socket + .write_all(&[5, 2, 0, 1, 0, 0, 0, 0, 0, 0]) + .await + .unwrap(); + } + }); + let settings = ClientSettings { + direct: Mutex::new(Vec::new()), + proxy: hyper_util::client::proxy::matcher::Matcher::builder() + .all(format!("{scheme}://127.0.0.1:{port}")) + .build(), + timeout: std::time::Duration::from_secs(3), + search: search_builder() + .no_proxy() + .proxy(reqwest::Proxy::all(format!("{scheme}://127.0.0.1:{port}")).unwrap()) + .build() + .unwrap(), + }; + assert!( + settings + .get_search(provider.spec(), &url, request_headers(), None) + .await + .is_err() + ); + server.await.unwrap(); + // A redirect or page read cannot inherit the search endpoint exception. + let local = parse("https://127.0.0.1/search").unwrap(); + let failure = settings + .get_search(provider.spec(), &local, request_headers(), None) + .await + .unwrap_err(); + assert!(failure.text.contains("supported first-party"), "{failure}"); + } + } #[tokio::test] async fn local_targets_are_rejected_before_a_connection_is_opened() { for url in [ @@ -411,12 +795,14 @@ mod network_tests { .unwrap(); }); let settings = ClientSettings { + direct: Mutex::new(Vec::new()), proxy: hyper_util::client::proxy::matcher::Matcher::builder() // Match the IPv4 listener exactly: resolving localhost may try // ::1 first, whose refusal can consume the Windows CI timeout. .all(format!("http://127.0.0.1:{port}")) .build(), timeout: std::time::Duration::from_secs(10), + search: search_builder().no_proxy().build().unwrap(), }; let url = parse("http://example.org/").unwrap(); assert_eq!( @@ -431,4 +817,77 @@ mod network_tests { ); server.await.unwrap(); } + #[tokio::test] + async fn form_queries_cross_direct_and_pinned_proxy_transports_without_redirect_replay() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + for proxy in [false, true] { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let form = "request=%5B%22Rust+%26+中文%22%5D"; + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + if proxy { + let mut head = Vec::new(); + while !head.ends_with(b"\r\n\r\n") { + head.push(socket.read_u8().await.unwrap()); + } + assert!( + String::from_utf8(head) + .unwrap() + .starts_with("CONNECT 1.1.1.1:80 HTTP/1.1") + ); + socket + .write_all(b"HTTP/1.1 200 Connection established\r\n\r\n") + .await + .unwrap(); + } + let mut head = Vec::new(); + while !head.ends_with(b"\r\n\r\n") { + head.push(socket.read_u8().await.unwrap()); + } + let head = String::from_utf8(head).unwrap().to_lowercase(); + assert!(head.starts_with("post /search http/1.1"), "{head}"); + assert!(head.contains("host: origin.test")); + assert!(head.contains("content-type: application/x-www-form-urlencoded")); + assert!(head.contains(&format!("content-length: {}", form.len()))); + let mut body = vec![0; form.len()]; + socket.read_exact(&mut body).await.unwrap(); + assert_eq!(body, form.as_bytes()); + socket.write_all(b"HTTP/1.1 302 Found\r\nLocation: http://127.0.0.1/private\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").await.unwrap(); + }); + let mut config = tw_config::Config::default(); + config.web_tools.proxy = tw_config::DIRECT.into(); + let mut settings = ClientSettings::new(&config).unwrap(); + let (url, pin) = if proxy { + settings.proxy = hyper_util::client::proxy::matcher::Matcher::builder() + .all(format!("http://{address}")) + .build(); + ( + "http://origin.test/search".parse().unwrap(), + "1.1.1.1:80".parse().unwrap(), + ) + } else { + ( + format!("http://origin.test:{}/search", address.port()) + .parse() + .unwrap(), + address, + ) + }; + // Loopback pins are injected only in this transport test. Normal + // execution validates public addresses before reaching this layer. + let response = settings + .get_with_headers( + &url, + &[pin], + request_headers(), + Profile::BrowserHttp1, + Some(form), + ) + .await + .unwrap(); + assert_eq!(response.status(), reqwest::StatusCode::FOUND); + server.await.unwrap(); + } + } } diff --git a/crates/tw-gateway/src/web_tools/quality.rs b/crates/tw-gateway/src/web_tools/quality.rs new file mode 100644 index 00000000..ea2c6b20 --- /dev/null +++ b/crates/tw-gateway/src/web_tools/quality.rs @@ -0,0 +1,316 @@ +//! A relevance floor for public search pages. HTTP 200 is not evidence that an +//! engine honored the query: some pages silently fall back to a broader search. + +use std::collections::BTreeSet; + +use tw_api::WebSource; + +fn cjk(c: char) -> bool { + matches!(c as u32, 0x3400..=0x9fff | 0x20000..=0x323af | 0x3040..=0x30ff | 0xac00..=0xd7af) +} + +fn word(s: &str) -> String { + // Common singular/plural differences should not hide otherwise relevant pages. + if s.len() > 4 && s.ends_with('s') && !s.ends_with("ss") { + s[..s.len() - 1].to_string() + } else { + s.to_string() + } +} + +fn terms(s: &str, query: bool) -> BTreeSet { + let mut terms = BTreeSet::new(); + for part in s.to_lowercase().split(|c: char| !c.is_alphanumeric()) { + let mut ascii = String::new(); + let mut previous = None; + for c in part.chars() { + if cjk(c) { + if !ascii.is_empty() { + terms.insert(word(&ascii)); + ascii.clear(); + } + if let Some(p) = previous { + terms.insert(format!("{p}{c}")); + } + previous = Some(c); + } else { + previous = None; + ascii.push(c); + } + } + if !ascii.is_empty() { + terms.insert(word(&ascii)); + } + } + if query { + terms.retain(|s| { + s.chars().count() > 1 + && !matches!( + s.as_str(), + "the" + | "an" + | "and" + | "or" + | "for" + | "of" + | "to" + | "in" + | "on" + | "is" + | "are" + | "with" + | "how" + | "what" + | "where" + | "when" + | "who" + | "why" + | "do" + | "does" + | "can" + | "please" + | "find" + | "search" + | "about" + | "tell" + | "me" + | "use" + | "using" + | "documentation" + | "official" + | "tutorial" + | "guide" + ) + }); + } + terms +} + +/// Reject clearly unrelated results while preserving the engine's ranking. +/// Keyword coverage must not promote a secondary source over the primary source +/// merely because a longer snippet repeats more query terms. +/// This is a lexical sanity check, not a claim to judge factual correctness. +/// Return whether the remaining set is usable. A lone partial match in a mostly +/// unrelated response must not stop failover; a real single-result response or +/// an exact lexical match still can. This does not establish factual accuracy. +pub(super) fn filter(query: &str, sources: &mut Vec) -> bool { + // Operators constrain the engine, not the words that must appear in a page. + let query = query + .split_whitespace() + .filter(|part| { + !part.starts_with('-') + && !part.to_ascii_lowercase().starts_with("site:") + && !part.to_ascii_lowercase().starts_with("filetype:") + }) + .collect::>() + .join(" "); + let wanted = terms(&query, true); + if wanted.is_empty() { + return true; + } + let score = |source: &WebSource| { + // A search/redirect URL can echo the entire query even when its contents + // are unrelated. Only the origin and path contribute URL evidence. + let address = reqwest::Url::parse(&source.url) + .map(|url| format!("{} {}", url.host_str().unwrap_or_default(), url.path())) + .unwrap_or_default(); + let visible = terms( + &format!("{} {} {}", source.title, address, source.text), + false, + ); + wanted.intersection(&visible).count() + }; + // One matching word out of a multi-word query is usually a broadened result. + // Keep this conservative: natural-language questions and cross-language + // results may share only a few terms, even when they answer the question. + let minimum = if wanted.len() <= 2 { 1 } else { 2 }; + let original = sources.len(); + let mut best = 0; + sources.retain(|s| { + let matched = score(s); + best = best.max(matched); + matched >= minimum + }); + !sources.is_empty() && (original < 4 || sources.len() * 4 > original || best == wanted.len()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn source(title: &str, url: &str, text: &str) -> WebSource { + WebSource { + title: title.into(), + url: url.into(), + text: text.into(), + engine: None, + published_at: None, + } + } + + #[test] + fn sparse_partial_matches_do_not_stop_failover() { + for (query, partial) in [ + ( + "SQLite release notes", + source( + "SQLite downloads", + "https://sqlite.org/download.html", + "Download the current release", + ), + ), + ( + "OpenAI Responses API reference", + source( + "OpenAI Platform", + "https://platform.openai.com/", + "Build with the API", + ), + ), + ] { + let mut single = vec![partial.clone()]; + assert!( + filter(query, &mut single), + "a genuine single result is allowed" + ); + let mut rows = vec![partial]; + rows.extend( + (0..7).map(|_| source("Unrelated", "https://example.org/", "Other topics")), + ); + assert!(!filter(query, &mut rows)); + assert_eq!( + rows.len(), + 1, + "the remaining partial match cannot establish usefulness" + ); + } + let mut rows = vec![source( + "Widget lock timeout", + "https://docs.example.org/widget", + "Reference", + )]; + rows.extend((0..7).map(|_| source("Unrelated", "https://example.org/", "Other topics"))); + assert!( + filter("Widget lock timeout guide", &mut rows), + "an exact match remains usable" + ); + assert_eq!(rows.len(), 1); + } + + #[test] + fn broad_and_unrelated_results_do_not_satisfy_a_specific_query() { + let mut rows = vec![ + source( + "Welcome to Python.org", + "https://python.org/", + "The Python programming language", + ), + source( + "Python documentation", + "https://docs.python.org/", + "Official Python documentation and tutorials", + ), + source("搜狗输入法", "https://sogou.com/", "输入法下载"), + source( + "Coroutines and Tasks", + "https://docs.python.org/3/library/asyncio-task.html", + "TaskGroup documentation for structured concurrency", + ), + ]; + filter("Python asyncio TaskGroup documentation", &mut rows); + assert_eq!(rows.len(), 1); + assert!(rows[0].url.contains("asyncio-task")); + } + + #[test] + fn broad_product_homepages_do_not_displace_specific_results() { + let mut rows = vec![ + source( + "SQLite Home Page", + "https://sqlite.org/", + "A small database engine", + ), + source( + "Release History", + "https://sqlite.org/changes.html", + "SQLite release notes", + ), + source( + "SQLite releases", + "https://example.org/sqlite", + "Release details", + ), + ]; + filter("SQLite release notes", &mut rows); + assert_eq!(rows.len(), 2); + assert!(rows[0].url.ends_with("changes.html")); + } + + #[test] + fn chinese_queries_and_inline_identifiers_have_meaningful_terms() { + let mut rows = vec![ + source( + "中华人民共和国劳动合同法", + "https://example.org/law", + "第三十八条 劳动者可以解除劳动合同", + ), + source("中华人民共和国", "https://example.org/about", "国家介绍"), + ]; + filter("劳动合同法 第三十八条", &mut rows); + assert_eq!(rows.len(), 1); + let mut rows = vec![source( + "OnceLock", + "https://doc.rust-lang.org/std/sync/struct.OnceLock.html", + "OnceLock documentation", + )]; + filter("Rust std::sync::OnceLock documentation", &mut rows); + assert_eq!(rows.len(), 1); + } + + #[test] + fn operators_and_question_wording_do_not_hide_relevant_results() { + let mut rows = vec![source( + "TaskGroup", + "https://docs.python.org/3/library/asyncio-task.html", + "TaskGroup is an asynchronous context manager", + )]; + filter( + "How does Python use TaskGroup to wait for several tasks to finish? site:docs.python.org -tutorial", + &mut rows, + ); + assert_eq!(rows.len(), 1); + filter("如何使用Python的TaskGroup等待所有任务结束", &mut rows); + assert_eq!(rows.len(), 1); + } + + #[test] + fn query_echo_in_a_url_is_not_relevance_evidence() { + let mut rows = vec![source( + "Other topics", + "https://example.org/search?q=Python+asyncio+TaskGroup+documentation", + "No results", + )]; + filter("Python asyncio TaskGroup documentation", &mut rows); + assert!(rows.is_empty()); + } + + #[test] + fn filtering_keeps_engine_order_when_an_extra_keyword_matches_a_later_result() { + let mut rows = vec![ + source( + "Release history", + "https://sqlite.org/changes.html", + "SQLite releases", + ), + source( + "SQLite release notes", + "https://example.org/", + "SQLite release notes", + ), + source("Unrelated", "https://example.org/other", "Other topics"), + ]; + filter("SQLite release notes", &mut rows); + assert_eq!(rows.len(), 2); + assert_eq!(rows[0].url, "https://sqlite.org/changes.html"); + } +} diff --git a/crates/tw-gateway/src/web_tools/session.rs b/crates/tw-gateway/src/web_tools/session.rs new file mode 100644 index 00000000..136ffafc --- /dev/null +++ b/crates/tw-gateway/src/web_tools/session.rs @@ -0,0 +1,406 @@ +//! Browser-compatible HTTP sessions for first-party search pages only. +//! Cookies stay in memory and are isolated by caller and engine by WebTools. + +use std::collections::HashMap; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use http::{HeaderMap, HeaderValue}; +use reqwest::cookie::{CookieStore, Jar}; +use tw_api::{SearchEngine, WebSearchLocale}; +use tw_types::Msg; + +use super::{ + error, + providers::{Kind, Profile, Spec}, + public, +}; + +#[derive(Default)] +pub(super) struct Session { + cookies: Jar, + cookie_bytes: AtomicUsize, + pub ready: tokio::sync::OnceCell<()>, +} + +pub(super) struct TimedSession { + pub created: Instant, + pub session: Session, +} + +impl Default for TimedSession { + fn default() -> Self { + Self { + created: Instant::now(), + session: Session::default(), + } + } +} + +#[derive(Clone, Hash, PartialEq, Eq)] +struct Key { + identity: String, + engine: SearchEngine, + language: String, + region: String, +} + +pub(super) struct Pool { + entries: Mutex)>>, + capacity: usize, + ttl: Duration, +} + +impl Default for Pool { + fn default() -> Self { + Self { + entries: Mutex::new(HashMap::new()), + capacity: 64, + ttl: Duration::from_secs(600), + } + } +} + +impl Pool { + pub fn get( + &self, + identity: &str, + spec: &Spec, + locale: &WebSearchLocale, + ) -> Result>, Msg> { + if spec.home.is_none() { + return Ok(None); + } + let key = Key { + identity: identity.into(), + engine: spec.id, + language: locale.language.clone(), + region: locale.region.clone(), + }; + let mut entries = self.entries.lock().unwrap_or_else(|e| e.into_inner()); + entries.retain(|_, (_, value)| { + Arc::strong_count(value) > 1 || value.created.elapsed() < self.ttl + }); + if let Some((used, value)) = entries.get_mut(&key) { + *used = Instant::now(); + return Ok(Some(value.clone())); + } + if entries.len() >= self.capacity { + // Evict one idle session rather than reject a new caller while stale + // idle cookies occupy every slot. In-flight sessions are never split. + let oldest = entries + .iter() + .filter(|(_, (_, value))| Arc::strong_count(value) == 1) + .min_by_key(|(_, (used, _))| *used) + .map(|(key, _)| key.clone()); + if let Some(oldest) = oldest { + entries.remove(&oldest); + } else { + return Err(error("too many active search sessions")); + } + } + let value = Arc::new(TimedSession::default()); + entries.insert(key, (Instant::now(), value.clone())); + Ok(Some(value)) + } +} + +#[derive(Clone, Copy)] +pub(super) struct Request<'a> { + pub spec: &'static Spec, + pub kind: Kind, + pub session: Option<&'a Session>, + pub locale: &'a WebSearchLocale, +} + +impl Request<'_> { + pub fn headers(&self, url: &reqwest::Url) -> Result { + if !self.spec.allows(url) { + return Err(error( + "search redirected outside its supported first-party pages", + )); + } + let mut headers = match self.spec.profile { + Profile::Basic => public::request_headers(), + Profile::BrowserHttp1 => browser_headers(self.kind, self.locale)?, + }; + if (self.session.is_some_and(|s| s.ready.get().is_some()) || self.kind == Kind::Data) + && let Some(home) = self.spec.home + { + // Only the origin is sent, never another caller's query or result URL. + headers.insert("referer", HeaderValue::from_static(home)); + } + if let Some(cookie) = self.session.and_then(|s| s.cookies.cookies(url)) { + headers.insert("cookie", cookie); + } + Ok(headers) + } + + pub fn remember(&self, url: &reqwest::Url, headers: &HeaderMap) -> Result<(), Msg> { + if !self.spec.allows(url) { + return Err(error( + "refused cookies from an unsupported search destination", + )); + } + let Some(session) = self.session else { + return Ok(()); + }; + let cookies: Vec<_> = headers.get_all("set-cookie").iter().collect(); + let bytes = cookies + .iter() + .map(|cookie| cookie.as_bytes().len()) + .sum::(); + if cookies.len() > 32 + || cookies.iter().any(|cookie| cookie.as_bytes().len() > 4096) + || session + .cookie_bytes + .fetch_add(bytes, Ordering::Relaxed) + .saturating_add(bytes) + > 1024 * 1024 + { + return Err(error("search session exceeded the cookie limit")); + } + session.cookies.set_cookies(&mut cookies.into_iter(), url); + Ok(()) + } +} + +fn browser_headers(kind: Kind, locale: &WebSearchLocale) -> Result { + let script = kind == Kind::Data; + let mut headers = HeaderMap::new(); + // The compatibility tokens request the normal web layout. Keep our own + // product name; do not rotate identities or use cookies from a browser. + headers.insert("user-agent", HeaderValue::from_static( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 ThinkWatch/1.0" + )); + headers.insert( + "accept", + HeaderValue::from_static(if script { + "*/*" + } else { + "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" + }), + ); + let language = match locale.language.as_str() { + "zh-Hans" => "zh-CN", + "zh-Hant" => "zh-TW", + "" => "en-US", + other => other, + }; + let base = language.split('-').next().unwrap_or(language); + let mut language = if language == base { + language.to_string() + } else { + format!("{language},{base};q=0.9") + }; + if base != "en" { + language.push_str(",en;q=0.8"); + } + headers.insert( + "accept-language", + HeaderValue::from_str(&language).map_err(|_| error("invalid search language"))?, + ); + if !script { + headers.insert("upgrade-insecure-requests", HeaderValue::from_static("1")); + } + Ok(headers) +} + +#[cfg(test)] +mod tests { + use super::super::providers; + use super::*; + + #[test] + fn cookie_sessions_are_independent_of_the_request_profile() { + static BASIC: Spec = Spec { + id: SearchEngine::Bing, + name: "Basic session", + min_interval_ms: 0, + profile: Profile::Basic, + home: Some("https://www.bing.com/"), + endpoints: &[providers::Endpoint { + host: "www.bing.com", + paths: &["/", "/search"], + }], + }; + let session = Session::default(); + let locale = WebSearchLocale::default(); + let request = Request { + spec: &BASIC, + kind: Kind::Document, + session: Some(&session), + locale: &locale, + }; + let url = "https://www.bing.com/".parse().unwrap(); + let mut response = HeaderMap::new(); + response.insert( + "set-cookie", + "session=test; Path=/; Secure".parse().unwrap(), + ); + request.remember(&url, &response).unwrap(); + session.ready.set(()).unwrap(); + let headers = request.headers(&url).unwrap(); + assert_eq!(headers["cookie"], "session=test"); + assert_eq!(headers["referer"], "https://www.bing.com/"); + assert_eq!(headers["accept-encoding"], "identity"); + } + + #[tokio::test] + async fn sessions_coalesce_warmup_isolate_locales_and_evict_only_idle_entries() { + let pool = Pool { + capacity: 2, + ..Pool::default() + }; + let spec = providers::get(SearchEngine::Bing).spec(); + let locale = WebSearchLocale::default(); + let first = pool.get("caller", spec, &locale).unwrap().unwrap(); + let same = pool.get("caller", spec, &locale).unwrap().unwrap(); + assert!(Arc::ptr_eq(&first, &same)); + let calls = AtomicUsize::new(0); + futures::future::join_all((0..4).map(|_| { + first.session.ready.get_or_init(|| async { + calls.fetch_add(1, Ordering::Relaxed); + tokio::task::yield_now().await; + }) + })) + .await; + assert_eq!(calls.load(Ordering::Relaxed), 1); + let english = WebSearchLocale { + language: "en".into(), + region: "US".into(), + }; + let second = pool.get("caller", spec, &english).unwrap().unwrap(); + assert!(!Arc::ptr_eq(&first, &second)); + assert!(pool.get("third", spec, &locale).is_err()); + drop(second); + assert!(pool.get("third", spec, &locale).is_ok()); + assert!(Arc::ptr_eq( + &first, + &pool.get("caller", spec, &locale).unwrap().unwrap() + )); + assert_eq!(pool.entries.lock().unwrap().len(), 2); + } + + #[test] + fn expired_sessions_are_replaced_after_their_last_user_releases_them() { + let pool = Pool { + ttl: Duration::ZERO, + ..Pool::default() + }; + let spec = providers::get(SearchEngine::Bing).spec(); + let locale = WebSearchLocale::default(); + let first = pool.get("caller", spec, &locale).unwrap().unwrap(); + first.session.ready.set(()).unwrap(); + assert!(Arc::ptr_eq( + &first, + &pool.get("caller", spec, &locale).unwrap().unwrap() + )); + drop(first); + assert!( + pool.get("caller", spec, &locale) + .unwrap() + .unwrap() + .session + .ready + .get() + .is_none() + ); + } + + #[test] + fn search_cookies_follow_scope_and_do_not_cross_callers_or_engines() { + let session = Session::default(); + let locale = WebSearchLocale { + language: "zh-Hans".into(), + region: "CN".into(), + }; + let request = Request { + spec: providers::get(SearchEngine::Bing).spec(), + kind: Kind::Document, + session: Some(&session), + locale: &locale, + }; + let home_url = "https://www.bing.com/".parse().unwrap(); + let mut headers = HeaderMap::new(); + headers.append( + "set-cookie", + "session=test; Path=/search; Secure; HttpOnly" + .parse() + .unwrap(), + ); + headers.append( + "set-cookie", + "foreign=test; Domain=example.org; Path=/".parse().unwrap(), + ); + request.remember(&home_url, &headers).unwrap(); + assert!(!request.headers(&home_url).unwrap().contains_key("cookie")); + let search = "https://www.bing.com/search".parse().unwrap(); + let headers = request.headers(&search).unwrap(); + assert_eq!(headers["cookie"], "session=test"); + assert_eq!(headers["accept-language"], "zh-CN,zh;q=0.9,en;q=0.8"); + assert!(headers["accept"].to_str().unwrap().starts_with("text/html")); + assert!(!headers.contains_key("referer")); + session.ready.set(()).unwrap(); + assert_eq!( + request.headers(&search).unwrap()["referer"], + "https://www.bing.com/" + ); + assert!( + request + .headers(&"https://example.org/search".parse().unwrap()) + .is_err() + ); + assert!( + request + .headers(&"https://duckduckgo.com/".parse().unwrap()) + .is_err() + ); + let other = Session::default(); + let other = Request { + session: Some(&other), + ..request + }; + assert!(!other.headers(&search).unwrap().contains_key("cookie")); + } + + #[test] + fn data_flow_without_session_keeps_its_existing_request_profile() { + let locale = WebSearchLocale::default(); + let request = Request { + spec: providers::get(SearchEngine::Duckduckgo).spec(), + kind: Kind::Data, + session: None, + locale: &locale, + }; + let data = "https://links.duckduckgo.com/d.js?q=test".parse().unwrap(); + let headers = request.headers(&data).unwrap(); + assert_eq!(headers, public::request_headers()); + assert!(!headers.contains_key("cookie")); + assert!(!headers.contains_key("referer")); + } + + #[test] + fn script_requests_have_origin_referer_and_cookie_budget_is_bounded() { + let session = Session::default(); + let locale = WebSearchLocale::default(); + let request = Request { + spec: providers::get(SearchEngine::Bing).spec(), + kind: Kind::Data, + session: Some(&session), + locale: &locale, + }; + let data = "https://www.bing.com/search?q=test".parse().unwrap(); + let headers = request.headers(&data).unwrap(); + assert_eq!(headers["accept"], "*/*"); + assert_eq!(headers["referer"], "https://www.bing.com/"); + assert!(!headers.contains_key("upgrade-insecure-requests")); + let mut response = HeaderMap::new(); + response.insert( + "set-cookie", + format!("a={}", "x".repeat(4096)).parse().unwrap(), + ); + assert!(request.remember(&data, &response).is_err()); + } +} diff --git a/docs/config.md b/docs/config.md index 6eaba1bb..624d7776 100644 --- a/docs/config.md +++ b/docs/config.md @@ -2031,7 +2031,7 @@ sync: Configure search, page reading and find-in-page under **Routing → Auxiliary** in Lite. `core` handles standalone Claude Code search requests without a helper model call, and runs hosted web tools in Anthropic Messages and OpenAI Responses requests through a bounded model/tool loop. Main-model calls are still metered. Client-side webpage fetching is unchanged. Web-tool responses are buffered and emitted as native JSON or SSE; keep-alives continue while waiting. Public HTTP/HTTPS text pages are supported; location-specific and cache-only native search modes require upstream execution. -Search uses Bing RSS and DuckDuckGo HTML without API keys. Challenges and unrecognized pages are failures, with per-engine pacing and cooldowns; they are never reported as empty search results. HTTP, HTTPS and SOCKS proxies tunnel to the public IPs checked by Core, keeping the original hostname for TLS and HTTP. Responses WebSocket connections use the same protected execution path. Only standalone helper searches can fall back upstream, when explicitly enabled. +Search connects directly to Google Docs web research, DuckDuckGo and Bing without API keys or a browser runtime. Automatic order is Google (Docs), DuckDuckGo, then Bing. Google (Docs) searches public web pages through a legacy endpoint; it depends on eligible public document contexts and may stop working if Google changes the service. It does not read or edit the context documents, does not support regional preferences, and enforces at least six seconds between requests. Document access failures can select another context; throttling and server errors cool down the engine and fall back to the next source. Challenges and unrecognized pages are failures, with per-engine pacing and cooldowns; they are never reported as empty search results. HTTP, HTTPS and SOCKS proxies tunnel to the public IPs checked by Core, keeping the original hostname for TLS and HTTP. Responses WebSocket connections use the same protected execution path. Only standalone helper searches can fall back upstream, when explicitly enabled. @@ -2039,7 +2039,7 @@ Search uses Bing RSS and DuckDuckGo HTML without API keys. Challenges and unreco | Field | Type | Default | Description | |---|---|---|---| | `mode` | `upstream` \| `core` | `upstream` | upstream forwards hosted tools; core executes supported web operations. | -| `engines` | list of strings | `[]` | Empty selects available engines automatically. Otherwise list bing and/or duckduckgo in priority order. | +| `engines` | list of strings | `[]` | Empty tries google_docs, duckduckgo, then bing. Otherwise list these engine IDs in priority order. | | `proxy` | string | `system` | system, direct, or an existing proxy name. | | `language` | string | `''` | BCP 47 language tag. Empty follows the system language on the Core host (including a remote Core); if unavailable, the engine decides. Lite provides a language picker. | | `region` | string | `''` | Two-letter country/region code. Empty follows the region in the Core host’s system locale; if unavailable, the engine decides. Search engines apply supported regional preferences on a best-effort basis. | @@ -2050,7 +2050,7 @@ Search uses Bing RSS and DuckDuckGo HTML without API keys. Challenges and unreco | `max_page_chars` | integer | `12000` | Returned page characters, 1000–50000. | | `timeout_secs` | integer | `30` | Whole operation timeout including queueing, 1–120 seconds. | | `max_concurrent` | integer | `4` | Concurrent web operations, 1–16. | -| `interval_ms` | integer | `1000` | Minimum interval per search engine, 250–60000 milliseconds. | +| `interval_ms` | integer | `1000` | Minimum interval per search engine, 250–60000 milliseconds. A source may enforce a higher minimum (google_docs: 6000). | | `cache_secs` | integer | `600` | Successful result lifetime, 0–86400 seconds. Zero disables caching. | | `max_tool_calls` | integer | `8` | Maximum web operations per model request, 1–32. | | `fallback_to_upstream` | bool | `false` | Allow standalone searches to fall back to the upstream before returning results. This may incur upstream fees. | diff --git a/docs/config.zh-CN.md b/docs/config.zh-CN.md index bf2e7094..f31677ec 100644 --- a/docs/config.zh-CN.md +++ b/docs/config.zh-CN.md @@ -1470,7 +1470,7 @@ sync: 在 Lite 的 **路由 → 辅助请求** 中配置搜索、网页读取和页内查找。`core` 直接处理 Claude Code 的独立搜索子请求,省去搜索助手模型调用;Anthropic Messages 和 OpenAI Responses 对话中的联网工具由 Core 执行后交回模型继续处理,主模型仍按实际用量计费。客户端直接抓取网页的行为不变。联网工具响应收齐后按原生 JSON 或 SSE 返回,等待期间保持心跳。支持公开 HTTP/HTTPS 文本网页;指定地理位置或仅使用厂商缓存的搜索模式需要沿用上游。 -搜索使用 Bing RSS 和 DuckDuckGo HTML,不需要搜索 API Key。验证码和无法识别的页面按失败处理,配合请求间隔、冷却和其他引擎重试,不会当作“没有结果”。HTTP、HTTPS 和 SOCKS 代理连接 Core 已校验的公开 IP,TLS 和 HTTP 保留原始域名。Responses WebSocket 使用同一套受保护的执行流程。只有独立搜索辅助请求可以按显式配置回退上游。 +搜索直接连接 Google Docs 网页检索、DuckDuckGo 和 Bing,无需 API Key 或浏览器运行时。自动顺序为 Google (Docs)、DuckDuckGo、Bing。Google (Docs) 通过旧版接口搜索公开网页,依赖仍可用的公开文档上下文,Google 调整服务后可能失效;不会读取或修改这些文档,不支持地区偏好,请求间隔至少 6 秒。文档访问失败可选择备用上下文,限流或服务器错误则冷却该引擎并切换其他搜索源。验证码和无法识别的页面按失败处理,配合请求间隔、冷却和其他引擎重试,不会当作“没有结果”。HTTP、HTTPS 和 SOCKS 代理连接 Core 已校验的公开 IP,TLS 和 HTTP 保留原始域名。Responses WebSocket 使用同一套受保护的执行流程。只有独立搜索辅助请求可以按显式配置回退上游。 @@ -1478,7 +1478,7 @@ sync: | 字段 | 类型 | 默认值 | 说明 | |---|---|---|---| | `mode` | `upstream` \| `core` | `upstream` | upstream 沿用上游;core 执行已适配的联网工具。 | -| `engines` | 字符串列表 | `[]` | 空列表自动选择可用引擎;手动填写 bing、duckduckgo,按顺序尝试。 | +| `engines` | 字符串列表 | `[]` | 空列表依次尝试 google_docs、duckduckgo、bing;也可手动按优先顺序填写这些引擎 ID。 | | `proxy` | 字符串 | `system` | system、direct 或已定义的代理名称。 | | `language` | 字符串 | `''` | BCP 47 语言标签。空值跟随运行 Core 的主机系统语言(包括远程 Core),无法识别时由引擎决定。Lite 中通过语言列表选择。 | | `region` | 字符串 | `''` | 两字母国家或地区代码。空值跟随 Core 主机系统语言中的地区,无法识别时由引擎决定。搜索引擎按支持范围尽量应用偏好。 | @@ -1489,7 +1489,7 @@ sync: | `max_page_chars` | 整数 | `12000` | 每页返回字符上限,1000–50000。 | | `timeout_secs` | 整数 | `30` | 包括排队在内的单次操作超时,1–120 秒。 | | `max_concurrent` | 整数 | `4` | 联网操作并发数,1–16。 | -| `interval_ms` | 整数 | `1000` | 同一搜索引擎的最小请求间隔,250–60000 毫秒。 | +| `interval_ms` | 整数 | `1000` | 同一搜索引擎的最小请求间隔,250–60000 毫秒。搜索源可要求更长间隔(google_docs:6000)。 | | `cache_secs` | 整数 | `600` | 成功结果缓存时间,0–86400 秒;0 关闭缓存。 | | `max_tool_calls` | 整数 | `8` | 每个模型请求最多执行联网工具次数,1–32。 | | `fallback_to_upstream` | 布尔 | `false` | 独立搜索返回结果之前失败时允许回退上游,可能产生上游费用。 | diff --git a/release-notes/0.98.0.md b/release-notes/0.98.0.md new file mode 100644 index 00000000..0819a0a5 --- /dev/null +++ b/release-notes/0.98.0.md @@ -0,0 +1,6 @@ +Core web search now connects directly to Google (Docs), DuckDuckGo and Bing without API keys or a browser runtime. Automatic search tries them in that order and switches sources when an engine fails or returns no usable results. + +- Google (Docs) searches public web pages through a legacy Google Docs research endpoint. Requests are spaced at least six seconds apart, identical concurrent searches share cached results, and throttled or failing engines enter cooldown. Eligible public document contexts are used only to make the search request; no document is edited. This legacy endpoint depends on Google continuing to provide it and does not support regional preferences. +- Search adapters share bounded requests, browser-style HTTP headers, compression decoding, connection reuse, and isolated in-memory sessions where needed. HTTP and SOCKS proxies support search-engine DNS resolution while page reads retain public-address checks and pinned connections. +- Relevance checks reject unrelated results instead of presenting them as a successful search. Domain restrictions, URL normalization, page-text extraction and failure messages have been improved. Error responses retain the original cause and retry timing. +- Core advertises available search engines to Lite, so the connected gateway determines which sources the desktop can select. Existing configurations remain opt-in through Core execution under Routing → Auxiliary.