From e85cfea12286f904b675de8541fe492daefcf599 Mon Sep 17 00:00:00 2001
From: fylorn <249551762+fylorn@users.noreply.github.com>
Date: Mon, 16 Nov 2026 15:01:53 +0800
Subject: [PATCH 1/2] feat(web-search): release Core 0.98.0 with direct Google
search
Public search pages can return challenges or unrelated results. Share bounded source adapters, HTTP sessions, relevance checks, pacing, caching and failover, and add the keyless Google Docs research endpoint. Advertise engine capabilities to desktop clients while retaining public-address protections for page reads.
---
Cargo.lock | 180 ++++-
Cargo.toml | 2 +-
bin/twcore/src/upgrade.rs | 6 +
crates/tw-api/src/web_tools.rs | 13 +
crates/tw-config/tests/manual/schema.rs | 8 +-
crates/tw-control/src/sync.rs | 6 +
crates/tw-control/src/web_tools.rs | 1 +
crates/tw-gateway/Cargo.toml | 3 +-
crates/tw-gateway/src/outbound.rs | 6 +
crates/tw-gateway/src/web_tools/cache.rs | 89 +++
crates/tw-gateway/src/web_tools/collect.rs | 286 +++----
crates/tw-gateway/src/web_tools/mod.rs | 748 +++++++++++++++---
.../src/web_tools/providers/bing.rs | 282 +++++++
.../src/web_tools/providers/duckduckgo.rs | 257 ++++++
.../src/web_tools/providers/google_docs.rs | 315 ++++++++
.../src/web_tools/{ => providers}/locale.rs | 37 +-
.../tw-gateway/src/web_tools/providers/mod.rs | 286 +++++++
crates/tw-gateway/src/web_tools/public.rs | 535 ++++++++++++-
crates/tw-gateway/src/web_tools/quality.rs | 316 ++++++++
crates/tw-gateway/src/web_tools/session.rs | 406 ++++++++++
docs/config.md | 6 +-
docs/config.zh-CN.md | 6 +-
release-notes/0.98.0.md | 6 +
23 files changed, 3415 insertions(+), 385 deletions(-)
create mode 100644 crates/tw-gateway/src/web_tools/cache.rs
create mode 100644 crates/tw-gateway/src/web_tools/providers/bing.rs
create mode 100644 crates/tw-gateway/src/web_tools/providers/duckduckgo.rs
create mode 100644 crates/tw-gateway/src/web_tools/providers/google_docs.rs
rename crates/tw-gateway/src/web_tools/{ => providers}/locale.rs (62%)
create mode 100644 crates/tw-gateway/src/web_tools/providers/mod.rs
create mode 100644 crates/tw-gateway/src/web_tools/quality.rs
create mode 100644 crates/tw-gateway/src/web_tools/session.rs
create mode 100644 release-notes/0.98.0.md
diff --git a/Cargo.lock b/Cargo.lock
index 7498487f..03ab7f4c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -79,6 +79,21 @@ dependencies = [
"memchr",
]
+[[package]]
+name = "alloc-no-stdlib"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c2fb6cfd47bf496ff64095c20eaba0c201404ee38714d4142fcfa1dc334fcc7a"
+
+[[package]]
+name = "alloc-stdlib"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b5c1865780388bfa186411ab5f247819487fc4864c6e9c3106611fa347586e1"
+dependencies = [
+ "alloc-no-stdlib",
+]
+
[[package]]
name = "allocator-api2"
version = "0.2.21"
@@ -177,6 +192,18 @@ version = "0.7.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56"
+[[package]]
+name = "async-compression"
+version = "0.4.52"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ad47a740fb906fffdd891e8b6d5fde2511600f7bfaebd68939fa08a14999459"
+dependencies = [
+ "compression-codecs",
+ "compression-core",
+ "pin-project-lite",
+ "tokio",
+]
+
[[package]]
name = "async-stream"
version = "0.3.6"
@@ -520,6 +547,27 @@ dependencies = [
"hybrid-array",
]
+[[package]]
+name = "brotli"
+version = "9.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "627b7cfb0578f0eafce087a90fcd40add1231900b69e4d4de2d456260a4f5d81"
+dependencies = [
+ "alloc-no-stdlib",
+ "alloc-stdlib",
+ "brotli-decompressor",
+]
+
+[[package]]
+name = "brotli-decompressor"
+version = "6.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75c74c9c51b9140663e196b5fb88bc94c334d7a497c8311cb759d8557874747f"
+dependencies = [
+ "alloc-no-stdlib",
+ "alloc-stdlib",
+]
+
[[package]]
name = "bumpalo"
version = "3.20.3"
@@ -709,6 +757,26 @@ dependencies = [
"memchr",
]
+[[package]]
+name = "compression-codecs"
+version = "0.4.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "84f840cff84d10085260e460023934d0af699e71212b09b2a41452e8bd9cda43"
+dependencies = [
+ "brotli",
+ "compression-core",
+ "flate2",
+ "memchr",
+ "zstd",
+ "zstd-safe",
+]
+
+[[package]]
+name = "compression-core"
+version = "0.4.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2ebdda89edc534bf8d5dd2857562633b6495557e5846552ade994b99ed13c36a"
+
[[package]]
name = "const-oid"
version = "0.10.2"
@@ -721,6 +789,17 @@ version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b"
+[[package]]
+name = "cookie"
+version = "0.18.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1a373e3602691c3cdea496d2f0ee5935151e6168fe87739483c463db1b2f2f87"
+dependencies = [
+ "percent-encoding",
+ "time",
+ "version_check",
+]
+
[[package]]
name = "cookie-factory"
version = "0.3.3"
@@ -730,6 +809,24 @@ dependencies = [
"futures",
]
+[[package]]
+name = "cookie_store"
+version = "0.22.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "15b2c103cf610ec6cae3da84a766285b42fd16aad564758459e6ecf128c75206"
+dependencies = [
+ "cookie",
+ "document-features",
+ "idna",
+ "log",
+ "publicsuffix",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "time",
+ "url",
+]
+
[[package]]
name = "core-foundation"
version = "0.9.4"
@@ -1107,6 +1204,15 @@ dependencies = [
"syn 3.0.5",
]
+[[package]]
+name = "document-features"
+version = "0.2.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61"
+dependencies = [
+ "litrs",
+]
+
[[package]]
name = "dtoa"
version = "1.0.11"
@@ -2104,6 +2210,12 @@ version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae"
+[[package]]
+name = "litrs"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4744e383959f0db86ede514b809b1c53251889093803c05267acc7d4e7030d70"
+
[[package]]
name = "lock_api"
version = "0.4.14"
@@ -2597,6 +2709,22 @@ dependencies = [
"unicode-ident",
]
+[[package]]
+name = "psl-types"
+version = "2.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac"
+
+[[package]]
+name = "publicsuffix"
+version = "2.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f42ea446cab60335f76979ec15e12619a2165b5ae2c12166bef27d283a9fadf"
+dependencies = [
+ "idna",
+ "psl-types",
+]
+
[[package]]
name = "pulley-interpreter"
version = "49.0.5"
@@ -2873,6 +3001,8 @@ checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3"
dependencies = [
"base64 0.22.1",
"bytes",
+ "cookie",
+ "cookie_store",
"futures-core",
"futures-util",
"h2",
@@ -3864,12 +3994,17 @@ version = "0.6.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840"
dependencies = [
+ "async-compression",
"bitflags",
"bytes",
+ "futures-core",
"futures-util",
"http",
"http-body",
+ "http-body-util",
"pin-project-lite",
+ "tokio",
+ "tokio-util",
"tower",
"tower-layer",
"tower-service",
@@ -4013,7 +4148,7 @@ dependencies = [
[[package]]
name = "tw-api"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"serde",
"serde_json",
@@ -4024,7 +4159,7 @@ dependencies = [
[[package]]
name = "tw-bedrock"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"aws-credential-types",
"aws-sigv4",
@@ -4045,7 +4180,7 @@ dependencies = [
[[package]]
name = "tw-breaker"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"serde",
"serde_json",
@@ -4053,7 +4188,7 @@ dependencies = [
[[package]]
name = "tw-config"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"blake3",
"icu_locale_core",
@@ -4081,7 +4216,7 @@ dependencies = [
[[package]]
name = "tw-control"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"axum",
"base64 0.22.1",
@@ -4127,7 +4262,7 @@ dependencies = [
[[package]]
name = "tw-dialect"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"memchr",
"serde",
@@ -4136,7 +4271,7 @@ dependencies = [
[[package]]
name = "tw-engine"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"chrono",
"serde",
@@ -4150,7 +4285,7 @@ dependencies = [
[[package]]
name = "tw-gateway"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"arc-swap",
"async-stream",
@@ -4198,13 +4333,14 @@ dependencies = [
"tw-pricing",
"tw-secret",
"tw-types",
+ "url",
"uuid",
"zstd",
]
[[package]]
name = "tw-guard"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"aho-corasick",
"base64 0.22.1",
@@ -4225,7 +4361,7 @@ dependencies = [
[[package]]
name = "tw-handover"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"libc",
"serde",
@@ -4238,7 +4374,7 @@ dependencies = [
[[package]]
name = "tw-link"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"serde",
"serde_json",
@@ -4252,7 +4388,7 @@ dependencies = [
[[package]]
name = "tw-observe"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"tokio",
"tracing",
@@ -4261,7 +4397,7 @@ dependencies = [
[[package]]
name = "tw-pack"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"age",
"blake3",
@@ -4285,7 +4421,7 @@ dependencies = [
[[package]]
name = "tw-plugin"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"libc",
"rand 0.10.2",
@@ -4301,7 +4437,7 @@ dependencies = [
[[package]]
name = "tw-pricing"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"arc-swap",
"flate2",
@@ -4314,14 +4450,14 @@ dependencies = [
[[package]]
name = "tw-secret"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"thiserror 2.0.20",
]
[[package]]
name = "tw-store"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"blake3",
"bytes",
@@ -4341,7 +4477,7 @@ dependencies = [
[[package]]
name = "tw-sync"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"age",
"axum",
@@ -4369,7 +4505,7 @@ dependencies = [
[[package]]
name = "tw-types"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"serde",
"serde_json",
@@ -4378,7 +4514,7 @@ dependencies = [
[[package]]
name = "tw-watch"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"libc",
"notify",
@@ -4389,7 +4525,7 @@ dependencies = [
[[package]]
name = "tw-yaml"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"saphyr-parser",
"serde_yaml_ng",
@@ -4399,7 +4535,7 @@ dependencies = [
[[package]]
name = "twcore"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"anyhow",
"axum",
diff --git a/Cargo.toml b/Cargo.toml
index 2fd657c0..13067cf5 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -61,7 +61,7 @@ categories = ["network-programming", "web-programming::http-server"]
# 二进制走 CalVer,crate 走 SemVer —— 两者是两回事,卖给不同的人。
# 这里是 crate 的版本。
-version = "0.97.0"
+version = "0.98.0"
[workspace.dependencies]
# ── 内部 crate ───────────────────────────────────────────
diff --git a/bin/twcore/src/upgrade.rs b/bin/twcore/src/upgrade.rs
index c2dcb45a..c6415730 100644
--- a/bin/twcore/src/upgrade.rs
+++ b/bin/twcore/src/upgrade.rs
@@ -285,6 +285,12 @@ fn client() -> Result {
/// 走环境变量里的代理(`HTTPS_PROXY`):服务器上出网常常只有这一条路。
fn builder() -> reqwest::ClientBuilder {
reqwest::Client::builder()
+ // Search opts into content decoding; release downloads retain their
+ // existing behavior when those reqwest features are unified.
+ .no_gzip()
+ .no_brotli()
+ .no_deflate()
+ .no_zstd()
.user_agent(concat!("twcore/", env!("CARGO_PKG_VERSION")))
.connect_timeout(Duration::from_secs(15))
.timeout(Duration::from_secs(300))
diff --git a/crates/tw-api/src/web_tools.rs b/crates/tw-api/src/web_tools.rs
index 96684c01..0f7ea7ad 100644
--- a/crates/tw-api/src/web_tools.rs
+++ b/crates/tw-api/src/web_tools.rs
@@ -17,6 +17,7 @@ pub enum WebToolsMode {
pub enum SearchEngine {
Bing,
Duckduckgo,
+ GoogleDocs,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
@@ -77,6 +78,18 @@ pub struct WebToolsView {
pub config_version: String,
/// Detected on the Core host, which may be remote. Empty fields mean unavailable.
pub system_locale: WebSearchLocale,
+ /// Engine capabilities advertised by the connected Core.
+ #[serde(default)]
+ pub available_engines: Vec,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "ts", derive(ts_rs::TS))]
+pub struct WebSearchEngine {
+ pub id: SearchEngine,
+ pub name: String,
+ /// The engine's minimum applies even if the user's configured interval is lower.
+ pub min_interval_ms: u32,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
diff --git a/crates/tw-config/tests/manual/schema.rs b/crates/tw-config/tests/manual/schema.rs
index 31e5167f..31c31549 100644
--- a/crates/tw-config/tests/manual/schema.rs
+++ b/crates/tw-config/tests/manual/schema.rs
@@ -1310,8 +1310,8 @@ pub fn sections() -> Vec {
Kind::Strs,
Def::Is("[]"),
t(
- "Empty selects available engines automatically. Otherwise list bing and/or duckduckgo in priority order.",
- "空列表自动选择可用引擎;手动填写 bing、duckduckgo,按顺序尝试。",
+ "Empty tries google_docs, duckduckgo, then bing. Otherwise list these engine IDs in priority order.",
+ "空列表依次尝试 google_docs、duckduckgo、bing;也可手动按优先顺序填写这些引擎 ID。",
),
),
row(
@@ -1403,8 +1403,8 @@ pub fn sections() -> Vec {
Kind::Int,
Def::Is("1000"),
t(
- "Minimum interval per search engine, 250–60000 milliseconds.",
- "同一搜索引擎的最小请求间隔,250–60000 毫秒。",
+ "Minimum interval per search engine, 250–60000 milliseconds. A source may enforce a higher minimum (google_docs: 6000).",
+ "同一搜索引擎的最小请求间隔,250–60000 毫秒。搜索源可要求更长间隔(google_docs:6000)。",
),
),
row(
diff --git a/crates/tw-control/src/sync.rs b/crates/tw-control/src/sync.rs
index f3790f88..ec055a2e 100644
--- a/crates/tw-control/src/sync.rs
+++ b/crates/tw-control/src/sync.rs
@@ -218,6 +218,12 @@ impl Host for CoreHost {
/// 同步位置的密码就跟着去了),证书照常校验
fn http(cfg: &tw_config::Config, proxy: &str) -> Result {
let mut b = reqwest::Client::builder()
+ // Search enables reqwest's decoding features workspace-wide. Keep sync
+ // downloads' existing byte semantics independent of that feature set.
+ .no_gzip()
+ .no_brotli()
+ .no_deflate()
+ .no_zstd()
.connect_timeout(Duration::from_secs(10))
// 一轮里的请求是接连发的;两轮之间的空闲连接不留:很多服务端几秒就关掉空闲连接,
// 留着的下一轮一发就断(发请求那一处另有一次重发)
diff --git a/crates/tw-control/src/web_tools.rs b/crates/tw-control/src/web_tools.rs
index ea5eac71..f504ffd5 100644
--- a/crates/tw-control/src/web_tools.rs
+++ b/crates/tw-control/src/web_tools.rs
@@ -15,6 +15,7 @@ async fn view(State(s): State) -> Json {
config: s.config().web_tools.clone(),
config_version: s.cfg.current().map(|c| c.version()).unwrap_or_default(),
system_locale: tw_config::web_tools::system_locale(),
+ available_engines: tw_gateway::web_tools::available_engines(),
})
}
diff --git a/crates/tw-gateway/Cargo.toml b/crates/tw-gateway/Cargo.toml
index 0785b080..2217796d 100644
--- a/crates/tw-gateway/Cargo.toml
+++ b/crates/tw-gateway/Cargo.toml
@@ -34,7 +34,8 @@ bytes = { workspace = true }
chrono = { workspace = true }
futures = { workspace = true }
http-body-util = { workspace = true }
-reqwest = { workspace = true }
+url = { workspace = true }
+reqwest = { workspace = true, features = ["cookies", "gzip", "brotli", "deflate", "zstd"] }
rustls = { workspace = true }
rustls-platform-verifier = { workspace = true }
tokio-rustls = { workspace = true }
diff --git a/crates/tw-gateway/src/outbound.rs b/crates/tw-gateway/src/outbound.rs
index 64e39411..ff5bba36 100644
--- a/crates/tw-gateway/src/outbound.rs
+++ b/crates/tw-gateway/src/outbound.rs
@@ -7,6 +7,12 @@ use tw_types::msg;
/// 的那批和不走代理的那批会慢慢长出不同的超时行为。
pub(crate) fn base_client_builder() -> reqwest::ClientBuilder {
reqwest::Client::builder()
+ // Web search opts into browser content decoding. Other outbound paths
+ // preserve their existing wire-body semantics despite feature unification.
+ .no_gzip()
+ .no_brotli()
+ .no_deflate()
+ .no_zstd()
// 分段超时。**没有整体超时** —— 一个跑了六分钟的
// Opus 任务不该被中间层掐断,让客户端自己决定何时放弃。
.connect_timeout(std::time::Duration::from_secs(10))
diff --git a/crates/tw-gateway/src/web_tools/cache.rs b/crates/tw-gateway/src/web_tools/cache.rs
new file mode 100644
index 00000000..440407cf
--- /dev/null
+++ b/crates/tw-gateway/src/web_tools/cache.rs
@@ -0,0 +1,89 @@
+//! Bounded, caller-keyed cache slots. Holding a slot coalesces concurrent work;
+//! eviction only removes one idle slot and cannot split an in-flight request.
+
+use std::collections::HashMap;
+use std::sync::{Arc, Mutex};
+use std::time::Instant;
+use tokio::sync::Mutex as AsyncMutex;
+use tw_types::Msg;
+
+type Slot = Arc>>;
+struct Entry {
+ used: Instant,
+ slot: Slot,
+}
+
+pub(super) struct Cache {
+ capacity: usize,
+ entries: Mutex>>,
+}
+
+impl Cache {
+ pub fn new(capacity: usize) -> Self {
+ Self {
+ capacity,
+ entries: Mutex::new(HashMap::new()),
+ }
+ }
+
+ pub fn get(&self, key: String) -> Result, Msg> {
+ let mut entries = self.entries.lock().unwrap_or_else(|e| e.into_inner());
+ if let Some(entry) = entries.get_mut(&key) {
+ entry.used = Instant::now();
+ return Ok(entry.slot.clone());
+ }
+ if entries.len() >= self.capacity {
+ let oldest = entries
+ .iter()
+ .filter(|(_, e)| Arc::strong_count(&e.slot) == 1)
+ .min_by_key(|(_, e)| e.used)
+ .map(|(key, _)| key.clone());
+ if let Some(oldest) = oldest {
+ entries.remove(&oldest);
+ } else {
+ return Err(super::error("too many pending web operations"));
+ }
+ }
+ let slot = Arc::new(AsyncMutex::new(None));
+ entries.insert(
+ key,
+ Entry {
+ used: Instant::now(),
+ slot: slot.clone(),
+ },
+ );
+ Ok(slot)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[tokio::test]
+ async fn eviction_retains_hot_values_and_coalesces_in_flight_work() {
+ let cache = Cache::new(2);
+ let first = cache.get("first".into()).unwrap();
+ *first.lock().await = Some((Instant::now(), "saved"));
+ let pending = cache.get("pending".into()).unwrap();
+ let guard = pending.lock().await;
+ let joined = cache.get("pending".into()).unwrap();
+ assert!(Arc::ptr_eq(&pending, &joined));
+ assert!(joined.try_lock().is_err());
+ assert!(
+ cache.get("new".into()).is_err(),
+ "active work cannot be evicted"
+ );
+ drop(guard);
+ drop(joined);
+ drop(pending);
+ drop(first);
+ // Touch the completed result: inserting another key must evict the older
+ // idle slot, not flush all successfully cached results.
+ drop(cache.get("first".into()).unwrap());
+ drop(cache.get("new".into()).unwrap());
+ let first = cache.get("first".into()).unwrap();
+ assert_eq!(first.lock().await.as_ref().unwrap().1, "saved");
+ assert_eq!(cache.entries.lock().unwrap().len(), 2);
+ }
+}
diff --git a/crates/tw-gateway/src/web_tools/collect.rs b/crates/tw-gateway/src/web_tools/collect.rs
index 561a8593..b1819ac5 100644
--- a/crates/tw-gateway/src/web_tools/collect.rs
+++ b/crates/tw-gateway/src/web_tools/collect.rs
@@ -11,18 +11,12 @@ fn clean(s: &str) -> String {
s.split_whitespace().collect::>().join(" ")
}
-pub(super) fn search(engine: SearchEngine, body: &str) -> Result, Msg> {
- if body.contains("anomaly.js") || body.contains("challenge-form") || body.contains("b_captcha")
- {
- return Err(error("search engine requires a verification challenge"));
- }
- match engine {
- SearchEngine::Bing => rss(body),
- SearchEngine::Duckduckgo => duckduckgo(body),
- }
-}
-
-fn source(engine: SearchEngine, url: &str, title: &str, text: &str) -> Option {
+pub(super) fn source(
+ engine: SearchEngine,
+ url: &str,
+ title: &str,
+ text: &str,
+) -> Option {
let mut url = super::public::parse(url).ok()?;
url.set_fragment(None);
let params: Vec<_> = url
@@ -44,126 +38,17 @@ fn source(engine: SearchEngine, url: &str, title: &str, text: &str) -> Option Result, Msg> {
- use quick_xml::events::Event;
- let mut reader = quick_xml::Reader::from_str(body);
- reader.config_mut().trim_text(false);
- let mut items = Vec::new();
- let mut in_item = false;
- let mut rss = false;
- let (mut tag, mut title, mut url, mut text, mut date) = (
- Vec::new(),
- String::new(),
- String::new(),
- String::new(),
- String::new(),
- );
- loop {
- match reader.read_event() {
- Ok(Event::Start(e)) => {
- tag = e.name().as_ref().as_bytes().to_vec();
- if tag == b"rss" {
- rss = true;
- }
- if tag == b"item" {
- in_item = true;
- title.clear();
- url.clear();
- text.clear();
- date.clear();
- }
- }
- Ok(event @ (Event::Text(_) | Event::CData(_) | Event::GeneralRef(_))) if in_item => {
- let value = match &event {
- Event::Text(e) => e.as_ref().to_string(),
- Event::CData(e) => e.as_ref().to_string(),
- Event::GeneralRef(e) => {
- quick_xml::escape::unescape(&format!("&{};", e.as_ref()))
- .map_err(|_| error("invalid search XML entity"))?
- .into_owned()
- }
- _ => unreachable!(),
- };
- match tag.as_slice() {
- b"title" => title.push_str(&value),
- b"link" => url.push_str(&value),
- b"description" => text.push_str(&value),
- b"pubDate" => date.push_str(&value),
- _ => {}
- }
- }
- Ok(Event::End(e)) if e.name().as_ref() == "item" => {
- if let Some(mut s) = source(
- SearchEngine::Bing,
- &url,
- &title,
- &clean(
- &Html::parse_fragment(&text)
- .root_element()
- .text()
- .collect::>()
- .join(" "),
- ),
- ) {
- if !date.is_empty() {
- s.published_at = Some(date.clone());
- }
- items.push(s);
- }
- in_item = false;
- }
- Ok(Event::Eof) => break,
- Err(_) => return Err(error("invalid search XML")),
- _ => {}
- }
- }
- if !rss {
- return Err(error("search engine did not return RSS results"));
- }
- Ok(items)
-}
-
-fn duckduckgo(body: &str) -> Result, Msg> {
- let doc = Html::parse_document(body);
- let mut result = Vec::new();
- for row in doc.select(&selector(".result")) {
- let Some(link) = row.select(&selector(".result__a")).next() else {
- continue;
- };
- let Some(href) = link.value().attr("href") else {
- continue;
- };
- let absolute = if href.starts_with("//") {
- format!("https:{href}")
- } else {
- href.to_string()
- };
- let Ok(url) = reqwest::Url::parse(&absolute) else {
- continue;
- };
- let destination = url
- .query_pairs()
- .find(|(k, _)| k == "uddg")
- .map(|(_, v)| v.into_owned())
- .unwrap_or(absolute);
- let title = link.text().collect::>().join(" ");
- let text = row
- .select(&selector(".result__snippet"))
- .next()
- .map(|n| n.text().collect::>().join(" "))
- .unwrap_or_default();
- if let Some(s) = source(SearchEngine::Duckduckgo, &destination, &title, &text) {
- result.push(s);
- }
- }
- if result.is_empty() && !body.contains("no-results") {
- return Err(error("search page layout was not recognized"));
- }
- Ok(result)
-}
-
-pub(super) fn page(body: &str) -> (String, String) {
+pub(super) fn page(body: &str) -> Result<(String, String), Msg> {
let mut doc = Html::parse_document(body);
+ if doc
+ .select(&selector(
+ "#challenge-form,#cf-challenge-running,#challenge-running",
+ ))
+ .next()
+ .is_some()
+ {
+ return Err(error("page requires a verification challenge"));
+ }
let remove: Vec<_> = doc
.select(&selector(
"script,style,noscript,nav,footer,header,aside,form,svg,template",
@@ -181,16 +66,66 @@ pub(super) fn page(body: &str) -> (String, String) {
.map(|n| clean(&n.text().collect::>().join(" ")))
.unwrap_or_default();
let root = doc
- .select(&selector("article,main,[role=main]"))
+ .select(&selector("main,[role=main]"))
.next()
+ .or_else(|| doc.select(&selector("article")).next())
+ .or_else(|| doc.select(&selector("body")).next())
.unwrap_or_else(|| doc.root_element());
- let text = root
- .text()
+ // Text nodes are not paragraphs: inline , and elements must
+ // not split a sentence into unrelated lines. Use an explicit stack so a
+ // deeply nested public page cannot overflow the call stack.
+ let mut text = String::new();
+ let mut stack: Vec<_> = root.children().rev().map(|node| (node, false)).collect();
+ while let Some((node, closing)) = stack.pop() {
+ if let Some(element) = node.value().as_element() {
+ let block = matches!(
+ element.name(),
+ "p" | "div"
+ | "section"
+ | "article"
+ | "main"
+ | "li"
+ | "ul"
+ | "ol"
+ | "pre"
+ | "blockquote"
+ | "h1"
+ | "h2"
+ | "h3"
+ | "h4"
+ | "h5"
+ | "h6"
+ | "br"
+ | "tr"
+ | "dt"
+ | "dd"
+ );
+ if block && !text.ends_with('\n') {
+ text.push('\n');
+ }
+ if closing && matches!(element.name(), "td" | "th") {
+ text.push(' ');
+ }
+ if !closing {
+ stack.push((node, true));
+ stack.extend(node.children().rev().map(|child| (child, false)));
+ }
+ } else if let Some(value) = node.value().as_text() {
+ text.push_str(value);
+ }
+ }
+ let text = text
+ .lines()
.map(clean)
.filter(|s| !s.is_empty())
.collect::>()
.join("\n");
- (title.chars().take(500).collect(), text)
+ if text.is_empty() {
+ return Err(error(
+ "page contained no readable text; it may require JavaScript",
+ ));
+ }
+ Ok((title.chars().take(500).collect(), text))
}
pub(super) fn excerpt(text: &str, query: &str, limit: usize) -> String {
@@ -199,19 +134,20 @@ pub(super) fn excerpt(text: &str, query: &str, limit: usize) -> String {
.filter(|s| s.chars().count() > 2)
.map(str::to_lowercase)
.collect();
- let mut lines: Vec<_> = text.lines().enumerate().collect();
- lines.sort_by_key(|(i, line)| {
+ let lines: Vec<_> = text.lines().collect();
+ let score = |line: &str| {
let l = line.to_lowercase();
- (
- std::cmp::Reverse(words.iter().filter(|w| l.contains(w.as_str())).count()),
- *i,
- )
- });
- lines
+ words.iter().filter(|w| l.contains(w.as_str())).count()
+ };
+ let best = lines
.iter()
- .take(12)
- .map(|(_, s)| *s)
- .collect::>()
+ .enumerate()
+ .max_by_key(|(i, line)| (score(line), std::cmp::Reverse(*i)))
+ .map(|(i, _)| i)
+ .unwrap_or(0);
+ // Preserve source order and surrounding context instead of sorting sentences
+ // by keyword count, which can change what the page actually says.
+ lines[best.saturating_sub(1)..lines.len().min(best + 11)]
.join("\n")
.chars()
.take(limit)
@@ -244,38 +180,52 @@ pub(super) fn find(text: &str, pattern: &str, limit: usize) -> String {
mod tests {
use super::*;
#[test]
- fn parses_search_sources_and_rejects_challenges() {
- let r = search(SearchEngine::Bing, "- Releasehttps://example.org/?utm_source=xVersion 3
").unwrap();
- assert_eq!(r[0].url, "https://example.org/");
- let html = r#""#;
- assert_eq!(
- search(SearchEngine::Duckduckgo, html).unwrap()[0].text,
- "Version 3"
- );
- assert!(search(SearchEngine::Duckduckgo, "
]]>"#).unwrap();
- assert_eq!(result[0].title, "Rust & Web 中文");
- assert_eq!(result[0].url, "https://example.org/?a=1&b=2");
- assert_eq!(result[0].text, "A & B");
+ fn inline_markup_does_not_break_sentences_or_phrase_searches() {
+ let (_, text) = page(
+ "DocUse asyncio.TaskGroup to await all tasks.
Errors cancel the remaining tasks.
",
+ ).unwrap();
+ assert_eq!(
+ text,
+ "Use asyncio.TaskGroup to await all tasks.\nErrors cancel the remaining tasks."
+ );
+ assert!(find(&text, "await all tasks", 1000).contains("Use asyncio.TaskGroup"));
+ }
+
+ #[test]
+ fn excerpts_keep_context_in_the_original_order() {
+ let text = "Introduction\nDo not use this on older versions.\nTaskGroup first appeared in Python 3.11.\nIn earlier versions, use gather.\nTaskGroup handles child tasks together.";
+ let extracted = excerpt(text, "TaskGroup Python", 1000);
+ assert!(extracted.starts_with("Do not use this on older versions."));
+ assert!(
+ extracted.find("first appeared").unwrap()
+ < extracted.find("In earlier versions").unwrap()
+ );
+ assert!(
+ extracted.find("In earlier versions").unwrap()
+ < extracted.find("handles child").unwrap()
+ );
+ assert_eq!(excerpt("", "anything", 100), "");
+ }
+
+ #[test]
+ fn empty_pages_and_verification_are_not_returned_as_content() {
+ assert!(page("").is_err());
+ assert!(page("").is_err());
+ // Cloudflare also injects scripts into successfully loaded normal pages.
+ assert!(page("Article text").is_ok());
+ let (_, text) = page("Advertisement").unwrap();
+ assert_eq!(text, "Version 3.11");
}
}
diff --git a/crates/tw-gateway/src/web_tools/mod.rs b/crates/tw-gateway/src/web_tools/mod.rs
index 379545e2..2430ba72 100644
--- a/crates/tw-gateway/src/web_tools/mod.rs
+++ b/crates/tw-gateway/src/web_tools/mod.rs
@@ -1,38 +1,107 @@
//! Search and public-page tools, shared by the gateway and the control-plane tester.
+mod cache;
mod collect;
mod history;
-mod locale;
pub(crate) mod protocol;
+mod providers;
mod public;
+mod quality;
+mod session;
pub(crate) mod wire;
use std::collections::HashMap;
-use std::sync::{Arc, Mutex};
+use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::sync::{Mutex as AsyncMutex, Semaphore};
use tw_api::{SearchEngine, WebToolRequest, WebToolResult, WebToolsConfig};
use tw_types::{Msg, msg};
-type PageCached = Arc>>;
-
-type Cached = Arc>>;
-
pub struct WebTools {
pub(crate) history: Arc,
config: WebToolsConfig,
client: public::ClientSettings,
slots: Semaphore,
- cache: Mutex>,
- pages: Mutex>,
+ cache: cache::Cache,
+ pages: cache::Cache<(String, String)>,
engines: AsyncMutex>,
+ sessions: session::Pool,
}
#[derive(Default)]
struct EngineState {
next: Option,
failed_until: Option,
+ last_failure: Option,
+}
+
+/// Capabilities come from the same registry that executes requests. Desktop
+/// clients can offer new engines without hard-coding an exhaustive enum list.
+pub fn available_engines() -> Vec {
+ providers::AUTOMATIC
+ .iter()
+ .map(|&engine| {
+ let spec = providers::get(engine).spec();
+ tw_api::WebSearchEngine {
+ id: engine,
+ name: spec.name.into(),
+ min_interval_ms: spec.min_interval_ms,
+ }
+ })
+ .collect()
+}
+
+fn engine_name(engine: SearchEngine) -> &'static str {
+ providers::get(engine).spec().name
+}
+
+fn failure_detail(e: &Msg) -> &str {
+ if e.code == "gw.web_tools.failed" {
+ e.args.get("detail").map(String::as_str).unwrap_or(&e.text)
+ } else {
+ &e.text
+ }
+}
+
+fn http_failure(status: reqwest::StatusCode, headers: &http::HeaderMap) -> Msg {
+ let mut failure = error(format!("source returned HTTP {}", status.as_u16()));
+ failure
+ .args
+ .insert("http_status".into(), status.as_u16().to_string());
+ if matches!(status.as_u16(), 429 | 503)
+ && let Some(value) = headers.get("retry-after").and_then(|v| v.to_str().ok())
+ {
+ let seconds = value.trim().parse::().ok().or_else(|| {
+ chrono::DateTime::parse_from_rfc2822(value)
+ .ok()
+ .map(|at| (at.timestamp() - chrono::Utc::now().timestamp()).max(0) as u64)
+ });
+ if let Some(seconds) = seconds {
+ failure
+ .args
+ .insert("retry_after_secs".into(), seconds.to_string());
+ }
+ }
+ failure
+}
+
+impl EngineState {
+ fn cooling_down(&self, engine: SearchEngine) -> Msg {
+ let seconds = self
+ .failed_until
+ .map(|until| until.saturating_duration_since(Instant::now()).as_secs() + 1)
+ .unwrap_or(0);
+ let cause = self
+ .last_failure
+ .as_ref()
+ .map(failure_detail)
+ .unwrap_or("previous search failed");
+ error(format!(
+ "{} is cooling down; retry in {seconds}s. Last failure: {cause}",
+ engine_name(engine)
+ ))
+ }
}
pub(crate) fn error(detail: impl Into) -> Msg {
@@ -113,10 +182,7 @@ impl WebTools {
result: WebToolResult,
) {
let key = Self::cache_key(identity, request, &self.locale()).unwrap();
- self.cache.lock().unwrap().insert(
- key,
- Arc::new(AsyncMutex::new(Some((Instant::now(), result)))),
- );
+ *self.cache.get(key).unwrap().lock().await = Some((Instant::now(), result));
}
pub fn new(config: &tw_config::Config) -> Result {
Ok(Self {
@@ -124,9 +190,10 @@ impl WebTools {
config: config.web_tools.clone(),
client: public::ClientSettings::new(config)?,
slots: Semaphore::new(config.web_tools.max_concurrent as usize),
- cache: Mutex::new(HashMap::new()),
- pages: Mutex::new(HashMap::new()),
+ cache: cache::Cache::new(256),
+ pages: cache::Cache::new(32),
engines: AsyncMutex::new(HashMap::new()),
+ sessions: session::Pool::default(),
})
}
@@ -155,19 +222,7 @@ impl WebTools {
} else {
format!("{key}\n{}", serde_json::json!([allowed, blocked]))
};
- let entry = {
- let mut cache = self.cache.lock().unwrap_or_else(|e| e.into_inner());
- if cache.len() >= 256 {
- cache.retain(|_, e| Arc::strong_count(e) > 1);
- }
- if cache.len() >= 256 && !cache.contains_key(&key) {
- return Err(error("too many pending web operations"));
- }
- cache
- .entry(key)
- .or_insert_with(|| Arc::new(AsyncMutex::new(None)))
- .clone()
- };
+ let entry = self.cache.get(key)?;
// Holding this lock coalesces concurrent requests without leaving a background
// operation behind when the last caller is cancelled.
let mut cached = entry.lock().await;
@@ -229,7 +284,7 @@ impl WebTools {
public::validate_domains(allowed_domains)?;
public::validate_domains(blocked_domains)?;
let engines = if self.config.engines.is_empty() {
- vec![SearchEngine::Bing, SearchEngine::Duckduckgo]
+ providers::AUTOMATIC.to_vec()
} else {
self.config.engines.clone()
};
@@ -243,49 +298,102 @@ impl WebTools {
.saturating_sub(Duration::from_millis(20));
let budget = remaining
/ (engine_count - index + usize::from(self.config.read_pages > 0)) as u32;
+ let mut searching = false;
let attempt = tokio::time::timeout(budget, async {
if !self.reserve(engine).await {
return Ok(None);
}
- self.search(engine, query, allowed_domains, blocked_domains, locale)
- .await
- .map(Some)
+ searching = true;
+ self.search(
+ identity,
+ engine,
+ query,
+ allowed_domains,
+ blocked_domains,
+ locale,
+ )
+ .await
+ .map(Some)
})
.await
- .unwrap_or_else(|_| Err(error("search engine timed out")));
+ .unwrap_or_else(|_| {
+ Err(error(if searching {
+ "search engine timed out"
+ } else {
+ "search rate-limit wait timed out"
+ }))
+ });
match attempt {
- Ok(None) => continue,
+ Ok(None) => {
+ let engines = self.engines.lock().await;
+ if let Some(state) = engines.get(&engine) {
+ result.warnings.push(state.cooling_down(engine));
+ }
+ }
Ok(Some(sources)) => {
+ if let Some(state) = self.engines.lock().await.get_mut(&engine) {
+ state.failed_until = None;
+ state.last_failure = None;
+ }
succeeded = true;
for source in sources {
if !result.sources.iter().any(|s| s.url == source.url) {
result.sources.push(source);
}
}
- if result.sources.len() >= self.config.max_results as usize {
+ // Engine order is failover order. Filling a healthy
+ // engine's shorter result set from a worse fallback
+ // can reintroduce unrelated pages.
+ if !result.sources.is_empty() {
break;
}
}
Err(e) => {
- self.engines
- .lock()
- .await
- .entry(engine)
- .or_default()
- .failed_until = Some(Instant::now() + Duration::from_secs(60));
+ let query_mismatch = e.code == "gw.web_tools.query_mismatch";
+ let cooldown = e
+ .args
+ .get("retry_after_secs")
+ .and_then(|v| v.parse::().ok())
+ .and_then(|seconds| {
+ Instant::now().checked_add(Duration::from_secs(seconds.max(1)))
+ })
+ .unwrap_or_else(|| Instant::now() + Duration::from_secs(60));
+ let e =
+ error(format!("{}: {}", engine_name(engine), failure_detail(&e)));
+ // A poor match for one query does not make the engine
+ // unavailable for every other caller for a minute.
+ if searching && !query_mismatch {
+ let mut engines = self.engines.lock().await;
+ let state = engines.entry(engine).or_default();
+ state.failed_until = Some(cooldown);
+ state.last_failure = Some(e.clone());
+ }
result.warnings.push(e);
}
}
}
if !succeeded {
- return Err(error(
- "all configured search engines failed or are cooling down",
- ));
+ // Do not discard the engine errors: a challenge, a broken proxy,
+ // and a timeout need different remedies. Retries during cooldown
+ // retain the original cause and say when another attempt is due.
+ let reasons = result
+ .warnings
+ .iter()
+ .map(failure_detail)
+ .collect::>()
+ .join("; ");
+ return Err(error(format!(
+ "all configured search engines failed or are cooling down. {reasons}"
+ )));
}
result.sources.truncate(self.config.max_results as usize);
let mut read = 0;
for source in &mut result.sources {
- if read >= self.config.read_pages || source.text.chars().count() >= 300 {
+ if read >= self.config.read_pages
+ || source.text.chars().count() >= 300
+ || reqwest::Url::parse(&source.url)
+ .is_ok_and(|url| url.path().to_ascii_lowercase().ends_with(".pdf"))
+ {
continue;
}
read += 1;
@@ -303,7 +411,19 @@ impl WebTools {
.await
.unwrap_or_else(|_| Err(error("optional page reading timed out")))
{
- Ok((_, text)) => source.text = collect::excerpt(&text, query, 3000),
+ Ok((_, text)) => {
+ let excerpt = collect::excerpt(&text, query, 3000);
+ if !excerpt.is_empty() && !source.text.contains(&excerpt) {
+ // The engine snippet may be more focused than the
+ // landing page. Keep it instead of replacing it
+ // with navigation text or unrelated paragraphs.
+ source.text = format!("{}\n\n{excerpt}", source.text)
+ .trim()
+ .chars()
+ .take(4000)
+ .collect();
+ }
+ }
Err(e) => result.warnings.push(e),
}
}
@@ -344,62 +464,101 @@ impl WebTools {
}
async fn reserve(&self, engine: SearchEngine) -> bool {
- let delay = {
- let mut engines = self.engines.lock().await;
- let state = engines.entry(engine).or_default();
- let now = Instant::now();
- if state.failed_until.is_some_and(|until| until > now) {
- return false;
- }
- let at = state.next.unwrap_or(now).max(now);
- state.next = Some(at + Duration::from_millis(u64::from(self.config.interval_ms)));
- at.saturating_duration_since(now)
- };
- tokio::time::sleep(delay).await;
- true
+ loop {
+ let delay = {
+ let mut engines = self.engines.lock().await;
+ let state = engines.entry(engine).or_default();
+ let now = Instant::now();
+ if state.failed_until.is_some_and(|until| until > now) {
+ return false;
+ }
+ let at = state.next.unwrap_or(now);
+ if at <= now {
+ state.next = Some(
+ now + Duration::from_millis(u64::from(
+ self.config
+ .interval_ms
+ .max(providers::get(engine).spec().min_interval_ms),
+ )),
+ );
+ return true;
+ }
+ at.saturating_duration_since(now)
+ };
+ // Reserve only when a request can actually start. Cancelled waiters
+ // must not accumulate future slots and delay unrelated searches.
+ tokio::time::sleep(delay).await;
+ }
}
async fn search(
&self,
+ identity: &str,
engine: SearchEngine,
query: &str,
allowed: &[String],
blocked: &[String],
locale: &tw_api::WebSearchLocale,
) -> Result, Msg> {
- let mut q = query.to_string();
- if !allowed.is_empty() {
- q.push_str(&format!(
- " ({})",
- allowed
- .iter()
- .map(|d| format!("site:{d}"))
- .collect::>()
- .join(" OR ")
- ));
- } else if !self.config.allowed_domains.is_empty() {
- q.push_str(&format!(
- " ({})",
- self.config
- .allowed_domains
- .iter()
- .map(|d| format!("site:{d}"))
- .collect::>()
- .join(" OR ")
- ));
+ let provider = providers::get(engine);
+ let spec = provider.spec();
+ let session = self.sessions.get(identity, spec, locale)?;
+ let context = session::Request {
+ spec,
+ kind: providers::Kind::Document,
+ session: session.as_ref().map(|s| &s.session),
+ locale,
+ };
+ if let Some(session) = &session {
+ session
+ .session
+ .ready
+ .get_or_try_init(|| async {
+ let home = spec
+ .home
+ .expect("session requires a homepage")
+ .parse()
+ .unwrap();
+ let (_, body) = self.fetch(home, None, Some(context), None).await?;
+ provider.validate_home(&body)
+ })
+ .await?;
}
- let mut url = reqwest::Url::parse(match engine {
- SearchEngine::Bing => "https://www.bing.com/search",
- SearchEngine::Duckduckgo => "https://html.duckduckgo.com/html/",
+ let query_context = providers::Query {
+ text: query.trim(),
+ allowed: if allowed.is_empty() {
+ &self.config.allowed_domains
+ } else {
+ allowed
+ },
+ locale,
+ };
+ let mut requests = 0;
+ let mut sources = providers::run(provider, &query_context, |request| {
+ let retry = requests > 0;
+ requests += 1;
+ async move {
+ // The first reservation is made by the scheduler. Further exchange
+ // steps with a source minimum must obey that same engine-wide pace.
+ if retry && spec.min_interval_ms > 0 && !self.reserve(engine).await {
+ return Err(error("search engine entered cooldown"));
+ }
+ let context = session::Request {
+ kind: request.kind,
+ ..context
+ };
+ self.fetch(request.url, None, Some(context), request.form.as_deref())
+ .await
+ .map(|(_, body)| body)
+ }
})
- .expect("built-in URL");
- url.query_pairs_mut().append_pair("q", &q);
- if engine == SearchEngine::Bing {
- url.query_pairs_mut().append_pair("format", "rss");
+ .await?;
+ let had_results = !sources.is_empty();
+ let usable = quality::filter(query, &mut sources);
+ if had_results && !usable {
+ return Err(msg!("gw.web_tools.query_mismatch" =>
+ "The search engine returned results that did not match the query."));
}
- locale::apply(&mut url, engine, locale);
- let (_, body) = self.fetch(url, None).await?;
- let mut sources = collect::search(engine, &body)?;
sources.retain(|s| {
reqwest::Url::parse(&s.url).ok().is_some_and(|url| {
self.allowed(&url) && public::domains_allow(&url, allowed, blocked)
@@ -427,19 +586,7 @@ impl WebTools {
"{identity}\n{url}\n{}",
serde_json::json!([allowed, blocked])
);
- let entry = {
- let mut pages = self.pages.lock().unwrap_or_else(|e| e.into_inner());
- if pages.len() >= 32 {
- pages.retain(|_, p| Arc::strong_count(p) > 1);
- }
- if pages.len() >= 32 && !pages.contains_key(&key) {
- return Err(error("too many pending page reads"));
- }
- pages
- .entry(key)
- .or_insert_with(|| Arc::new(AsyncMutex::new(None)))
- .clone()
- };
+ let entry = self.pages.get(key)?;
let mut cache = entry.lock().await;
if let Some((at, page)) = &*cache
&& at.elapsed() < Duration::from_secs(u64::from(self.config.cache_secs))
@@ -463,9 +610,11 @@ impl WebTools {
if !self.allowed(&url) {
return Err(error("page is excluded by the configured domain rules"));
}
- let (kind, body) = self.fetch(url, Some((allowed, blocked))).await?;
+ let (kind, body) = self
+ .fetch(url, Some((allowed, blocked)), None, None)
+ .await?;
if kind.contains("html") {
- Ok(collect::page(&body))
+ collect::page(&body)
} else if kind.starts_with("text/") {
Ok((String::new(), body))
} else {
@@ -477,17 +626,36 @@ impl WebTools {
&self,
mut url: reqwest::Url,
filters: Option<(&[String], &[String])>,
+ search: Option>,
+ form: Option<&str>,
) -> Result<(String, String), Msg> {
use futures::StreamExt;
for _ in 0..6 {
- let addresses = public::check(&url).await?;
if let Some((allowed, blocked)) = filters
&& (!self.allowed(&url) || !public::domains_allow(&url, allowed, blocked))
{
return Err(error("redirect is excluded by the configured domain rules"));
}
- let response = self.client.get(&url, &addresses).await?;
+ let response = match search {
+ Some(context) => {
+ let response = self
+ .client
+ .get_search(context.spec, &url, context.headers(&url)?, form)
+ .await?;
+ context.remember(&url, response.headers())?;
+ response
+ }
+ None => {
+ let addresses = public::check(&url).await?;
+ self.client.get(&url, &addresses).await?
+ }
+ };
if response.status().is_redirection() {
+ // Form queries never follow redirects: do not replay their body
+ // into a login, verification, or different document endpoint.
+ if form.is_some() {
+ return Err(http_failure(response.status(), response.headers()));
+ }
let to = response
.headers()
.get("location")
@@ -501,10 +669,7 @@ impl WebTools {
continue;
}
if !response.status().is_success() {
- return Err(error(format!(
- "source returned HTTP {}",
- response.status().as_u16()
- )));
+ return Err(http_failure(response.status(), response.headers()));
}
let kind = response
.headers()
@@ -512,6 +677,10 @@ impl WebTools {
.and_then(|v| v.to_str().ok())
.unwrap_or("application/octet-stream")
.to_lowercase();
+ // Reject binary documents before trying to decode them as UTF-8.
+ if search.is_none() && !kind.contains("html") && !kind.starts_with("text/") {
+ return Err(error("page must be HTML or plain text"));
+ }
let mut bytes = Vec::new();
let mut stream = response.bytes_stream();
while let Some(chunk) = stream.next().await {
@@ -532,11 +701,358 @@ impl WebTools {
#[cfg(test)]
mod tests {
use super::*;
+
+ #[test]
+ fn rate_limits_keep_server_retry_after_seconds_and_dates() {
+ let mut headers = http::HeaderMap::new();
+ headers.insert("retry-after", "120".parse().unwrap());
+ assert_eq!(
+ http_failure(reqwest::StatusCode::TOO_MANY_REQUESTS, &headers).arg("retry_after_secs"),
+ "120"
+ );
+ let future = chrono::Utc::now() + chrono::Duration::minutes(2);
+ headers.insert("retry-after", future.to_rfc2822().parse().unwrap());
+ let seconds: u64 = http_failure(reqwest::StatusCode::SERVICE_UNAVAILABLE, &headers)
+ .arg("retry_after_secs")
+ .parse()
+ .unwrap();
+ assert!((118..=120).contains(&seconds));
+ headers.insert("retry-after", "invalid".parse().unwrap());
+ assert!(
+ http_failure(reqwest::StatusCode::TOO_MANY_REQUESTS, &headers)
+ .arg("retry_after_secs")
+ .is_empty()
+ );
+ }
+
+ #[tokio::test]
+ #[ignore = "live traditional-engine relevance, filters, and concurrent cache checks"]
+ async fn live_direct_search_matrix() {
+ live_engine_matrix(&[SearchEngine::Duckduckgo]).await;
+ }
+
+ #[tokio::test]
+ #[ignore = "live Bing relevance with a configured Chinese locale, filters, and cache"]
+ async fn live_bing_search_matrix() {
+ live_engine_matrix(&[SearchEngine::Bing]).await;
+ }
+
+ #[tokio::test]
+ #[ignore = "live Bing-first search with DuckDuckGo failover, filters, and cache"]
+ async fn live_bing_failover_matrix() {
+ live_engine_matrix(&[SearchEngine::Bing, SearchEngine::Duckduckgo]).await;
+ }
+
+ #[tokio::test]
+ #[ignore = "live Google Docs relevance, domain filters, and concurrent cache checks"]
+ async fn live_google_docs_search_matrix() {
+ live_engine_matrix(&[SearchEngine::GoogleDocs]).await;
+ }
+
+ #[tokio::test]
+ #[ignore = "live automatic search with Google Docs and traditional-engine failover"]
+ async fn live_automatic_search_matrix() {
+ live_engine_matrix(providers::AUTOMATIC).await;
+ }
+
+ async fn live_engine_matrix(engines: &[SearchEngine]) {
+ let mut config = tw_config::Config::default();
+ config.web_tools.engines = engines.to_vec();
+ config.web_tools.language = "zh-Hans".into();
+ config.web_tools.region = "CN".into();
+ config.web_tools.read_pages = 0;
+ let tools = WebTools::new(&config).unwrap();
+ let request = |query: &str| WebToolRequest::Search {
+ query: query.into(),
+ allowed_domains: vec![],
+ blocked_domains: vec![],
+ };
+ let mut failures = Vec::new();
+ for (query, domain, path) in [
+ (
+ "Python asyncio TaskGroup documentation",
+ "python.org",
+ "asyncio-task",
+ ),
+ ("Rust OnceLock documentation", "rust-lang.org", "OnceLock"),
+ ("SQLite release notes", "sqlite.org", "changes"),
+ ("劳动合同法 第三十八条", "gov.cn", ""),
+ (
+ "2025 Nobel physics prize winners",
+ "nobelprize.org",
+ "physics/2025",
+ ),
+ (
+ "Apple Mac mini M4 technical specifications",
+ "apple.com",
+ "121555",
+ ),
+ ("中华人民共和国民法典 第一千零六十二条", "gov.cn", ""),
+ ("OpenAI Responses API reference", "openai.com", "responses"),
+ ] {
+ let result = match tools.execute("matrix", &request(query)).await {
+ Ok(result) => result,
+ Err(error) => {
+ failures.push(format!("{query}: {error}"));
+ continue;
+ }
+ };
+ eprintln!(
+ "{query}: {} sources, {} ms, {:?}, {:?}",
+ result.sources.len(),
+ result.elapsed_ms,
+ result.sources.first().and_then(|s| s.engine),
+ result
+ .sources
+ .iter()
+ .take(3)
+ .map(|s| &s.url)
+ .collect::>()
+ );
+ if engines.len() == 1 {
+ assert!(result.warnings.is_empty(), "{query}: {:?}", result.warnings);
+ }
+ if !result.sources.iter().any(|source| {
+ reqwest::Url::parse(&source.url).is_ok_and(|url| {
+ public::domains_allow(&url, &[domain.into()], &[]) && url.path().contains(path)
+ })
+ }) {
+ failures.push(format!("{query}: missing {domain}/{path}"));
+ }
+ assert!(
+ result
+ .sources
+ .iter()
+ .all(|s| s.engine.is_some_and(|e| engines.contains(&e)))
+ );
+ }
+ assert!(
+ failures.is_empty(),
+ "{engines:?} failed relevance checks: {failures:?}"
+ );
+ let req = request("PostgreSQL transaction isolation documentation");
+ let results =
+ futures::future::join_all((0..4).map(|_| tools.execute("matrix", &req))).await;
+ let results: Vec<_> = results.into_iter().collect::>().unwrap();
+ assert_eq!(results.iter().filter(|r| r.cached).count(), 3);
+ assert!(results.iter().all(|r| !r.sources.is_empty()));
+ for (allowed_domains, blocked_domains) in [
+ (vec!["sqlite.org".into()], vec![]),
+ (vec![], vec!["sqlite.org".into()]),
+ ] {
+ let result = tools
+ .execute(
+ "matrix",
+ &WebToolRequest::Search {
+ query: "SQLite release notes".into(),
+ allowed_domains: allowed_domains.clone(),
+ blocked_domains: blocked_domains.clone(),
+ },
+ )
+ .await
+ .unwrap();
+ assert!(!result.sources.is_empty());
+ assert!(result.sources.iter().all(|s| public::domains_allow(
+ &reqwest::Url::parse(&s.url).unwrap(),
+ &allowed_domains,
+ &blocked_domains
+ )));
+ }
+ }
+
+ #[tokio::test]
+ async fn source_minimum_interval_and_capabilities_share_the_registry() {
+ let mut config = tw_config::Config::default();
+ config.web_tools.interval_ms = 250;
+ let tools = WebTools::new(&config).unwrap();
+ assert!(tools.reserve(SearchEngine::GoogleDocs).await);
+ let next = tools.engines.lock().await[&SearchEngine::GoogleDocs]
+ .next
+ .unwrap();
+ assert!(next.saturating_duration_since(Instant::now()) > Duration::from_millis(5900));
+ assert!(
+ tokio::time::timeout(
+ Duration::from_millis(10),
+ tools.reserve(SearchEngine::GoogleDocs)
+ )
+ .await
+ .is_err()
+ );
+ assert_eq!(
+ tools.engines.lock().await[&SearchEngine::GoogleDocs].next,
+ Some(next)
+ );
+ let available = available_engines();
+ assert_eq!(available[0].id, SearchEngine::GoogleDocs);
+ assert_eq!(available[0].min_interval_ms, 6000);
+ }
+
+ #[tokio::test]
+ async fn cancelled_waiters_do_not_reserve_more_engine_slots() {
+ let tools = WebTools::new(&tw_config::Config::default()).unwrap();
+ assert!(tools.reserve(SearchEngine::Bing).await);
+ let next = tools.engines.lock().await[&SearchEngine::Bing].next;
+ for _ in 0..3 {
+ assert!(
+ tokio::time::timeout(Duration::from_millis(1), tools.reserve(SearchEngine::Bing))
+ .await
+ .is_err()
+ );
+ assert_eq!(tools.engines.lock().await[&SearchEngine::Bing].next, next);
+ }
+ }
+
+ #[tokio::test]
+ async fn rate_limit_wait_timeout_does_not_mark_the_engine_unavailable() {
+ let mut config = tw_config::Config::default();
+ config.web_tools.timeout_secs = 1;
+ config.web_tools.read_pages = 0;
+ config.web_tools.engines = vec![SearchEngine::Bing];
+ let tools = WebTools::new(&config).unwrap();
+ tools.engines.lock().await.insert(
+ SearchEngine::Bing,
+ EngineState {
+ next: Some(Instant::now() + Duration::from_secs(60)),
+ ..Default::default()
+ },
+ );
+ let failure = tools
+ .execute(
+ "test",
+ &WebToolRequest::Search {
+ query: "SQLite release notes".into(),
+ allowed_domains: vec![],
+ blocked_domains: vec![],
+ },
+ )
+ .await
+ .unwrap_err();
+ assert!(failure.text.contains("rate-limit wait"), "{failure}");
+ assert!(
+ tools.engines.lock().await[&SearchEngine::Bing]
+ .failed_until
+ .is_none()
+ );
+ }
+
+ #[tokio::test]
+ async fn unavailable_engines_keep_their_causes_and_retry_deadline() {
+ let mut config = tw_config::Config::default();
+ config.web_tools.engines = vec![SearchEngine::Bing, SearchEngine::Duckduckgo];
+ let tools = WebTools::new(&config).unwrap();
+ let until = Instant::now() + Duration::from_secs(60);
+ for (engine, reason) in [
+ (SearchEngine::Bing, "source returned HTTP 429"),
+ (
+ SearchEngine::Duckduckgo,
+ "search engine requires a verification challenge",
+ ),
+ ] {
+ tools.engines.lock().await.insert(
+ engine,
+ EngineState {
+ next: None,
+ failed_until: Some(until),
+ last_failure: Some(error(format!("{}: {reason}", engine_name(engine)))),
+ },
+ );
+ }
+ for query in ["SQLite release notes", "Rust documentation"] {
+ let failure = tools
+ .execute(
+ "test",
+ &WebToolRequest::Search {
+ query: query.into(),
+ allowed_domains: vec![],
+ blocked_domains: vec![],
+ },
+ )
+ .await
+ .unwrap_err();
+ assert!(failure.text.contains("Bing"), "{failure}");
+ assert!(failure.text.contains("429"), "{failure}");
+ assert!(failure.text.contains("DuckDuckGo"), "{failure}");
+ assert!(failure.text.contains("verification challenge"), "{failure}");
+ assert!(failure.text.contains("retry in"), "{failure}");
+ assert_eq!(failure.text.matches("Web tools:").count(), 1, "{failure}");
+ for state in tools.engines.lock().await.values() {
+ assert_eq!(
+ state.failed_until,
+ Some(until),
+ "retry must not extend cooldown"
+ );
+ }
+ }
+ }
+
+ #[tokio::test]
+ #[ignore = "uses the public DuckDuckGo search engine"]
+ async fn live_search_duckduckgo() {
+ live_duckduckgo_after_cooldown(false).await;
+ }
+
+ #[tokio::test]
+ #[ignore = "live DuckDuckGo fallback while Google Docs is cooling down"]
+ async fn live_search_google_cooldown_falls_back() {
+ live_duckduckgo_after_cooldown(true).await;
+ }
+
+ async fn live_duckduckgo_after_cooldown(google_failed: bool) {
+ let mut config = tw_config::Config::default();
+ config.web_tools.engines = vec![SearchEngine::Duckduckgo];
+ if google_failed {
+ config.web_tools.engines.insert(0, SearchEngine::GoogleDocs);
+ }
+ config.web_tools.read_pages = 0;
+ let tools = WebTools::new(&config).unwrap();
+ if google_failed {
+ tools.engines.lock().await.insert(
+ SearchEngine::GoogleDocs,
+ EngineState {
+ failed_until: Some(Instant::now() + Duration::from_secs(60)),
+ last_failure: Some(error("Google (Docs): source returned HTTP 500")),
+ ..Default::default()
+ },
+ );
+ }
+ let result = tools
+ .execute(
+ "live-test",
+ &WebToolRequest::Search {
+ query: "SQLite release notes".into(),
+ allowed_domains: vec![],
+ blocked_domains: vec![],
+ },
+ )
+ .await
+ .unwrap();
+ if google_failed {
+ assert!(result.warnings.iter().any(|w| w.text.contains("500")));
+ }
+ assert!(
+ result.sources.iter().any(|s| {
+ reqwest::Url::parse(&s.url).is_ok_and(|url| {
+ url.host_str()
+ .is_some_and(|host| host == "sqlite.org" || host.ends_with(".sqlite.org"))
+ && url.path().contains("changes")
+ })
+ }),
+ "release notes query must find SQLite's release history: {:?}",
+ result.sources
+ );
+ assert!(
+ result
+ .sources
+ .iter()
+ .all(|s| s.engine == Some(SearchEngine::Duckduckgo))
+ );
+ }
+
#[tokio::test]
#[ignore = "uses public search engines and public webpages"]
async fn live_search_read_and_find() {
let mut config = tw_config::Config::default();
- config.web_tools.engines = vec![SearchEngine::Bing];
+ config.web_tools.engines = vec![SearchEngine::Duckduckgo];
config.web_tools.read_pages = 0;
let tools = WebTools::new(&config).unwrap();
let search = tools
@@ -550,7 +1066,17 @@ mod tests {
)
.await
.unwrap();
- assert!(!search.sources.is_empty());
+ assert!(
+ search.sources.iter().any(|s| {
+ reqwest::Url::parse(&s.url).is_ok_and(|url| {
+ url.host_str()
+ .is_some_and(|host| host == "sqlite.org" || host.ends_with(".sqlite.org"))
+ && url.path().contains("changes")
+ })
+ }),
+ "release notes query must find SQLite's release history: {:?}",
+ search.sources
+ );
let page = tools
.execute(
"live-test",
diff --git a/crates/tw-gateway/src/web_tools/providers/bing.rs b/crates/tw-gateway/src/web_tools/providers/bing.rs
new file mode 100644
index 00000000..c22c042d
--- /dev/null
+++ b/crates/tw-gateway/src/web_tools/providers/bing.rs
@@ -0,0 +1,282 @@
+use super::super::{collect::source, error};
+use scraper::{Html, Selector};
+use std::sync::LazyLock;
+use tw_api::{SearchEngine, WebSource};
+use tw_types::Msg;
+
+static CHALLENGE: LazyLock = LazyLock::new(|| {
+ Selector::parse("#challenge-form, #anomaly-form, #b_captcha, .b_captcha, #turnstile-widget")
+ .unwrap()
+});
+static ROWS: LazyLock = LazyLock::new(|| Selector::parse("#b_results .b_algo").unwrap());
+static LINK: LazyLock = LazyLock::new(|| Selector::parse("h2 a[href]").unwrap());
+static SNIPPET: LazyLock =
+ LazyLock::new(|| Selector::parse(".b_caption p, .b_snippet").unwrap());
+static EMPTY: LazyLock = LazyLock::new(|| Selector::parse("#b_results .b_no").unwrap());
+fn clean(s: &str) -> String {
+ s.split_whitespace().collect::>().join(" ")
+}
+
+fn verification(doc: &Html) -> Result<(), Msg> {
+ if doc.select(&CHALLENGE).next().is_some() {
+ Err(error("search engine requires a verification challenge"))
+ } else {
+ Ok(())
+ }
+}
+
+fn html_results(body: &str) -> Result, Msg> {
+ use base64::Engine;
+ let doc = Html::parse_document(body);
+ verification(&doc)?;
+ let mut sources = Vec::new();
+ let base = reqwest::Url::parse(URL).unwrap();
+ for row in doc.select(&ROWS) {
+ let Some(link) = row.select(&LINK).next() else {
+ continue;
+ };
+ let Some(href) = link.attr("href") else {
+ continue;
+ };
+ let Ok(url) = base.join(href) else { continue };
+ let destination = if matches!(
+ url.host_str(),
+ Some("www.bing.com" | "cn.bing.com" | "bing.com")
+ ) {
+ if !matches!(url.path(), "/ck/a" | "/ck/a/") {
+ continue;
+ }
+ let Some(encoded) = url
+ .query_pairs()
+ .find_map(|(key, value)| (key == "u").then(|| value.into_owned()))
+ else {
+ continue;
+ };
+ let Some(encoded) = encoded.strip_prefix("a1") else {
+ continue;
+ };
+ let Ok(bytes) = base64::engine::general_purpose::URL_SAFE_NO_PAD
+ .decode(encoded.trim_end_matches('='))
+ else {
+ continue;
+ };
+ let Ok(destination) = String::from_utf8(bytes) else {
+ continue;
+ };
+ destination
+ } else {
+ url.to_string()
+ };
+ let title = link.text().collect::();
+ let snippet = row
+ .select(&SNIPPET)
+ .next()
+ .map(|node| node.text().collect::())
+ .unwrap_or_default();
+ if let Some(source) = source(SearchEngine::Bing, &destination, &title, &snippet) {
+ sources.push(source);
+ }
+ }
+ if sources.is_empty() && doc.select(&EMPTY).next().is_none() {
+ return Err(error("Bing search page layout was not recognized"));
+ }
+ Ok(sources)
+}
+
+fn rss(body: &str) -> Result, Msg> {
+ use quick_xml::events::Event;
+ let mut reader = quick_xml::Reader::from_str(body);
+ reader.config_mut().trim_text(false);
+ let mut items = Vec::new();
+ let mut in_item = false;
+ let mut rss = false;
+ let (mut tag, mut title, mut url, mut text) =
+ (Vec::new(), String::new(), String::new(), String::new());
+ loop {
+ match reader.read_event() {
+ Ok(Event::Start(e)) => {
+ tag = e.name().as_ref().as_bytes().to_vec();
+ if tag == b"rss" {
+ rss = true;
+ }
+ if tag == b"item" {
+ in_item = true;
+ title.clear();
+ url.clear();
+ text.clear();
+ }
+ }
+ Ok(event @ (Event::Text(_) | Event::CData(_) | Event::GeneralRef(_))) if in_item => {
+ let value = match &event {
+ Event::Text(e) => e.as_ref().to_string(),
+ Event::CData(e) => e.as_ref().to_string(),
+ Event::GeneralRef(e) => {
+ quick_xml::escape::unescape(&format!("&{};", e.as_ref()))
+ .map_err(|_| error("invalid search XML entity"))?
+ .into_owned()
+ }
+ _ => unreachable!(),
+ };
+ match tag.as_slice() {
+ b"title" => title.push_str(&value),
+ b"link" => url.push_str(&value),
+ b"description" => text.push_str(&value),
+ _ => {}
+ }
+ }
+ Ok(Event::End(e)) if e.name().as_ref() == "item" => {
+ if let Some(s) = source(
+ SearchEngine::Bing,
+ &url,
+ &title,
+ &clean(
+ &Html::parse_fragment(&text)
+ .root_element()
+ .text()
+ .collect::>()
+ .join(" "),
+ ),
+ ) {
+ // Bing RSS dates describe the search item, not a verified
+ // publication date of the linked page. Do not invent recency.
+ items.push(s);
+ }
+ in_item = false;
+ }
+ Ok(Event::Eof) => break,
+ Err(_) => return Err(error("invalid search XML")),
+ _ => {}
+ }
+ }
+ if !rss {
+ return Err(error("search engine did not return RSS results"));
+ }
+ Ok(items)
+}
+
+fn search(body: &str) -> Result, Msg> {
+ if body.trim_start().starts_with(" &'static Spec {
+ &SPEC
+ }
+ fn start(&self, query: &Query<'_>) -> Result {
+ let mut q = query.text.to_owned();
+ if !query.allowed.is_empty() {
+ q.push_str(&format!(
+ " ({})",
+ query
+ .allowed
+ .iter()
+ .map(|d| format!("site:{d}"))
+ .collect::>()
+ .join(" OR ")
+ ));
+ }
+ // Language is supplied by the browser headers. Forced cc/setlang URL
+ // parameters can select a regional backend that broadens the query.
+ let mut url = reqwest::Url::parse(URL).unwrap();
+ url.query_pairs_mut().append_pair("q", &q);
+ Ok(Request {
+ url,
+ kind: Kind::Document,
+ form: None,
+ })
+ }
+ fn parse(&self, _: &Request, body: &str, _: &Query<'_>) -> Result {
+ search(body).map(Step::Done)
+ }
+ fn validate_home(&self, body: &str) -> Result<(), Msg> {
+ verification(&Html::parse_document(body))
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ #[test]
+ fn bing_web_results_exclude_ads_and_decode_only_known_tracking_links() {
+ use base64::Engine;
+ let encoded = base64::engine::general_purpose::URL_SAFE_NO_PAD
+ .encode("https://docs.example.org/guide?a=1&b=2#section");
+ let body = format!(
+ r#"
+
+ Use TaskGroup & await tasks.
+ Plain snippet
+
+
+
"#
+ );
+ let rows = search(&body).unwrap();
+ assert_eq!(rows.len(), 2);
+ assert_eq!(rows[0].url, "https://docs.example.org/guide?a=1&b=2");
+ assert_eq!(rows[0].title, "The official guide");
+ assert_eq!(rows[0].text, "Use TaskGroup & await tasks.");
+ assert!(rows[0].published_at.is_none());
+ assert_eq!(rows[1].text, "Plain snippet");
+ assert!(
+ search(r#"- No results
"#)
+ .unwrap()
+ .is_empty()
+ );
+ assert!(search(r#"Verify
"#).is_err());
+ assert!(
+ search(r#""#)
+ .unwrap_err()
+ .text
+ .contains("verification")
+ );
+ // Merely mentioning a challenge in scripts or an article is not one.
+ assert!(
+ verification(&Html::parse_document(
+ r#""#
+ ))
+ .is_ok()
+ );
+ assert!(search("Unexpected layout").is_err());
+ }
+}
+#[cfg(test)]
+mod entity_tests {
+ use super::*;
+ #[test]
+ fn rss_preserves_entities_cdata_and_query_parameters() {
+ let result = search( r#"- Rust & Web 中文https://example.org/?a=1&b=2A & B]]>
"#).unwrap();
+ assert_eq!(result[0].title, "Rust & Web 中文");
+ assert_eq!(result[0].url, "https://example.org/?a=1&b=2");
+ assert_eq!(result[0].text, "A & B");
+ }
+
+ #[test]
+ fn search_item_dates_are_not_page_publication_dates() {
+ let result = search( "- Old pagehttps://example.org/Mon, 16 Nov 2026 00:00:00 GMT
").unwrap();
+ assert!(result[0].published_at.is_none());
+ }
+}
diff --git a/crates/tw-gateway/src/web_tools/providers/duckduckgo.rs b/crates/tw-gateway/src/web_tools/providers/duckduckgo.rs
new file mode 100644
index 00000000..3214f3c8
--- /dev/null
+++ b/crates/tw-gateway/src/web_tools/providers/duckduckgo.rs
@@ -0,0 +1,257 @@
+//! Read the same first-party organic results used by DuckDuckGo's regular site.
+//! The HTML endpoint frequently challenges even when this normal flow works.
+//! Never execute returned JavaScript or consume ads / generated answer text.
+
+use scraper::{Html, Selector};
+use serde::Deserialize;
+use std::sync::LazyLock;
+use tw_api::{SearchEngine, WebSource};
+use tw_types::Msg;
+
+use super::super::{collect, error};
+use super::{Endpoint, Kind, Profile, Provider, Query, Request, Spec, Step};
+
+static CHALLENGE: LazyLock = LazyLock::new(|| {
+ Selector::parse("#challenge-form, #anomaly-form, script[src*='anomaly.js']").unwrap()
+});
+static PRELOAD: LazyLock = LazyLock::new(|| {
+ Selector::parse("script#deep_preload_script[src], link#deep_preload_link[href]").unwrap()
+});
+
+pub(super) fn is_data_url(url: &reqwest::Url) -> bool {
+ url.scheme() == "https"
+ && matches!(
+ url.host_str(),
+ Some("links.duckduckgo.com" | "duckduckgo.com")
+ )
+ && url.port_or_known_default() == Some(443)
+ && url.path() == "/d.js"
+ && url.username().is_empty()
+ && url.password().is_none()
+ && url.fragment().is_none()
+}
+
+pub(super) fn data_url(body: &str, query: &str) -> Result {
+ let doc = Html::parse_document(body);
+ if doc.select(&CHALLENGE).next().is_some() {
+ return Err(error("search engine requires a verification challenge"));
+ }
+ let link = doc
+ .select(&PRELOAD)
+ .next()
+ .and_then(|node| node.attr("src").or_else(|| node.attr("href")))
+ .ok_or_else(|| error("DuckDuckGo search page layout was not recognized"))?;
+ let base = reqwest::Url::parse(URL).unwrap();
+ let url = base
+ .join(link)
+ .map_err(|_| error("invalid search data URL"))?;
+ // This URL comes from untrusted HTML. Do not grant its host proxy remote DNS
+ // unless it is exactly a known first-party data endpoint for this query.
+ let queries: Vec<_> = url.query_pairs().filter(|(key, _)| key == "q").collect();
+ if !is_data_url(&url) || queries.len() != 1 || queries[0].1 != query {
+ return Err(error("DuckDuckGo returned an unexpected search data URL"));
+ }
+ Ok(url)
+}
+
+#[derive(Deserialize)]
+struct Row {
+ #[serde(default)]
+ u: Option,
+ #[serde(default)]
+ t: Option,
+ #[serde(default)]
+ a: Option,
+ #[serde(default)]
+ n: Option,
+}
+
+fn text(html: &str) -> String {
+ Html::parse_fragment(html).root_element().text().collect()
+}
+
+pub(super) fn results(body: &str) -> Result, Msg> {
+ // Only the 'd' (organic) array. The 'a' array and deepPayload can contain
+ // advertisements and AI answers. Parse JSON as data, never evaluate code.
+ let data = ["DDG.pageLayout.load('d',", "DDG.pageLayout.load(\"d\","]
+ .iter()
+ .find_map(|marker| body.split_once(marker).map(|(_, tail)| tail))
+ .ok_or_else(|| {
+ error(
+ if body.contains("isJsaChallenge") || body.contains("challenge-form") {
+ "search engine requires browser verification"
+ } else {
+ "DuckDuckGo did not return organic search results"
+ },
+ )
+ })?;
+ let mut stream = serde_json::Deserializer::from_str(data).into_iter::>();
+ let rows = stream
+ .next()
+ .ok_or_else(|| error("DuckDuckGo search results are missing"))?
+ .map_err(|_| error("invalid DuckDuckGo search results"))?;
+ if !data[stream.byte_offset()..].trim_start().starts_with(')') {
+ return Err(error("invalid DuckDuckGo search result wrapper"));
+ }
+ let mut sources = Vec::new();
+ let mut malformed = false;
+ for row in rows {
+ match (row.u, row.t) {
+ (Some(url), Some(title)) if !title.trim().is_empty() => {
+ if let Some(source) = collect::source(
+ SearchEngine::Duckduckgo,
+ &url,
+ &text(&title),
+ &text(&row.a.unwrap_or_default()),
+ ) {
+ sources.push(source);
+ } else {
+ malformed = true;
+ }
+ }
+ // A final pagination object is not a result; never follow it here.
+ (None, None) if row.n.is_some() => {}
+ _ => malformed = true,
+ }
+ }
+ if malformed && sources.is_empty() {
+ return Err(error("DuckDuckGo search result layout was not recognized"));
+ }
+ Ok(sources)
+}
+
+const URL: &str = "https://duckduckgo.com/";
+static SPEC: Spec = Spec {
+ id: SearchEngine::Duckduckgo,
+ name: "DuckDuckGo",
+ profile: Profile::Basic,
+ min_interval_ms: 0,
+ home: None,
+ endpoints: &[
+ Endpoint {
+ host: "duckduckgo.com",
+ paths: &["/", "/d.js"],
+ },
+ Endpoint {
+ host: "links.duckduckgo.com",
+ paths: &["/d.js"],
+ },
+ ],
+};
+pub(super) struct Duckduckgo;
+impl Provider for Duckduckgo {
+ fn spec(&self) -> &'static Spec {
+ &SPEC
+ }
+ fn start(&self, query: &Query<'_>) -> Result {
+ let mut url = reqwest::Url::parse(URL).unwrap();
+ url.query_pairs_mut()
+ .append_pair("q", query.text)
+ .append_pair("ia", "web");
+ if let Some(market) = super::locale::duckduckgo_market(query.locale) {
+ url.query_pairs_mut().append_pair("kl", market);
+ }
+ Ok(Request {
+ url,
+ kind: Kind::Document,
+ form: None,
+ })
+ }
+ fn parse(&self, request: &Request, body: &str, query: &Query<'_>) -> Result {
+ match request.kind {
+ Kind::Document => data_url(body, query.text).map(|url| {
+ Step::Fetch(Request {
+ url,
+ kind: Kind::Data,
+ form: None,
+ })
+ }),
+ Kind::Data => results(body).map(Step::Done),
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn discovers_first_party_data_without_executing_scripts() {
+ let url = data_url(
+ r#""#,
+ "Rust OnceLock",
+ ).unwrap();
+ assert_eq!(url.query_pairs().count(), 2);
+ assert!(is_data_url(&url));
+ assert!(
+ data_url(
+ r#""#,
+ "中文"
+ )
+ .is_ok()
+ );
+ }
+
+ #[test]
+ fn rejects_changed_queries_and_untrusted_data_destinations() {
+ for url in [
+ "https://evil.example/d.js?q=Rust",
+ "https://links.duckduckgo.com.evil.example/d.js?q=Rust",
+ "https://127.0.0.1/d.js?q=Rust",
+ "http://links.duckduckgo.com/d.js?q=Rust",
+ "https://links.duckduckgo.com:8443/d.js?q=Rust",
+ "https://user@links.duckduckgo.com/d.js?q=Rust",
+ "https://links.duckduckgo.com/redirect?q=Rust",
+ "https://links.duckduckgo.com/d.js?q=Python",
+ "https://links.duckduckgo.com/d.js?q=Rust&q=Python",
+ ] {
+ assert!(
+ data_url(
+ &format!(r#""#),
+ "Rust"
+ )
+ .is_err(),
+ "{url}"
+ );
+ }
+ assert!(
+ data_url("", "Rust")
+ .unwrap_err()
+ .text
+ .contains("verification")
+ );
+ }
+
+ #[test]
+ fn reads_only_organic_json_and_preserves_markup_text_and_escaped_punctuation() {
+ let body = r#"window.execDeep = function() { DDG.pageLayout.load('a',[{"u":"https://ads.example/","t":"Ad"}]); DDG.deep.deepPayload={"answer":"invented answer"}; DDG.pageLayout.load('d',[{"u":"https://docs.example/a?x=1&y=2&utm_source=search#top","t":"Rust & 中文","a":"Use OnceLock with \"quoted ], text\"."},{"n":"/d.js?next=1"}]); }"#;
+ let found = results(body).unwrap();
+ assert_eq!(found.len(), 1);
+ assert_eq!(found[0].url, "https://docs.example/a?x=1&y=2");
+ assert_eq!(found[0].title, "Rust & 中文");
+ assert_eq!(found[0].text, "Use OnceLock with \"quoted ], text\".");
+ assert!(found[0].published_at.is_none());
+ assert!(results("DDG.pageLayout.load('d',[]);").unwrap().is_empty());
+ }
+
+ #[test]
+ fn malformed_data_and_challenges_are_failures_not_empty_successes() {
+ for body in [
+ "DDG.pageLayout.load('a',[]);",
+ "DDG.pageLayout.load('d',[{bad}]);",
+ "DDG.pageLayout.load('d',[{\"new_field\":1}]);",
+ "DDG.pageLayout.load('d',[] + injected());",
+ "