Sync upstream dev into re-dev (2026-10-19, evening): #1318, #1320, #1264, #1319 and 7 more

Cargo.toml

@@ -78,6 +78,8 @@ feed-rs = "2"
 cron = "0.12"
 infer = "0.22"
 sha2 = "0.11"
+glob = "0.3"
+sha1 = "0.10"
 dom_smoothie = "0.16"
 dom_query = "0.26"
 htmd = "0.5"

crates/utopia-cli/src/main.rs

@@ -82,7 +82,7 @@ struct ManifestDataDir {
 /// not a side effect of a code change.
 // 是迁移文件的**个数**，不是最大的编号（守卫 `schema_version_policy_compares_against_current`
 // 按个数比）：编号有空缺时两者不同——0071 由一个开放 PR 占着，0072 先落，个数是 71
-const CURRENT_SCHEMA_VERSION: u32 = 105;
+const CURRENT_SCHEMA_VERSION: u32 = 107;
 
 fn main() -> anyhow::Result<()> {
     dotenvy::dotenv().ok();

crates/utopia-core/src/models.rs

@@ -271,6 +271,7 @@ pub enum SourceKind {
     Url,
     Rss,
     GithubIssues,
+    GithubRepo,
     JiraIssues,
     S3,
     AzureBlob,

crates/utopia-extract/src/align.rs

@@ -67,10 +67,12 @@ the keys of its candidate classes. The classes themselves are listed once under
 each with its key, its label and its definition. For each item, answer with the key of the one \
 candidate class that every thing of this kind is an instance of, or null.\n\
 \n\
-Output exactly one JSON object and nothing else, one pair per item:\n\
-{\"b\": [[12, \"organization\"], [13, null]]}\n\
-The first element of a pair is the item's id; the second is the chosen key, copied exactly as \
-listed under that item, or null.\n\
+Output exactly one JSON object and nothing else, one pair per item. Format (uppercase words \
+and the ellipsis are placeholders, not literal output):\n\
+{\"b\": [[ITEM_ID, CLASS_KEY_OR_NULL], ...]}\n\
+Copy the exact integer after \"Item\" in the user message into that item's ITEM_ID slot. \
+Never invent an id or renumber the items. Answer every supplied id exactly once. \
+The second element is the chosen key, copied exactly as listed under that item, or null.\n\
 \n\
 1. Choose a candidate only when every thing of this kind is an instance of that class by the \
 class's definition. The class may be broader than the word: a bakery is an organization, a \
@@ -636,7 +638,7 @@ mod tests {
         let user = &msgs[1].content;
 
         assert!(system.contains("every thing of this kind is an instance of that class"));
-        assert!(system.contains("{\"b\": [[12, \"organization\"], [13, null]]}"));
+        assert!(system.contains("{\"b\": [[ITEM_ID, CLASS_KEY_OR_NULL], ...]}"));
         assert!(system.contains("never merely related"));
         assert!(system.contains("Never invent a key"));
 

crates/utopia-extract/src/phrase_align.rs

@@ -126,7 +126,9 @@ impl Direction {
     }
 }
 
-/// 系统消息。例子是中性的，不出自任何测量语料；规则 1 的「每一条」是整条路的判据
+/// 系统消息。例子是中性的，不出自任何测量语料；规则 1 的「每一条」是整条路的判据。
+/// Reply slots use placeholders: a local model copied the old example's 12/13/14
+/// into a batch numbered 0/1/2, leaving every signature unanswered (#1321).
 const PHRASE_SYSTEM: &str = "\
 You bind the relation phrases documents use to the properties of a knowledge base's ontology. \
 Each numbered item is one signature: a phrase as the documents wrote it, the class of the thing \
@@ -144,9 +146,13 @@ states by that property's definition, and the direction: \"forward\" when the st
 subject is the property's subject, \"reverse\" when the statement's object is; or null. For a \
 state, also say which end of it a statement dated at a single moment marks (rule 6).\n\
 \n\
-Output exactly one JSON object and nothing else, one entry per item:\n\
-{\"b\": [[12, \"headquartered_in\", \"forward\", \"none\"], [13, \"member_of\", \"forward\", \"start\"], \
-[14, null, null, null]]}\n\
+Output exactly one JSON object and nothing else, one entry per item. Format (uppercase words \
+and the ellipsis are placeholders, not literal output):\n\
+{\"b\": [[ITEM_ID, PROPERTY_KEY_OR_NULL, DIRECTION_OR_NULL, MARKS_OR_NULL], ...]}\n\
+Copy the exact integer after \"Item\" in the user message into that item's ITEM_ID slot. \
+Never invent an id or renumber the items. Answer every supplied id exactly once. \
+The other slots contain the chosen property's key, direction and marks, as described below; \
+when no property fits, all three are null.\n\
 \n\
 1. Choose a property only when each of the statements under this signature states that \
 property, by the definition as given. The property may be broader than the phrase: \"opened a \

crates/utopia-llm/src/lib.rs

@@ -386,6 +386,9 @@ pub struct Reply {
     pub finish_reason: Option<String>,
     /// 端点报的用量（最后一帧）。不报就是 `None`——账上不编数字
     pub usage: Option<Usage>,
+    /// 回复在复读，是我们停了读（#1301），不是端点收的尾。与 `finish_reason` 分开记：
+    /// 记成 `length` 的话丢弃行会说「撞上了 token 上限」，看的人就去查那个上限了
+    pub repeating: bool,
 }
 
 /// 一次调用的 token 用量，端点自己报的
@@ -510,6 +513,28 @@ fn strip_reasoning(reply: &str) -> &str {
     }
 }
 
+/// 回复的尾巴是不是同一段在复读（#1301）。
+///
+/// 温度 0 的小模型会掉进复读：同一行陈述一遍遍写下去，直到撞上
+/// [`MAX_COMPLETION_TOKENS`]。本地 qwen3-8b 上约 3% 的抽取调用如此，每次占一个槽
+/// 一个多小时，吃掉六成的生成量。上限本身不能降（开推理的端点上它是思考预算），
+/// 所以看尾巴：同一段（至少 30 字节）一字不差连着出现五次，就不是在答题了——
+/// 真实的陈述行彼此不同。按字节比，多字节字符切在哪都不影响。
+///
+/// 只算**最短周期**也有 30 字节的尾巴：每条陈述以原文引文开头，一段 150 字节以上的
+/// 短周期串照样能拆成 30 字节以上的段——目录行 200 个点的引导线、一排二十个空
+/// `<td></td>`，不加这一条就可能被当成复读截断（看每 256 字节的检查落在哪），引文之后
+/// 的全丢了。真复读是整行在重复；短于 30 字节的行复读时这里不截（一夜 57 次截断里有
+/// 4 次是这种），宁可让它跑到上限，也不截断引文
+fn repeats_at_tail(text: &[u8]) -> bool {
+    const REPEATS: usize = 5;
+    (30..=600.min(text.len() / REPEATS)).any(|span| {
+        let tail = &text[text.len() - span * REPEATS..];
+        tail.chunks(span).all(|piece| piece == &tail[..span])
+            && !(1..30).any(|shift| tail[shift..] == tail[..tail.len() - shift])
+    })
+}
+
 impl LlmClient {
     pub fn new(base_url: &str, api_key: Option<&str>, model: &str) -> Self {
         Self::with_timeouts(base_url, api_key, model, CONNECT_TIMEOUT, READ_TIMEOUT)
@@ -698,7 +723,8 @@ impl LlmClient {
         let (mut saw_frame, mut ended) = (false, false);
         let mut finish_reason: Option<String> = None;
         let mut usage: Option<Usage> = None;
-        while let Some(part) = bytes.next().await {
+        let (mut checked, mut repeating) = (0, false);
+        'read: while let Some(part) = bytes.next().await {
             let part = part.map_err(Unreachable)?;
             // 网络片段可能断在 UTF-8 字符中间，等完整 SSE 帧到齐再解码。
             buf.push(&part);
@@ -712,6 +738,24 @@ impl LlmClient {
                     &mut finish_reason,
                     &mut usage,
                 );
+                // 复读了就不再读，回复记成 `repeating`：抽取当它是半截的，前面完整的陈述照收，
+                // 不再等它写到 65,536 个 token。丢掉响应流，连接一关，端点那边也就停了。按帧看
+                // 而不是按网络片段看：一个片段可能装着整段复读。每多 256 字节看一次
+                if !ended && answer.len() >= checked + 256 {
+                    checked = answer.len();
+                    if repeats_at_tail(answer.as_bytes()) {
+                        tracing::warn!(
+                            model = %self.model,
+                            call = %call_label(messages),
+                            chars = answer.len(),
+                            "回复在复读，按截断收下"
+                        );
+                        repeating = true;
+                        ended = true;
+                        buf = SseBuffer::default();
+                        break 'read;
+                    }
+                }
             }
         }
         // **收尾那一帧也算**：末尾不跟空行的实现有的是，丢掉它就是丢掉答案的尾巴
@@ -749,6 +793,7 @@ impl LlmClient {
             text: strip_reasoning(&answer).to_string(),
             finish_reason,
             usage,
+            repeating,
         })
     }
 
@@ -1998,6 +2043,55 @@ mod tests {
         }
     }
 
+    fn content_frame(text: &str) -> String {
+        format!(
+            "data: {}\n\n",
+            json!({"choices": [{"delta": {"content": text}}]})
+        )
+    }
+
+    async fn streamed_reply(sse: &str) -> Reply {
+        let (addr, server) = an_http_response("200 OK", "text/event-stream", sse).await;
+        let user = ChatMessage {
+            role: "user".into(),
+            content: "hi".into(),
+        };
+        let reply = client_at(addr)
+            .chat_at_streaming(&[user], Some(0.0))
+            .await
+            .unwrap();
+        server.await.unwrap();
+        reply
+    }
+
+    /// 复读的回复按截断收下（#1301）：这条流没有收尾，没有护栏就是「流断在半路」
+    #[tokio::test]
+    async fn a_reply_that_keeps_repeating_comes_back_cut() {
+        let row = r#"[null,"a","is on","b",null,{},null,null],"#;
+        let sse =
+            content_frame(r#"{"e":[["a","thing",true]],"s":["#) + &content_frame(row).repeat(200);
+        let reply = streamed_reply(&sse).await;
+        assert!(reply.repeating);
+        assert!(!reply.hit_token_ceiling(), "the endpoint never said length");
+        assert!(
+            reply.text.len() < row.len() * 20,
+            "read {} bytes past the loop",
+            reply.text.len()
+        );
+    }
+
+    /// 引文里一条 200 个点的目录引导线不是复读：周期是一个点，不是一整行
+    #[tokio::test]
+    async fn a_quoted_dot_leader_comes_back_whole() {
+        let sse = content_frame(r#"{"e":[["Introduction","section",true]],"s":[["Introduction "#)
+            + &content_frame(&".".repeat(200))
+            + &content_frame(r#" 3","Introduction","is on page",null,"3",{},null,null]],"n":[]}"#)
+            + "data: [DONE]\n\n";
+        let reply = streamed_reply(&sse).await;
+        assert!(!reply.repeating);
+        assert!(reply.text.ends_with(r#""n":[]}"#), "{}", reply.text);
+    }
+
     #[tokio::test]
     async fn a_raw_stream_cut_before_its_finish_is_an_error() {
         use futures_util::TryStreamExt;

crates/utopia-server/Cargo.toml

@@ -17,6 +17,8 @@ utopia-search.workspace = true
 utopia-llm.workspace = true
 utopia-extract = { path = "../utopia-extract" }
 sha2.workspace = true
+glob.workspace = true
+sha1.workspace = true
 futures-util.workspace = true
 async-stream.workspace = true
 tokio.workspace = true

crates/utopia-server/src/api/admin_routes.rs

@@ -79,17 +79,25 @@ pub async fn put_deployment(
             .worker_concurrency
             .store(n as usize, std::sync::atomic::Ordering::Relaxed);
     }
-    // 按模型的限额即时生效：闸门每次调用前读库，发现限额变了就换一把新信号量
     if let Some(l) = &req.default_ontology_lang {
         utopia_store::access::set_default_ontology_lang(&state.pool, l).await?;
     }
+    let limits_changed = req.default_model_concurrency.is_some() || req.model_limit.is_some();
     if let Some(n) = req.default_model_concurrency {
         utopia_store::model_limits::set_default(&state.pool, n).await?;
     }
     if let Some(m) = req.model_limit {
         utopia_store::model_limits::set(&state.pool, &m.base_url, &m.model, m.max_concurrent)
             .await?;
     }
+    // 按模型的限额在这里就地生效：闸门调大放行、调小收紧，不等下一个任务来取许可时才读
+    // ——那时可能所有 worker 都等在闸门上，唯一在飞的还是一次复读（#1302）。限额已经存下了，
+    // 调闸门失败只记一笔：下一次取许可照样读到新限额
+    if limits_changed {
+        if let Err(e) = state.model_gates.refresh(&state.pool).await {
+            tracing::warn!(error = %e, "限额已保存，闸门要等下一次取许可才按新限额调");
+        }
+    }
     Ok(Json(json!({ "ok": true })))
 }
 

crates/utopia-server/src/api/token_read_tests.rs

@@ -1,4 +1,4 @@
-//! 个人令牌读实体（0014 决定 7）：搜实体、实体详情、认知历史、事实证据四条读路由
+//! 个人令牌读图（0014 决定 7）：搜实体、实体详情、认知历史、事实证据、图总览、邻域六条读路由
 //! 收任何一枚活着的令牌。角色照旧是上限，`kb_ids` 排除了的库答 404；改实体仍只认会话。
 use crate::state::AppState;
 use axum::body::Body;
@@ -81,13 +81,15 @@ impl Fixture {
             .1
     }
 
-    fn read_paths(&self) -> [String; 4] {
+    fn read_paths(&self) -> [String; 6] {
         let (kb, entity, fact) = (self.kb, self.entity, self.fact);
         [
             format!("/kbs/{kb}/entities?q=Zhang"),
             format!("/kbs/{kb}/entities/{entity}"),
             format!("/kbs/{kb}/entities/{entity}/history"),
             format!("/kbs/{kb}/facts/{fact}/evidence"),
+            format!("/kbs/{kb}/graph/overview"),
+            format!("/kbs/{kb}/graph/neighborhood?entity={entity}"),
         ]
     }
 
@@ -109,8 +111,7 @@ impl Fixture {
 }
 
 #[tokio::test]
-async fn a_read_token_of_a_base_viewer_reads_its_entities_facts_and_history() -> anyhow::Result<()>
-{
+async fn a_read_token_of_a_base_viewer_gets_200_on_every_graph_read() -> anyhow::Result<()> {
     let Some(f) = Fixture::new().await? else {
         return Ok(());
     };
@@ -128,7 +129,7 @@ async fn a_read_token_of_a_base_viewer_reads_its_entities_facts_and_history() ->
 }
 
 #[tokio::test]
-async fn a_token_limited_to_another_base_gets_404_on_every_entity_read() -> anyhow::Result<()> {
+async fn a_token_limited_to_another_base_gets_404_on_every_graph_read() -> anyhow::Result<()> {
     let Some(f) = Fixture::new().await? else {
         return Ok(());
     };

crates/utopia-server/src/auth.rs

@@ -177,7 +177,8 @@ impl FromRequestParts<AppState> for AuthUser {
 /// **不替换 `AuthUser`。** `AuthUser` 把每个 Bearer 都当 JWT 解，令牌在那几百条
 /// 路由上一律是 401——那是有意的：哪条路由准令牌进来，要在 0014 里说出理由。
 /// 用这个提取器的只有文档原件（0052，含按推送时的 external_id 读回）、「把一个库搭起来」
-/// 那几条、分块检索和读图那几条（决定 7：实体、图视图、本体、抽取、导出）。
+/// 那几条、分块检索和读图那几条：搜实体、实体详情、认知历史、事实证据、图总览、邻域、
+/// 本体、抽取、导出（决定 7）。
 ///
 /// 令牌只收窄：`pat` 在的时候，路由先问 [`Self::require_write`] 或
 /// [`Self::reaches`]，角色照旧由 `require_kb` 与工作区角色判。

crates/utopia-server/src/extraction.rs

@@ -292,7 +292,7 @@ pub(crate) async fn resolve_handle(
     needs_adjudication: &mut bool,
     human_reviews_found: &mut bool,
 ) -> anyhow::Result<Uuid> {
-    let normalized = utopia_store::resolution::normalize_name(name).to_lowercase();
+    let normalized = utopia_store::resolution::name_key(name);
     let response_excluded = response_claims
         .get(&normalized)
         .cloned()
@@ -602,7 +602,7 @@ mod tests {
                 &pool,
                 kb,
                 Some(person),
-                "Zhang Wei",
+                "**Zhang Wei**",
                 None,
                 NameRecall::Full(None),
                 None,