From 5669717db7d6f0938d3ac16d8b11ea2366dc61d3 Mon Sep 17 00:00:00 2001 From: Szabo Bogdan Date: Fri, 23 Oct 2026 11:51:33 +0200 Subject: [PATCH 1/9] A token reads a document's extractions and exports its base (#1266) Signed-off-by: Bogdan Szabo Co-authored-by: Claude Opus 5.5 Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- .../utopia-server/src/api/documents_routes.rs | 11 +++-- crates/utopia-server/src/api/export_routes.rs | 19 ++++--- .../utopia-server/src/api/token_read_tests.rs | 49 +++++++++++++++++-- crates/utopia-server/src/auth.rs | 3 +- ...ty-from-the-person-scope-from-the-token.md | 14 +++++- docs/design/access-and-audit.md | 3 +- 6 files changed, 77 insertions(+), 22 deletions(-) diff --git a/crates/utopia-server/src/api/documents_routes.rs b/crates/utopia-server/src/api/documents_routes.rs index cff69bcca..62746b758 100644 --- a/crates/utopia-server/src/api/documents_routes.rs +++ b/crates/utopia-server/src/api/documents_routes.rs @@ -85,12 +85,13 @@ pub(crate) async fn require_reader_kb( state: &AppState, reader: &SessionOrToken, kb_id: Uuid, -) -> ApiResult<()> { - utopia_store::access::require_kb(&state.pool, &reader.user, kb_id, Role::Viewer).await?; +) -> ApiResult { + let kb = + utopia_store::access::require_kb(&state.pool, &reader.user, kb_id, Role::Viewer).await?; if !reader.reaches(kb_id) { return Err(AppError::NotFound.into()); } - Ok(()) + Ok(kb) } /// Serve the retained original named by the document ledger. @@ -448,11 +449,11 @@ pub async fn detail( /// 反向证据链：文档各分块抽出的事实（文档查看器右栏）。 pub async fn extractions( State(state): State, - AuthUser(user): AuthUser, + reader: SessionOrToken, Path(id): Path, ) -> ApiResult> { let doc = utopia_store::documents::get(&state.pool, id).await?; - utopia_store::access::require_kb(&state.pool, &user, doc.kb_id, Role::Viewer).await?; + require_reader_kb(&state, &reader, doc.kb_id).await?; let facts = utopia_store::graph::document_extractions(&state.pool, id).await?; Ok(Json(json!({ "facts": facts }))) } diff --git a/crates/utopia-server/src/api/export_routes.rs b/crates/utopia-server/src/api/export_routes.rs index 4f1ca0f51..18c2049b8 100644 --- a/crates/utopia-server/src/api/export_routes.rs +++ b/crates/utopia-server/src/api/export_routes.rs @@ -16,12 +16,11 @@ use axum::extract::{Path, Query, State}; use axum::http::{header, HeaderMap, HeaderValue, StatusCode}; use axum::response::{IntoResponse, Response}; use serde::Deserialize; -use utopia_core::models::Role; use utopia_core::AppError; use uuid::Uuid; -use super::graph_routes::require_kb; -use crate::auth::AuthUser; +use super::documents_routes::require_reader_kb; +use crate::auth::SessionOrToken; use crate::error::ApiResult; use crate::rdf::{self, Format, Names, SharedBuf, Sink}; use crate::state::AppState; @@ -38,27 +37,31 @@ pub struct ExportQuery { pub async fn export( State(state): State, - AuthUser(user): AuthUser, + reader: SessionOrToken, Path(kb_id): Path, Query(q): Query, ) -> ApiResult { - let kb = require_kb(&state, &user, kb_id, Role::Viewer).await?; + let kb = require_reader_kb(&state, &reader, kb_id).await?; let format = Format::parse(q.format.as_deref()).ok_or_else(|| { AppError::Validation("Unknown `format` (expected turtle or jsonld)".into()) })?; let names = Names::new(kb_id, q.base.as_deref()).map_err(AppError::Validation)?; - // 导出是一次「整个库离开这台机器」的动作，台账要记下（0014 的同一条理由）。 + // 导出是一次「整个库离开这台机器」的动作，台账要记下（0014 的同一条理由）； + // 令牌导出的还要记下是哪一枚（0014 决定 7）。 // 必须在快照事务之前写完并放掉连接：下面那条事务一占就是整个流的时长， // 占着它再回头要连接，会把最小池（两条）上的并发导出互相饿死 let _ = utopia_store::audit::record( &state.pool, Some(kb_id), - user.id, + reader.user.id, "kb.exported", "knowledge_base", Some(kb_id), - serde_json::json!({ "format": format.extension() }), + utopia_store::audit::via_token( + serde_json::json!({ "format": format.extension() }), + reader.token_id(), + ), ) .await; diff --git a/crates/utopia-server/src/api/token_read_tests.rs b/crates/utopia-server/src/api/token_read_tests.rs index 289b35100..22887e4fe 100644 --- a/crates/utopia-server/src/api/token_read_tests.rs +++ b/crates/utopia-server/src/api/token_read_tests.rs @@ -1,8 +1,9 @@ //! 个人令牌读图（0014 决定 7）：搜实体、实体详情、认知历史、事实证据、图总览、邻域、本体、 -//! 某类的实例这些读路由收任何一枚活着的令牌。角色照旧是上限，`kb_ids` 排除了的库答 404；改实体仍只认会话。 +//! 某类的实例、文档的抽取结果和整库导出这些读路由收任何一枚活着的令牌。角色照旧是上限，`kb_ids` 排除了的库答 404；改实体仍只认会话。 use crate::state::AppState; use axum::body::Body; use axum::http::{Request, StatusCode}; +use serde_json::json; use std::sync::Arc; use tower::ServiceExt; use uuid::Uuid; @@ -18,6 +19,7 @@ struct Fixture { person: Uuid, entity: Uuid, fact: Uuid, + doc: Uuid, _dir: tempfile::TempDir, } @@ -30,7 +32,7 @@ impl Fixture { utopia_store::db::migrate(&pool).await?; let id = Uuid::now_v7; let (org, ws, reader, kb, other_kb) = (id(), id(), id(), id(), id()); - let (person, knows, entity, peer, fact) = (id(), id(), id(), id(), id()); + let (person, knows, entity, peer, fact, doc) = (id(), id(), id(), id(), id(), id()); // 插进去的只有本地生成的 UUID 和写死的日期 sqlx::raw_sql(&format!( "INSERT INTO organizations(id,name) VALUES ('{org}','token-read-test'); @@ -48,7 +50,8 @@ impl Fixture { ('{entity}','{kb}','{person}','Zhang San','2026-01-01'), ('{peer}','{kb}','{person}','Li Si','2026-01-01'); INSERT INTO facts(id,kb_id,subject_id,predicate_id,object_id,recorded_at,confidence) - VALUES ('{fact}','{kb}','{entity}','{knows}','{peer}','2026-01-01',0.9);" + VALUES ('{fact}','{kb}','{entity}','{knows}','{peer}','2026-01-01',0.9); + INSERT INTO documents(id,kb_id,filename,sha256) VALUES ('{doc}','{kb}','spec.md','x');" )) .execute(&pool) .await?; @@ -72,6 +75,7 @@ impl Fixture { person, entity, fact, + doc, _dir: dir, })) } @@ -83,8 +87,9 @@ impl Fixture { .1 } - fn read_paths(&self) -> [String; 8] { - let (kb, person, entity, fact) = (self.kb, self.person, self.entity, self.fact); + fn read_paths(&self) -> [String; 10] { + let (kb, person, entity, fact, doc) = + (self.kb, self.person, self.entity, self.fact, self.doc); [ format!("/kbs/{kb}/entities?q=Zhang"), format!("/kbs/{kb}/entities/{entity}"), @@ -94,6 +99,8 @@ impl Fixture { format!("/kbs/{kb}/graph/neighborhood?entity={entity}"), format!("/kbs/{kb}/ontology"), format!("/kbs/{kb}/ontology/entity-types/{person}/entities"), + format!("/documents/{doc}/extractions"), + format!("/kbs/{kb}/export?format=turtle"), ] } @@ -149,3 +156,35 @@ async fn a_token_limited_to_another_base_gets_404_on_every_graph_read() -> anyho f.cleanup().await?; result } + +#[tokio::test] +async fn an_export_by_token_is_audited_under_its_owner_with_the_token_id() -> anyhow::Result<()> { + let Some(f) = Fixture::new().await? else { + return Ok(()); + }; + let result = async { + let token = f.read_token(f.kb).await; + let token_id = utopia_store::tokens::authenticate(&f.pool, &token) + .await? + .token_id; + let path = format!("/kbs/{}/export?format=turtle", f.kb); + let status = f.get(&path, &token).await?; + anyhow::ensure!(status == StatusCode::OK, "{path}: {status}"); + + let audited: (Option, serde_json::Value) = sqlx::query_as( + "SELECT actor_id, detail FROM audit_events WHERE kb_id = $1 AND action = 'kb.exported'", + ) + .bind(f.kb) + .fetch_one(&f.pool) + .await?; + let expected = ( + Some(f.reader), + json!({ "format": "ttl", "token_id": token_id }), + ); + anyhow::ensure!(audited == expected, "{audited:?}"); + Ok(()) + } + .await; + f.cleanup().await?; + result +} diff --git a/crates/utopia-server/src/auth.rs b/crates/utopia-server/src/auth.rs index e45f46416..a40be2926 100644 --- a/crates/utopia-server/src/auth.rs +++ b/crates/utopia-server/src/auth.rs @@ -177,7 +177,8 @@ impl FromRequestParts for AuthUser { /// **不替换 `AuthUser`。** `AuthUser` 把每个 Bearer 都当 JWT 解，令牌在那几百条 /// 路由上一律是 401——那是有意的：哪条路由准令牌进来，要在 0014 里说出理由。 /// 用这个提取器的只有文档原件（0052）、「把一个库搭起来」那几条和读图那几条：搜实体、 -/// 实体详情、认知历史、事实证据、图总览、邻域、本体、某类的实例（决定 7）。 +/// 实体详情、认知历史、事实证据、图总览、邻域、本体、某类的实例、文档的抽取结果、 +/// 整库导出（决定 7）。 /// /// 按推送时的 `external_id` 读回一篇文档（#1276）也是文档原件，开给令牌的理由写在决定 7。 /// diff --git a/docs/decisions/0014-identity-from-the-person-scope-from-the-token.md b/docs/decisions/0014-identity-from-the-person-scope-from-the-token.md index 747d41e69..8c10d626a 100644 --- a/docs/decisions/0014-identity-from-the-person-scope-from-the-token.md +++ b/docs/decisions/0014-identity-from-the-person-scope-from-the-token.md @@ -1,6 +1,6 @@ # 0014 · Identity from the person, scope from the token -- **Status**: Implemented · 2026-10-16 · #180, #318, #1243 · open: tokens across workspaces +- **Status**: Implemented · 2026-10-16 · #180, #318, #1243, #1262 · open: tokens across workspaces - **Written**: 2026-09-01 · condensed into English 2026-09-03 - **Related**: migration `0014_data_source_grants` gave data sources a grant layer; this is the same question where a machine knocks. [0004](0004-language-and-localization.md) has the server speak English only, MCP error codes included. [0015](0015-recording-a-sentence-is-not-asserting-a-fact.md) removes the main objection to opening `remember` over MCP. @@ -111,6 +111,15 @@ push documents in. without it the client can check what landed only by searching for its own text. The source's push key is not a credential here, so a leaked key stays push-only. + A document's extractions (`GET /documents/{id}/extractions`) and the base's RDF export + (`GET /kbs/{id}/export`) take a token on the same terms. The extractions show what extraction + made of a pushed document; the document's base is resolved first, as for its content, so a + token outside `kb_ids` gets the same 404 whether or not the document exists. A read token may + export the whole base because, with the reads above open, its owner could already copy + everything a Viewer sees out one piece at a time; requiring write scope would only push a + mirror to hold a write token for a read-only job. The control is the `kb.exported` audit row, + which names the owner and, through `via_token`, the token. + ## Dead ends - **Base-level machine tokens**, one per base and unrelated to a person. It would be a third @@ -138,7 +147,8 @@ push documents in. - 2026-10-16: decision 7 opens four graph reads to any token; decision 6's "only these routes and document content" now also counts them. 2026-10-19: the graph overview and neighbourhood join them, and so does reading a pushed document back by its `external_id` (#1276). The - ontology and a class's instances follow (#1265). + ontology and a class's instances follow (#1265), then a document's extractions and the + base's export (#1266). ## Open questions diff --git a/docs/design/access-and-audit.md b/docs/design/access-and-audit.md index 372ee8be0..2f533da5a 100644 --- a/docs/design/access-and-audit.md +++ b/docs/design/access-and-audit.md @@ -21,7 +21,8 @@ reading the source's push key, importing an ontology and deciding a kind word, w role unchanged as the ceiling; any token lists a workspace's bases; a token limited by `kb_ids` creates no base. These routes, document content and eight graph reads (entity search, detail, history, fact evidence, overview, neighbourhood, ontology, a class's instances; Viewer suffices, -`kb_ids` answers 404) take `SessionOrToken`; apart from MCP, every other route is session-only [0014 d6, d7]. `ingest_token` on a source is a different kind: plaintext, push-only [0014]. +`kb_ids` answers 404), a document's extractions and the base's export (its `kb.exported` row +names the token) take `SessionOrToken`; apart from MCP, every other route is session-only [0014 d6, d7]. `ingest_token` on a source is a different kind: plaintext, push-only [0014]. **A client reads a pushed document back by its `external_id`.** `GET /kbs/{id}/sources/{sid}/documents/by-external-id?external_id=…` finds an `api` or `statements` From b39d9283f7e1cb8e82313f083d9197a67da3c5ed Mon Sep 17 00:00:00 2001 From: Quintin Botes Date: Fri, 23 Oct 2026 11:56:25 +0200 Subject: [PATCH 2/9] A base can follow a gitlab.com project's markdown files, fetching only the files that changed (#1327) (#1365) Signed-off-by: Quintin Botes Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- crates/utopia-core/src/models.rs | 2 + crates/utopia-server/src/github_repo.rs | 23 +- crates/utopia-server/src/gitlab_repo.rs | 129 +++++++++++ .../src/gitlab_repo_sync_tests.rs | 177 ++++++++++++++ crates/utopia-server/src/ingest_sources.rs | 181 +++++++++++++++ crates/utopia-server/src/main.rs | 1 + .../tests/fixtures/gitlab_repo.json | 215 ++++++++++++++++++ .../store/a_viewer_never_sees_a_credential.rs | 4 + docs/design/sources.md | 5 +- web/src/api.ts | 4 +- web/src/i18n/en.ts | 7 + web/src/i18n/zh.ts | 6 + web/src/pages/Library.tsx | 48 ++-- web/src/pages/SourcesRail.tsx | 2 + web/src/pages/githubRepoConfig.test.ts | 6 + web/src/pages/githubRepoConfig.ts | 8 +- web/src/sourceKinds.test.ts | 3 + web/src/sourceKinds.ts | 5 +- 18 files changed, 792 insertions(+), 34 deletions(-) create mode 100644 crates/utopia-server/src/gitlab_repo.rs create mode 100644 crates/utopia-server/src/gitlab_repo_sync_tests.rs create mode 100644 crates/utopia-server/tests/fixtures/gitlab_repo.json diff --git a/crates/utopia-core/src/models.rs b/crates/utopia-core/src/models.rs index a32be0210..15dbb2fbc 100644 --- a/crates/utopia-core/src/models.rs +++ b/crates/utopia-core/src/models.rs @@ -272,6 +272,8 @@ pub enum SourceKind { Rss, GithubIssues, GithubRepo, + /// gitlab.com 项目默认分支的文件（#1327）；自建 GitLab 没有验证过 + GitlabRepo, JiraIssues, S3, AzureBlob, diff --git a/crates/utopia-server/src/github_repo.rs b/crates/utopia-server/src/github_repo.rs index cd7735bd5..df554a346 100644 --- a/crates/utopia-server/src/github_repo.rs +++ b/crates/utopia-server/src/github_repo.rs @@ -55,18 +55,27 @@ impl FileFilter { } fn eligible(&self, e: &TreeEntry) -> bool { + self.path_ok(&e.path, &e.kind, &e.mode) && e.size.is_some_and(|s| self.size_ok(s)) + } + + /// 只看树条目本身就能判的那一半。GitLab 的树不带大小（#1327），大小要另问一次， + /// 所以两半分开 + pub(crate) fn path_ok(&self, path: &str, kind: &str, mode: &str) -> bool { let opts = glob::MatchOptions { case_sensitive: true, require_literal_separator: true, require_literal_leading_dot: false, }; - // 子模块是 commit、目录是 tree，符号链接指向别处——都不是文件本体。 - // 空文件成不了文档（`ingest_item` 对空字节什么也不建），收进来就每轮都得重取一遍 - e.kind == "blob" - && e.mode != "120000" - && e.size.is_some_and(|s| s > 0 && s <= self.max_file_bytes) - && self.include.iter().any(|p| p.matches_with(&e.path, opts)) - && !self.exclude.iter().any(|p| p.matches_with(&e.path, opts)) + // 子模块是 commit、目录是 tree，符号链接指向别处——都不是文件本体 + kind == "blob" + && mode != "120000" + && self.include.iter().any(|p| p.matches_with(path, opts)) + && !self.exclude.iter().any(|p| p.matches_with(path, opts)) + } + + /// 空文件成不了文档（`ingest_item` 对空字节什么也不建），收进来就每轮都得重取一遍 + pub(crate) fn size_ok(&self, size: i64) -> bool { + size > 0 && size <= self.max_file_bytes } } diff --git a/crates/utopia-server/src/gitlab_repo.rs b/crates/utopia-server/src/gitlab_repo.rs new file mode 100644 index 000000000..119499fbd --- /dev/null +++ b/crates/utopia-server/src/gitlab_repo.rs @@ -0,0 +1,129 @@ +//! GitLab 项目文件来源（gitlab.com）：默认分支头的一棵文件树，一条路径一篇文档，与 +//! [`crate::github_repo`] 同一个形状（#1327）。同步在 `ingest_sources`；这里只放不联网的那一半。 +//! +//! 与 GitHub 不同的三处：树是分页的（keyset，每页至多 100 条，没有 `truncated`）；树条目 +//! 有 blob SHA 却没有大小，于是大小上限要等 `HEAD` 文件拿到 `X-Gitlab-Size` 才判得了； +//! 提交时间带毫秒，落库前截到秒（0013）。 + +use chrono::{DateTime, SubsecRound as _, Utc}; +use serde::Deserialize; +use std::collections::HashSet; + +use crate::github_repo::FileFilter; + +/// 一棵树最多翻这么多页（每页 100 条）。翻不完就当树被截断：本轮照收列出来的文件， +/// 但不判「不在来源中」——缺席可能只是还没列到，与 GitHub 的 `truncated` 同一个处理 +pub const MAX_TREE_PAGES: usize = 100; + +/// 空文件的 git blob SHA 是固定的。树里没有大小，靠它在问大小之前就把空文件挡掉；不挡的话， +/// 空文件从不落库，每一轮都要为它白发一次 `HEAD` +pub const EMPTY_BLOB: &str = "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"; + +#[derive(Debug, Clone, Deserialize)] +pub struct TreeEntry { + /// git blob SHA，与 GitHub 树里的 `sha` 是同一个值 + pub id: String, + pub path: String, + #[serde(rename = "type")] + pub kind: String, + pub mode: String, +} + +/// 按路径排序、同一 blob 只留第一条，与 [`crate::github_repo::choose`] 一致。大小此时还不知道， +/// 由同步在 `HEAD` 之后再判 +pub fn choose(entries: Vec, filter: &FileFilter) -> Vec { + let mut entries: Vec<_> = entries + .into_iter() + .filter(|e| e.id != EMPTY_BLOB && filter.path_ok(&e.path, &e.kind, &e.mode)) + .collect(); + entries.sort_by(|a, b| a.path.cmp(&b.path)); + let mut seen = HashSet::new(); + entries.retain(|e| seen.insert(e.id.clone())); + entries +} + +/// `Link` 头里 `rel="next"` 的那一页。只认与 API 同源的地址：这个地址是响应给的， +/// 照单全收就等于让对方决定我们带着令牌去访问哪里 +pub fn next_page(link: Option<&str>, api_base: &reqwest::Url) -> Option { + link?.split(',').find_map(|part| { + let (target, params) = part.split_once(';')?; + if !params.split(';').any(|p| p.trim() == r#"rel="next""#) { + return None; + } + let url = reqwest::Url::parse(target.trim().strip_prefix('<')?.strip_suffix('>')?).ok()?; + (url.origin() == api_base.origin() && url.path().starts_with(api_base.path())) + .then_some(url) + }) +} + +/// `committed_date` 形如 `2026-10-05T14:30:00.000+00:00`，截到秒 +pub fn commit_time(s: &str) -> Option> { + DateTime::parse_from_rfc3339(s) + .ok() + .map(|t| t.with_timezone(&Utc).trunc_subsecs(0)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn entry(path: &str, kind: &str, mode: &str, id: &str) -> TreeEntry { + TreeEntry { + id: id.into(), + path: path.into(), + kind: kind.into(), + mode: mode.into(), + } + } + + #[test] + fn the_tree_keeps_markdown_files_once_and_leaves_links_and_folders_out() { + let f = FileFilter::from_config(&serde_json::json!({})).unwrap(); + let chosen = choose( + vec![ + entry("docs/guide.md", "blob", "100644", "9292"), + entry("docs/copy-of-guide.md", "blob", "100644", "9292"), + entry("docs", "tree", "040000", "5964"), + entry("link.md", "blob", "120000", "eafc"), + entry(".gitlab/notes.md", "blob", "100644", "3e42"), + entry("node_modules/pkg/README.md", "blob", "100644", "08e8"), + entry("src/main.rs", "blob", "100644", "f328"), + entry("README.md", "blob", "100644", "9e4e"), + entry("notes/empty.md", "blob", "100644", EMPTY_BLOB), + ], + &f, + ); + let paths: Vec<_> = chosen.iter().map(|e| e.path.as_str()).collect(); + assert_eq!(paths, ["README.md", "docs/copy-of-guide.md"]); + } + + #[test] + fn only_a_next_page_on_the_same_api_is_followed() { + let base = reqwest::Url::parse("https://gitlab.com/api/v4").unwrap(); + let link = r#"; rel="next""#; + assert_eq!( + next_page(Some(link), &base).map(|u| u + .query_pairs() + .find(|(k, _)| k == "page_token") + .unwrap() + .1 + .into_owned()), + Some("x".into()) + ); + assert!(next_page( + Some(r#"; rel="next""#), + &base + ) + .is_none()); + assert!(next_page(Some(r#"; rel="next""#), &base).is_none()); + assert!(next_page(Some(r#"; rel="first""#), &base).is_none()); + assert!(next_page(None, &base).is_none()); + } + + #[test] + fn a_commit_time_is_cut_to_the_second() { + let t = commit_time("2026-10-05T14:30:00.987+02:00").unwrap(); + assert_eq!(t.to_rfc3339(), "2026-10-05T12:30:00+00:00"); + assert!(commit_time("yesterday").is_none()); + } +} diff --git a/crates/utopia-server/src/gitlab_repo_sync_tests.rs b/crates/utopia-server/src/gitlab_repo_sync_tests.rs new file mode 100644 index 000000000..876940e06 --- /dev/null +++ b/crates/utopia-server/src/gitlab_repo_sync_tests.rs @@ -0,0 +1,177 @@ +//! 用录下的真实 GitLab 响应跑项目文件同步（#1327）：树翻页、大小先 `HEAD` 再取、同一 blob +//! 只收一处、没变不再取、树没翻完时不判「不在来源中」。 +use super::sync_gitlab_repo_with_client; +use serde_json::{json, Value}; +use std::sync::{Arc, Mutex}; +use uuid::Uuid; +use wiremock::{Mock, MockServer, Request, ResponseTemplate}; + +const PROJECT: &str = "/projects/QuintinBotes%2Futopia-gitlab-fixture/repository"; + +/// 阶段 0：录下的树分两页给。阶段 1：去掉 README.md，每一页都说还有下一页，翻到上限也翻不完。 +/// 阶段 2：去掉 README.md，一页给完 +fn tree_page(fixture: &Value, stage: u8, uri: &str, page: Option<&str>) -> ResponseTemplate { + let mut entries = fixture["tree"].as_array().unwrap().clone(); + if stage > 0 { + entries.retain(|e| e["path"] != "README.md"); + } + let next = format!( + r#"<{uri}{PROJECT}/tree?page_token=2&pagination=keyset&recursive=true>; rel="next""# + ); + match (stage, page) { + (0, None) => ResponseTemplate::new(200) + .set_body_json(json!(entries[..10])) + .insert_header("link", next.as_str()), + (0, Some(_)) => ResponseTemplate::new(200).set_body_json(json!(entries[10..])), + (1, _) => ResponseTemplate::new(200) + .set_body_json(json!(entries)) + .insert_header("link", next.as_str()), + _ => ResponseTemplate::new(200).set_body_json(json!(entries)), + } +} + +#[tokio::test] +async fn a_gitlab_project_sync_takes_each_changed_file_once() -> anyhow::Result<()> { + let Some(url) = utopia_store::test_db::url() else { + return Ok(()); + }; + let pool = sqlx::PgPool::connect(&url).await?; + let (org, ws, kb, sid) = ( + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + ); + let result = async { + // 3000 字节的上限把 3368 字节的 big.md 挡在外面 + sqlx::raw_sql(&format!( + r#"INSERT INTO organizations(id,name) VALUES('{org}','gitlab-test'); + INSERT INTO workspaces(id,org_id,name) VALUES('{ws}','{org}','gitlab-test'); + INSERT INTO knowledge_bases(id,workspace_id,name) VALUES('{kb}','{ws}','gitlab-test'); + INSERT INTO sources(id,kb_id,kind,name,config) + VALUES('{sid}','{kb}','gitlab_repo','fixture', + '{{"project":"QuintinBotes/utopia-gitlab-fixture","max_file_bytes":3000}}');"# + )) + .execute(&pool) + .await?; + let source = utopia_store::sources::get(&pool, sid).await?; + let dir = tempfile::tempdir()?; + let cfg = utopia_core::config::AppConfig { + data_dir: dir.path().to_string_lossy().into_owned(), + ..Default::default() + }; + let search = Arc::new(utopia_search::SearchIndex::open( + &dir.path().join("search"), + )?); + let state = crate::state::AppState::new(pool.clone(), &cfg, search, "test-only".into()); + + let fixture: Value = + serde_json::from_str(include_str!("../tests/fixtures/gitlab_repo.json"))?; + let server = MockServer::start().await; + let stage = Arc::new(Mutex::new(0u8)); + let (current, fx, uri) = (stage.clone(), fixture.clone(), server.uri()); + Mock::given(wiremock::matchers::any()) + .respond_with(move |request: &Request| { + let path = request.url.path(); + let Some(rest) = path.strip_prefix(PROJECT) else { + return ResponseTemplate::new(404); + }; + let query = |key: &str| { + request + .url + .query_pairs() + .find(|(k, _)| k == key) + .map(|(_, v)| v.into_owned()) + }; + if rest == "/tree" { + tree_page(&fx, *current.lock().unwrap(), &uri, query("page_token").as_deref()) + } else if let Some(file) = rest.strip_prefix("/files/") { + let file = file.replace("%2F", "/"); + ResponseTemplate::new(200) + .insert_header("x-gitlab-size", fx["sizes"][&file].to_string().as_str()) + } else if rest == "/commits" { + ResponseTemplate::new(200).set_body_json(fx["commits"][&query("path").unwrap()].clone()) + } else if let Some(sha) = rest.strip_prefix("/blobs/").and_then(|r| r.strip_suffix("/raw")) { + ResponseTemplate::new(200).set_body_string(fx["blobs"][sha].as_str().unwrap()) + } else { + ResponseTemplate::new(404) + } + }) + .mount(&server) + .await; + let (http, uri) = (reqwest::Client::builder().no_proxy().build()?, server.uri()); + let sync = || sync_gitlab_repo_with_client(&state, &source, &http, &uri); + let doc = |path: &str| { + let key = format!("gitlab:QuintinBotes/utopia-gitlab-fixture:{path}"); + let pool = pool.clone(); + async move { utopia_store::documents::find_by_external_key(&pool, sid, &key).await } + }; + let requests = || async { server.received_requests().await.unwrap() }; + + let first = sync().await?; + anyhow::ensure!(first.created == 3, "第一轮建了 {}", first.created); + for path in ["README.md", "docs/copy-of-guide.md", "docs/nested/deep.mdx"] { + anyhow::ensure!(doc(path).await?.is_some(), "{path} 该收"); + } + for path in [ + "docs/guide.md", + "big.md", + "notes/empty.md", + ".gitlab/notes.md", + "node_modules/pkg/README.md", + "vendor/lib/README.md", + "link.md", + "src/main.rs", + ] { + anyhow::ensure!(doc(path).await?.is_none(), "{path} 不该收"); + } + // 第二次提交的日期，毫秒截掉 + let time = doc("docs/copy-of-guide.md").await?.and_then(|d| d.doc_time); + anyhow::ensure!(time == Some("2026-10-05T14:30:00Z".parse()?), "{time:?}"); + let sent = requests().await; + let heads: Vec<_> = sent.iter().filter(|r| r.method == wiremock::http::Method::HEAD).collect(); + // README、copy-of-guide、deep.mdx、big.md；空文件按 SHA 挡掉，不问 + anyhow::ensure!(heads.len() == 4, "只有路径过了筛选的才问大小：{}", heads.len()); + let blobs = sent.iter().filter(|r| r.url.path().ends_with("/raw")).count(); + anyhow::ensure!(blobs == 3, "超限与空文件不该下载：取了 {blobs} 个"); + anyhow::ensure!( + sent.iter().filter(|r| r.url.path().ends_with("/tree")).count() == 2, + "树该翻两页" + ); + + let seen = requests().await.len(); + let second = sync().await?; + let again: Vec<_> = requests().await[seen..] + .iter() + .filter(|r| !r.url.path().ends_with("/tree")) + .map(|r| (r.method.to_string(), r.url.path().to_string())) + .collect(); + // 没变的一个都不再取；超限的 big.md 不落库，每轮再问一次大小，但不下载 + anyhow::ensure!( + second.total() == 0 + && again.len() == 1 + && again[0].0 == "HEAD" + && again[0].1.ends_with("/files/big.md"), + "没变又取了：{again:?}" + ); + *stage.lock().unwrap() = 1; + sync().await?; + let missing = || async { anyhow::Ok(doc("README.md").await?.unwrap().missing_since) }; + anyhow::ensure!(missing().await?.is_none(), "没翻完的树不该判消失"); + *stage.lock().unwrap() = 2; + sync().await?; + anyhow::ensure!(missing().await?.is_some(), "完整的树里没了才判消失"); + anyhow::ensure!(doc("docs/nested/deep.mdx").await?.unwrap().missing_since.is_none()); + anyhow::Ok(()) + } + .await; + sqlx::query("DELETE FROM knowledge_bases WHERE id=$1") + .bind(kb) + .execute(&pool) + .await?; + sqlx::query("DELETE FROM organizations WHERE id=$1") + .bind(org) + .execute(&pool) + .await?; + result +} diff --git a/crates/utopia-server/src/ingest_sources.rs b/crates/utopia-server/src/ingest_sources.rs index 26e64209d..114c6d635 100644 --- a/crates/utopia-server/src/ingest_sources.rs +++ b/crates/utopia-server/src/ingest_sources.rs @@ -84,6 +84,7 @@ pub async fn sync_source(state: &AppState, source_id: Uuid) -> anyhow::Result<() Some(SourceKind::Custom) => sync_custom(state, &source, since).await, Some(SourceKind::GithubIssues) => sync_github_issues(state, &source, since).await, Some(SourceKind::GithubRepo) => sync_github_repo(state, &source).await, + Some(SourceKind::GitlabRepo) => sync_gitlab_repo(state, &source).await, Some(SourceKind::JiraIssues) => sync_jira_issues(state, &source, since).await, Some(SourceKind::S3 | SourceKind::AzureBlob | SourceKind::Gcs) => { sync_object_storage(state, &source).await @@ -987,6 +988,183 @@ async fn sync_github_repo_with_client( Ok(stats) } +/// GitLab 项目文件（gitlab.com，#1327）：与 [`sync_github_repo`] 同一个形状，一条路径一篇文档， +/// 按 blob SHA 判「没变」。树要翻页；新出现或变了的文件多一次 `HEAD`，先拿到大小再决定取不取。 +async fn sync_gitlab_repo(state: &AppState, source: &Source) -> anyhow::Result { + let url = reqwest::Url::parse("https://gitlab.com/")?; + let http = crate::http_fetch::client_for( + &url, + crate::http_fetch::Reach::Content, + crate::http_fetch::Limits::default(), + ) + .await?; + sync_gitlab_repo_with_client(state, source, &http, "https://gitlab.com/api/v4").await +} + +/// 带令牌发一个请求并判状态。GitLab 限流回 429，说清楚是限流，正确的动作是等或配令牌 +async fn gitlab_checked( + http: &reqwest::Client, + method: reqwest::Method, + url: reqwest::Url, + auth: Option<&str>, +) -> anyhow::Result { + let mut req = http.request(method, url); + if let Some(a) = auth { + req = req.header(reqwest::header::AUTHORIZATION, a); + } + let resp = req.send().await?; + let status = resp.status(); + if status == reqwest::StatusCode::TOO_MANY_REQUESTS { + anyhow::bail!("GitLab 限流：请求太多，稍后再同步；未认证的配额更低，配一个 read_api 令牌"); + } + if !status.is_success() { + anyhow::bail!("HTTP {status} from GitLab"); + } + Ok(resp) +} + +// HTTP 接缝让回归测试走同一条同步与落库路径，生产固定访问 gitlab.com 的 API。 +async fn sync_gitlab_repo_with_client( + state: &AppState, + source: &Source, + http: &reqwest::Client, + api_base: &str, +) -> anyhow::Result { + use crate::{github_repo as gr, gitlab_repo as gl}; + let project = source.config["project"] + .as_str() + .map(str::trim) + .filter(|s| !s.is_empty()) + .ok_or_else(|| { + anyhow::anyhow!("gitlab_repo source is missing config.project (group/name)") + })?; + // 子组可以有好几层，但每一段都得有字 + if project.split('/').count() < 2 || project.split('/').any(str::is_empty) { + anyhow::bail!( + "config.project should look like group/name or group/subgroup/name, got {project:?}" + ); + } + let auth = source.config["auth_header"] + .as_str() + .map(str::trim) + .filter(|s| !s.is_empty()); + let filter = gr::FileFilter::from_config(&source.config)?; + let base = reqwest::Url::parse(api_base)?; + // 每一段各自编码：项目路径里的 `/` 与文件路径里的 `/` 都得成 `%2F` + let at = |segments: &[&str]| -> anyhow::Result { + let mut u = base.clone(); + u.path_segments_mut() + .map_err(|_| anyhow::anyhow!("API base cannot take a path"))? + .pop_if_empty() + .extend(["projects", project, "repository"]) + .extend(segments); + Ok(u) + }; + + let mut tree_url = at(&["tree"])?; + tree_url + .query_pairs_mut() + .append_pair("recursive", "true") + .append_pair("per_page", "100") + .append_pair("pagination", "keyset"); + let (mut entries, mut next, mut pages) = (Vec::::new(), Some(tree_url), 0); + while let Some(url) = next.take() { + if pages == gl::MAX_TREE_PAGES { + next = Some(url); + break; + } + let resp = gitlab_checked(http, reqwest::Method::GET, url, auth).await?; + let link = resp + .headers() + .get(reqwest::header::LINK) + .and_then(|v| v.to_str().ok()) + .map(String::from); + entries.extend(resp.json::>().await?); + next = gl::next_page(link.as_deref(), &base); + pages += 1; + } + let (truncated, chosen) = (next.is_some(), gl::choose(entries, &filter)); + let keys: Vec = chosen + .iter() + .map(|e| format!("gitlab:{project}:{}", e.path)) + .collect(); + let mut stats = SyncStats::default(); + for (entry, key) in chosen.iter().zip(&keys) { + // 没变的整条跳过：树里没有大小，但已落库的字节算出同一个 git SHA 就够了 + let old = + utopia_store::documents::find_by_external_key(&state.pool, source.id, key).await?; + let unchanged = match old { + Some(d) => state + .blob + .get(&d.sha256) + .await + .is_ok_and(|b| gr::git_blob_sha(&b) == entry.id), + None => false, + }; + if unchanged { + continue; + } + // 先问大小，大小上限得在下载之前判。超限的文件不落库，于是每一轮都会为它再问一次： + // 一次 HEAD，不下载正文。真撞上限流时，再把挡掉的 blob SHA 记下来 + let mut fu = at(&["files", &entry.path])?; + fu.query_pairs_mut().append_pair("ref", "HEAD"); + let head = gitlab_checked(http, reqwest::Method::HEAD, fu, auth).await?; + let size = head + .headers() + .get("x-gitlab-size") + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.parse::().ok()); + let Some(size) = size else { + tracing::warn!(%project, path = %entry.path, "GitLab 没有给出文件大小，跳过这个文件"); + continue; + }; + if !filter.size_ok(size) { + continue; + } + // 文档日期取最后一次动过这条路径的提交，只在新出现或变了时问 + let mut cu = at(&["commits"])?; + cu.query_pairs_mut() + .append_pair("path", &entry.path) + .append_pair("per_page", "1"); + let commits: serde_json::Value = gitlab_checked(http, reqwest::Method::GET, cu, auth) + .await? + .json() + .await?; + let doc_time = commits[0]["committed_date"] + .as_str() + .and_then(gl::commit_time); + let bytes = gitlab_checked( + http, + reqwest::Method::GET, + at(&["blobs", &entry.id, "raw"])?, + auth, + ) + .await? + .bytes() + .await?; + let action = ingest_item( + state, + source.kb_id, + source.id, + key, + &entry.path, + "text/markdown", + &bytes, + doc_time, + ) + .await?; + stats.absorb(action); + } + if truncated { + tracing::warn!(%project, pages = gl::MAX_TREE_PAGES, "GitLab 文件树没翻完，本轮只摄入列出的文件，不做不在来源中对账"); + } else { + utopia_store::documents::reconcile_missing(&state.pool, source.id, &keys) + .await + .map_err(|e| anyhow::anyhow!(e))?; + } + Ok(stats) +} + /// Jira 工单：一张工单一篇文档，正文里带**字段级**的变更史。 /// /// 比 GitHub 那条路省：`search?expand=changelog` 一次调用就带回工单本体、 @@ -1416,3 +1594,6 @@ mod github_sync_tests; #[cfg(test)] #[path = "github_repo_sync_tests.rs"] mod github_repo_sync_tests; +#[cfg(test)] +#[path = "gitlab_repo_sync_tests.rs"] +mod gitlab_repo_sync_tests; diff --git a/crates/utopia-server/src/main.rs b/crates/utopia-server/src/main.rs index 3ad724e30..a08b47d71 100644 --- a/crates/utopia-server/src/main.rs +++ b/crates/utopia-server/src/main.rs @@ -11,6 +11,7 @@ mod extraction; mod extraction_open; mod github_issues; mod github_repo; +mod gitlab_repo; mod governance; mod http_fetch; mod implication; diff --git a/crates/utopia-server/tests/fixtures/gitlab_repo.json b/crates/utopia-server/tests/fixtures/gitlab_repo.json new file mode 100644 index 000000000..686d22633 --- /dev/null +++ b/crates/utopia-server/tests/fixtures/gitlab_repo.json @@ -0,0 +1,215 @@ +{ + "note": "取自 gitlab.com/QuintinBotes/utopia-gitlab-fixture 的真实 GitLab 响应（#1327），条目和字段已裁到我们读的那些。sizes 是 HEAD files/:path?ref=HEAD 的 X-Gitlab-Size；commits 是 commits?path=…&per_page=1 的第一条；blobs 是 blobs/:sha/raw 的原文。", + "tree": [ + { + "id": "59b6118b73cbcb3122d252045b26d6c7d02efc94", + "path": ".gitlab", + "type": "tree", + "mode": "040000" + }, + { + "id": "59640446df63d4d2b440c3f6a507d8623c725b3c", + "path": "docs", + "type": "tree", + "mode": "040000" + }, + { + "id": "1874e3dcfe298a170dafc6ceb499f46f04151ba2", + "path": "docs/nested", + "type": "tree", + "mode": "040000" + }, + { + "id": "7a5166199491d8fa764ff635ac43f6fdbdd6c6a4", + "path": "node_modules", + "type": "tree", + "mode": "040000" + }, + { + "id": "d4e81d17fba4bf2637d5d5a2e73ebea2cdbe07f0", + "path": "node_modules/pkg", + "type": "tree", + "mode": "040000" + }, + { + "id": "244272f25fe72c258f73baecc92eb343a8bda9d7", + "path": "notes", + "type": "tree", + "mode": "040000" + }, + { + "id": "5d90422423db5ef6b431e8b9e60e0baf04b8742a", + "path": "src", + "type": "tree", + "mode": "040000" + }, + { + "id": "1bd4a12d379be6929bc1baa24b8e3cd01c398d53", + "path": "vendor", + "type": "tree", + "mode": "040000" + }, + { + "id": "d4e81d17fba4bf2637d5d5a2e73ebea2cdbe07f0", + "path": "vendor/lib", + "type": "tree", + "mode": "040000" + }, + { + "id": "3e426bd76155c7e0885219a9465f467c9cff59f2", + "path": ".gitlab/notes.md", + "type": "blob", + "mode": "100644" + }, + { + "id": "9e4eb8b5657c5b9f1abfbef693a6e743d92a1862", + "path": "README.md", + "type": "blob", + "mode": "100644" + }, + { + "id": "928fe85d3c0b086cc74cbd9acf419d075a1a1387", + "path": "big.md", + "type": "blob", + "mode": "100644" + }, + { + "id": "9292cffaa5dba9013a3b89201041c96584661f0a", + "path": "docs/copy-of-guide.md", + "type": "blob", + "mode": "100644" + }, + { + "id": "9292cffaa5dba9013a3b89201041c96584661f0a", + "path": "docs/guide.md", + "type": "blob", + "mode": "100644" + }, + { + "id": "f9b2af405d33985a3b1f3b21b3aae6b6c08fd6c9", + "path": "docs/nested/deep.mdx", + "type": "blob", + "mode": "100644" + }, + { + "id": "eafc2aac7162cad4e1f335fab78ae18b297e49cc", + "path": "link.md", + "type": "blob", + "mode": "120000" + }, + { + "id": "08e8dd66c3cd6625b1ef680260b6861732ed1ab9", + "path": "node_modules/pkg/README.md", + "type": "blob", + "mode": "100644" + }, + { + "id": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", + "path": "notes/empty.md", + "type": "blob", + "mode": "100644" + }, + { + "id": "f328e4d9d04c31d0d70d16d21a07d1613be9d577", + "path": "src/main.rs", + "type": "blob", + "mode": "100644" + }, + { + "id": "08e8dd66c3cd6625b1ef680260b6861732ed1ab9", + "path": "vendor/lib/README.md", + "type": "blob", + "mode": "100644" + } + ], + "sizes": { + ".gitlab/notes.md": 34, + "README.md": 61, + "big.md": 3368, + "docs/copy-of-guide.md": 51, + "docs/guide.md": 51, + "docs/nested/deep.mdx": 38, + "link.md": 13, + "node_modules/pkg/README.md": 11, + "notes/empty.md": 0, + "src/main.rs": 13, + "vendor/lib/README.md": 11 + }, + "commits": { + ".gitlab/notes.md": [ + { + "id": "8cdce7f6f4c5d3048293693813009afe9867448c", + "committed_date": "2026-10-01T09:00:00.000+00:00" + } + ], + "README.md": [ + { + "id": "8cdce7f6f4c5d3048293693813009afe9867448c", + "committed_date": "2026-10-01T09:00:00.000+00:00" + } + ], + "big.md": [ + { + "id": "8cdce7f6f4c5d3048293693813009afe9867448c", + "committed_date": "2026-10-01T09:00:00.000+00:00" + } + ], + "docs/copy-of-guide.md": [ + { + "id": "189e30cff625d4c238cfb1059941383b711b2a81", + "committed_date": "2026-10-05T14:30:00.000+00:00" + } + ], + "docs/guide.md": [ + { + "id": "189e30cff625d4c238cfb1059941383b711b2a81", + "committed_date": "2026-10-05T14:30:00.000+00:00" + } + ], + "docs/nested/deep.mdx": [ + { + "id": "8cdce7f6f4c5d3048293693813009afe9867448c", + "committed_date": "2026-10-01T09:00:00.000+00:00" + } + ], + "link.md": [ + { + "id": "8cdce7f6f4c5d3048293693813009afe9867448c", + "committed_date": "2026-10-01T09:00:00.000+00:00" + } + ], + "node_modules/pkg/README.md": [ + { + "id": "8cdce7f6f4c5d3048293693813009afe9867448c", + "committed_date": "2026-10-01T09:00:00.000+00:00" + } + ], + "notes/empty.md": [ + { + "id": "8cdce7f6f4c5d3048293693813009afe9867448c", + "committed_date": "2026-10-01T09:00:00.000+00:00" + } + ], + "src/main.rs": [ + { + "id": "8cdce7f6f4c5d3048293693813009afe9867448c", + "committed_date": "2026-10-01T09:00:00.000+00:00" + } + ], + "vendor/lib/README.md": [ + { + "id": "8cdce7f6f4c5d3048293693813009afe9867448c", + "committed_date": "2026-10-01T09:00:00.000+00:00" + } + ] + }, + "blobs": { + "3e426bd76155c7e0885219a9465f467c9cff59f2": "# Hidden\n\nInside a hidden folder.\n", + "9e4eb8b5657c5b9f1abfbef693a6e743d92a1862": "# Fixture project\n\nRecorded by the Utopia gitlab_repo tests.\n", + "928fe85d3c0b086cc74cbd9acf419d075a1a1387": "# Big\n\nLorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet. \n", + "9292cffaa5dba9013a3b89201041c96584661f0a": "# Guide\n\nThe guide a base should take in, revised.\n", + "f9b2af405d33985a3b1f3b21b3aae6b6c08fd6c9": "# Deep\n\nAn mdx file two folders down.\n", + "08e8dd66c3cd6625b1ef680260b6861732ed1ab9": "# Vendored\n", + "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391": "" + } +} \ No newline at end of file diff --git a/crates/utopia-store/tests/store/a_viewer_never_sees_a_credential.rs b/crates/utopia-store/tests/store/a_viewer_never_sees_a_credential.rs index 68674ec1b..edac1b87a 100644 --- a/crates/utopia-store/tests/store/a_viewer_never_sees_a_credential.rs +++ b/crates/utopia-store/tests/store/a_viewer_never_sees_a_credential.rs @@ -60,6 +60,10 @@ async fn a_viewer_never_sees_a_credential() -> anyhow::Result<()> { "github_repo", serde_json::json!({ "repo": "o/r", "auth_header": "Bearer r" }), ), + ( + "gitlab_repo", + serde_json::json!({ "project": "g/p", "auth_header": "Bearer l" }), + ), ( "jira_issues", serde_json::json!({ "base_url": "https://j.test", "project": "P", "auth_header": "Basic j" }), diff --git a/docs/design/sources.md b/docs/design/sources.md index d543cec15..0e6869d2f 100644 --- a/docs/design/sources.md +++ b/docs/design/sources.md @@ -21,7 +21,10 @@ keeps the stored value; a truncated round says so [0013]. A ticket is one docume carries its history as dated declarative sentences and a `## History` list; GitHub fetches events per ticket; timestamps are written to the second [0013]. A GitHub repository file is one document keyed by its path and dated by the last commit that touched it; a file whose blob SHA has not -changed is not fetched again, and identical files at several paths go in once [#1303]. +changed is not fetched again, and identical files at several paths go in once [#1303]. A GitLab project file +(gitlab.com; self-managed is not verified) takes the same shape; its tree is paged, so a tree that +runs past the page cap is a truncated round, and its entries carry no size, so the size cap is read +from a `HEAD` on the file before anything is downloaded [#1327]. **Statements.** A `statements` source takes the open extraction contract itself (`e`, `s`, `n`) on `POST /sources/{id}/statements`, with the `api` push's identity, versions, tombstones and run history; the payload is the document and its single chunk, and extraction parses the chunk instead of prompting a model, then runs the unchanged path, so a pushed statement is an open statement with the item as its evidence (null offsets) and reaches the typed graph only through alignment. The contract has no slot for a property or a class and unknown keys are refused; a second push under the same identity marks the earlier statements stale rather than closing them; tables belong on a mount, not here [0054]. diff --git a/web/src/api.ts b/web/src/api.ts index 47cf1bc88..6a66651f5 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -229,12 +229,12 @@ export interface SourceView { repo?: string; /** github_issues：PR 在 GitHub 模型里也是工单，默认不收 */ include_pull_requests?: boolean; - /** github_repo：给了就整列替换默认的 glob（markdown；隐藏目录、node_modules、vendor） */ + /** github_repo / gitlab_repo：给了就整列替换默认的 glob（markdown；隐藏目录、node_modules、vendor） */ include?: string[]; exclude?: string[]; /** jira_issues：站点地址，如 https://issues.apache.org/jira */ base_url?: string; - /** jira_issues：项目 key，如 KAFKA */ + /** jira_issues：项目 key，如 KAFKA；gitlab_repo：项目路径，如 group/subgroup/name */ project?: string; /** false = 这个来源下的文档只检索、不抽取（schema 文档，0035 决定 7）；缺省抽取 */ extract?: boolean; diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts index e425d569c..bda621ff9 100644 --- a/web/src/i18n/en.ts +++ b/web/src/i18n/en.ts @@ -641,6 +641,7 @@ export const en = { custom: "Custom", github_issues: "GitHub issues", github_repo: "GitHub repository files", + gitlab_repo: "GitLab project files", jira_issues: "Jira issues", s3: "S3 / MinIO", azure_blob: "Azure Blob", @@ -663,6 +664,10 @@ export const en = { github_repo: "Reads markdown files off a repository's default branch. Each file becomes one " + "document dated by its last commit; unchanged files are not fetched again.", + gitlab_repo: + "Reads markdown files off a gitlab.com project's default branch, the same way as " + + "GitHub repository files. A private project needs a token with read_api. " + + "Self-managed GitLab has not been verified.", s3: "Reads documents out of an S3 bucket, or anything speaking the same protocol " + "(MinIO, Ceph, R2). **Leave the endpoint empty for AWS**; fill it in for a " + @@ -741,6 +746,7 @@ export const en = { rssHydrationCounts: (c: { pending: number; queued: number; retrying: number; complete: number; terminal: number }) => `pending ${c.pending} · queued ${c.queued} · retrying ${c.retrying} · complete ${c.complete} · terminal ${c.terminal}`, repoField: "Repository (owner/name)", + gitlabProjectField: "Project (group/name, subgroups allowed)", jiraUrlField: "Jira site URL", jiraProjectField: "Project key", s3BucketField: "Bucket", @@ -759,6 +765,7 @@ export const en = { notionTokenField: "Internal integration token", notionQueryField: "Search term (optional — empty takes every shared page)", tokenField: "GitHub token (optional, never shown again)", + gitlabTokenField: "GitLab token with read_api (optional, never shown again)", includePullRequests: "Treat pull requests as tickets too", includeGlobsField: "Include globs (comma-separated, optional; replaces the default)", excludeGlobsField: "Exclude globs (comma-separated, optional; replaces the default)", diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.ts index 7b76fa52e..f776ef233 100644 --- a/web/src/i18n/zh.ts +++ b/web/src/i18n/zh.ts @@ -580,6 +580,7 @@ export const zh: Strings = { custom: "自定义", github_issues: "GitHub 工单", github_repo: "GitHub 仓库文件", + gitlab_repo: "GitLab 项目文件", jira_issues: "Jira 工单", s3: "S3 / MinIO", azure_blob: "Azure Blob", @@ -601,6 +602,9 @@ export const zh: Strings = { github_repo: "读取一个仓库默认分支上的 markdown 文件。每个文件成为一篇文档，" + "日期取最后一次改动它的提交；没变的文件不会重抓。", + gitlab_repo: + "读取一个 gitlab.com 项目默认分支上的 markdown 文件，做法与 GitHub 仓库文件相同。" + + "私有项目要一个带 read_api 的令牌。自建 GitLab 没有验证过。", s3: "从 S3 桶里读文档，或者任何说同一套协议的东西（MinIO、Ceph、R2）。" + "**端点留空就是 AWS**，填了就是自建。每个对象成为一篇文档，" + @@ -674,6 +678,7 @@ export const zh: Strings = { rssHydrationCounts: (c: { pending: number; queued: number; retrying: number; complete: number; terminal: number }) => `待处理 ${c.pending} · 排队 ${c.queued} · 重试 ${c.retrying} · 完成 ${c.complete} · 终止 ${c.terminal}`, repoField: "仓库（owner/name）", + gitlabProjectField: "项目（group/name，可带子组）", jiraUrlField: "Jira 地址", jiraProjectField: "项目 key", s3BucketField: "桶名", @@ -692,6 +697,7 @@ export const zh: Strings = { notionTokenField: "内部集成令牌", notionQueryField: "搜索词（可选——留空则取全部已分享页面）", tokenField: "GitHub 令牌（可选，不再显示）", + gitlabTokenField: "带 read_api 的 GitLab 令牌（可选，不再显示）", includePullRequests: "把 PR 也当工单收进来", includeGlobsField: "包含的 glob（逗号分隔，可选；填了就替换默认值）", excludeGlobsField: "排除的 glob（逗号分隔，可选；填了就替换默认值）", diff --git a/web/src/pages/Library.tsx b/web/src/pages/Library.tsx index 8062f446f..22badcaa5 100644 --- a/web/src/pages/Library.tsx +++ b/web/src/pages/Library.tsx @@ -1375,6 +1375,7 @@ function SourceModal({ kind === "custom" || kind === "github_issues" || kind === "github_repo" || + kind === "gitlab_repo" || kind === "jira_issues" || kind === "s3" || kind === "azure_blob" || @@ -1402,6 +1403,13 @@ function SourceModal({ ? includePrs ? { include_pull_requests: true } : {} : { include: globList(includeGlobs), exclude: globList(excludeGlobs) }), } + : kind === "gitlab_repo" + ? { + project: repo.trim(), + ...(authHeader.trim() ? { auth_header: authHeader.trim() } : {}), + include: globList(includeGlobs), + exclude: globList(excludeGlobs), + } : kind === "jira_issues" ? { base_url: jiraUrl.trim(), @@ -1806,21 +1814,21 @@ function SourceModal({ )} )} - {(kind === "github_issues" || kind === "github_repo") && ( + {(kind === "github_issues" || kind === "github_repo" || kind === "gitlab_repo") && ( <> {field( - S.library.repoField, + kind === "gitlab_repo" ? S.library.gitlabProjectField : S.library.repoField, setRepo(e.target.value)} />, )} {field( - S.library.tokenField, + kind === "gitlab_repo" ? S.library.gitlabTokenField : S.library.tokenField, setAuthHeader(e.target.value)} />, @@ -1888,7 +1896,8 @@ function SourceEditModal({ ); const [endpoint, setEndpoint] = useState(cfg.endpoint ?? ""); const [authHeader, setAuthHeader] = useState(""); - const [repo, setRepo] = useState(cfg.repo ?? ""); + // GitLab 的项目路径存在 `project`（与 Jira 的项目 key 同名不同义），表单里同用这一格 + const [repo, setRepo] = useState((kind === "gitlab_repo" ? cfg.project : cfg.repo) ?? ""); const [includePrs, setIncludePrs] = useState(Boolean(cfg.include_pull_requests)); const [includeGlobs, setIncludeGlobs] = useState((cfg.include ?? []).join(", ")); const [excludeGlobs, setExcludeGlobs] = useState((cfg.exclude ?? []).join(", ")); @@ -1911,8 +1920,8 @@ function SourceEditModal({ (kind === "github_issues" && (repo.trim() !== (cfg.repo ?? "") || includePrs !== Boolean(cfg.include_pull_requests))) || - (kind === "github_repo" && - (repo.trim() !== (cfg.repo ?? "") || + ((kind === "github_repo" || kind === "gitlab_repo") && + (repo.trim() !== ((kind === "gitlab_repo" ? cfg.project : cfg.repo) ?? "") || String(globList(includeGlobs) ?? "") !== String(cfg.include ?? "") || String(globList(excludeGlobs) ?? "") !== String(cfg.exclude ?? ""))); @@ -1936,13 +1945,12 @@ function SourceEditModal({ ...(authHeader.trim() ? { auth_header: authHeader.trim() } : {}), include_pull_requests: includePrs, } - : kind === "github_repo" - ? githubRepoConfig(cfg, { - repo, - authHeader, - include: includeGlobs, - exclude: excludeGlobs, - }) + : kind === "github_repo" || kind === "gitlab_repo" + ? githubRepoConfig( + cfg, + { repo, authHeader, include: includeGlobs, exclude: excludeGlobs }, + kind === "gitlab_repo" ? "project" : "repo", + ) : undefined; return api.updateSource(kbId, source.id, { name: name.trim(), @@ -2064,21 +2072,21 @@ function SourceEditModal({

)} - {(kind === "github_issues" || kind === "github_repo") && ( + {(kind === "github_issues" || kind === "github_repo" || kind === "gitlab_repo") && ( <> {field( - S.library.repoField, + kind === "gitlab_repo" ? S.library.gitlabProjectField : S.library.repoField, setRepo(e.target.value)} />, )} {field( - S.library.tokenField, + kind === "gitlab_repo" ? S.library.gitlabTokenField : S.library.tokenField, setAuthHeader(e.target.value)} />, diff --git a/web/src/pages/SourcesRail.tsx b/web/src/pages/SourcesRail.tsx index e881dfbad..5e9e5fe98 100644 --- a/web/src/pages/SourcesRail.tsx +++ b/web/src/pages/SourcesRail.tsx @@ -73,6 +73,7 @@ export const KIND_ICON: Record = { custom: Puzzle, github_issues: CircleDot, github_repo: FileText, + gitlab_repo: FileText, jira_issues: SquareKanban, s3: HardDrive, azure_blob: Cloud, @@ -90,6 +91,7 @@ export const SYNCING_KINDS = new Set([ "custom", "github_issues", "github_repo", + "gitlab_repo", "jira_issues", "s3", "azure_blob", diff --git a/web/src/pages/githubRepoConfig.test.ts b/web/src/pages/githubRepoConfig.test.ts index c97e2269a..24f270cab 100644 --- a/web/src/pages/githubRepoConfig.test.ts +++ b/web/src/pages/githubRepoConfig.test.ts @@ -15,3 +15,9 @@ it("keeps what the edit form does not show, and a cleared glob list falls back t max_file_bytes: 2000, }); }); + +it("a GitLab source keeps its project path under project", () => { + const saved = { project: "g/p", max_file_bytes: 2000 }; + const sent = githubRepoConfig(saved, { repo: " g/sub/p ", authHeader: "Bearer t", include: "", exclude: "" }, "project"); + expect(JSON.parse(JSON.stringify(sent))).toEqual({ project: "g/sub/p", auth_header: "Bearer t", max_file_bytes: 2000 }); +}); diff --git a/web/src/pages/githubRepoConfig.ts b/web/src/pages/githubRepoConfig.ts index 00399e56f..8cbd072dc 100644 --- a/web/src/pages/githubRepoConfig.ts +++ b/web/src/pages/githubRepoConfig.ts @@ -6,15 +6,17 @@ export const globList = (value: string) => { return list.length ? list : undefined; }; -/** 编辑 github_repo 来源时提交的配置。保存是整份替换，所以表单里不露面的键（比如 - * `max_file_bytes`）从库里那份原样带回去；令牌留空就不发，后端保留原值 */ +/** 编辑 github_repo / gitlab_repo 来源时提交的配置。保存是整份替换，所以表单里不露面的键 + * （比如 `max_file_bytes`）从库里那份原样带回去；令牌留空就不发，后端保留原值。 + * GitLab 把项目路径存在 `project`，GitHub 存在 `repo` */ export function githubRepoConfig( saved: NonNullable, form: { repo: string; authHeader: string; include: string; exclude: string }, + key: "repo" | "project" = "repo", ) { return { ...saved, - repo: form.repo.trim(), + [key]: form.repo.trim(), ...(form.authHeader.trim() ? { auth_header: form.authHeader.trim() } : {}), include: globList(form.include), exclude: globList(form.exclude), diff --git a/web/src/sourceKinds.test.ts b/web/src/sourceKinds.test.ts index e579cffbd..e3765b2a9 100644 --- a/web/src/sourceKinds.test.ts +++ b/web/src/sourceKinds.test.ts @@ -6,6 +6,9 @@ it.each([ ["github_repo", { repo: "" }, false], ["github_repo", { repo: "https://github.com/o/r" }, false], ["github_repo", { repo: " o/r " }, true], + ["gitlab_repo", { repo: "name" }, false], + ["gitlab_repo", { repo: "https://gitlab.com/g/p" }, false], + ["gitlab_repo", { repo: " group/sub/name " }, true], ["jira_issues", { jiraUrl: "https://j.test", jiraProject: "" }, false], ["jira_issues", { jiraUrl: "https://j.test", jiraProject: "P" }, true], ["folder", {}, true], diff --git a/web/src/sourceKinds.ts b/web/src/sourceKinds.ts index 6c410e2ab..3eb713d67 100644 --- a/web/src/sourceKinds.ts +++ b/web/src/sourceKinds.ts @@ -14,6 +14,7 @@ export const CREATABLE_SOURCE_KINDS = [ "rss", "github_issues", "github_repo", + "gitlab_repo", "jira_issues", "s3", "azure_blob", @@ -31,7 +32,7 @@ export type CreatableSourceKind = (typeof CREATABLE_SOURCE_KINDS)[number]; export type SourceKind = CreatableSourceKind | "memory" | "upload"; /** - * GitHub、Jira 来源同步缺不得的几项填了没有。缺了照样建得出来、存得下，之后每一轮同步 + * GitHub、GitLab、Jira 来源同步缺不得的几项填了没有。缺了照样建得出来、存得下，之后每一轮同步 * 都只会报错。仓库要是 `owner/name`：粘一整条 GitHub 链接进来，后端同样每轮拒收 */ export function hasSyncFields( @@ -41,6 +42,8 @@ export function hasSyncFields( if (kind === "github_issues" || kind === "github_repo") { return /^[^/\s]+\/[^/\s]+$/.test(f.repo?.trim() ?? ""); } + // GitLab 的项目路径可以有好几层子组：group/sub/name + if (kind === "gitlab_repo") return /^[^/\s]+(\/[^/\s]+)+$/.test(f.repo?.trim() ?? ""); if (kind === "jira_issues") return Boolean(f.jiraUrl?.trim() && f.jiraProject?.trim()); return true; } From 47894659d2fc7353a74c24f8a7b17132257c5cc4 Mon Sep 17 00:00:00 2001 From: Virtue039 <77153652+virtue039@users.noreply.github.com> Date: Fri, 23 Oct 2026 18:01:50 +0800 Subject: [PATCH 3/9] A regression case can expect no binding while a person's binding remains authoritative (#1334) * A regression case can expect no binding while a person's binding remains authoritative Signed-off-by: Virtue039 * Let negative regression cases report conflicts without vetoing adoption Signed-off-by: Virtue039 * Keep MCP test teardown from racing index background work Signed-off-by: Virtue039 --------- Signed-off-by: Virtue039 Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- crates/utopia-cli/src/main.rs | 2 +- crates/utopia-server/src/api/mcp_tests.rs | 12 +- .../src/api/ontology_regression_routes.rs | 27 ++- .../api/ontology_regression_routes_tests.rs | 164 ++++++++++++- crates/utopia-server/src/ontology_agent.rs | 10 +- .../utopia-server/src/ontology_agent_tests.rs | 131 +++++++++- .../src/phrase_alignment_tests.rs | 41 ++++ .../src/export_provenance_integrity.sql | 2 +- .../utopia-store/src/ontology_regressions.rs | 92 +++++--- .../tests/store/ontology_regressions.rs | 223 +++++++++++++++++- docs/design/ontology.md | 8 + ..._regression_case_can_expect_no_binding.sql | 12 + web/src/api.ts | 20 +- web/src/i18n/en.ts | 8 +- web/src/i18n/zh.ts | 8 +- web/src/pages/AddOntologyRegressionDialog.tsx | 20 +- .../pages/OntologyRegressionsPanel.test.ts | 23 +- web/src/pages/OntologyRegressionsPanel.tsx | 9 +- 18 files changed, 734 insertions(+), 78 deletions(-) create mode 100644 migrations/0118_a_regression_case_can_expect_no_binding.sql diff --git a/crates/utopia-cli/src/main.rs b/crates/utopia-cli/src/main.rs index 0d41dcf34..8f0fe0364 100644 --- a/crates/utopia-cli/src/main.rs +++ b/crates/utopia-cli/src/main.rs @@ -82,7 +82,7 @@ struct ManifestDataDir { /// not a side effect of a code change. // 是迁移文件的**个数**，不是最大的编号（守卫 `schema_version_policy_compares_against_current` // 按个数比）：编号有空缺时两者不同——0071 由一个开放 PR 占着，0072 先落，个数是 71 -const CURRENT_SCHEMA_VERSION: u32 = 104; +const CURRENT_SCHEMA_VERSION: u32 = 105; fn main() -> anyhow::Result<()> { dotenvy::dotenv().ok(); diff --git a/crates/utopia-server/src/api/mcp_tests.rs b/crates/utopia-server/src/api/mcp_tests.rs index 1c6f16659..ee16cd1b2 100644 --- a/crates/utopia-server/src/api/mcp_tests.rs +++ b/crates/utopia-server/src/api/mcp_tests.rs @@ -193,7 +193,17 @@ impl Fixture { .await?; let dir = self.dir.clone(); drop(self); - std::fs::remove_dir_all(dir)?; + // Tantivy can finish a background file operation after the state is dropped. + // Retry only that teardown race; persistent or unrelated IO errors still fail. + for attempt in 0..10 { + match std::fs::remove_dir_all(&dir) { + Ok(()) => break, + Err(e) if e.kind() == std::io::ErrorKind::DirectoryNotEmpty && attempt < 9 => { + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + Err(e) => return Err(e.into()), + } + } Ok(()) } } diff --git a/crates/utopia-server/src/api/ontology_regression_routes.rs b/crates/utopia-server/src/api/ontology_regression_routes.rs index 976b81b39..c4d01369e 100644 --- a/crates/utopia-server/src/api/ontology_regression_routes.rs +++ b/crates/utopia-server/src/api/ontology_regression_routes.rs @@ -11,11 +11,21 @@ use utopia_core::models::Role; use utopia_store::{access::require_kb, ontology_regressions as cases}; use uuid::Uuid; +#[derive(Default, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ExpectedBinding { + #[default] + Bound, + None, +} + #[derive(Deserialize)] pub struct Create { pub statement_id: Uuid, - pub relation_type_id: Uuid, - pub direction: String, + #[serde(default)] + pub expected_binding: ExpectedBinding, + pub relation_type_id: Option, + pub direction: Option, } pub async fn list( @@ -34,13 +44,24 @@ pub async fn create( Json(req): Json, ) -> ApiResult<(StatusCode, Json)> { require_kb(&state.pool, &user, kb, Role::Editor).await?; + let valid = match req.expected_binding { + ExpectedBinding::Bound => req.relation_type_id.is_some() && req.direction.is_some(), + ExpectedBinding::None => req.relation_type_id.is_none() && req.direction.is_none(), + }; + if !valid { + return Err(utopia_core::AppError::invalid( + "bad_case_expectation", + "A bound expectation needs a property and direction; a none expectation has neither.", + ) + .into()); + } let id = cases::add( &state.pool, kb, cases::NewCase { statement_id: req.statement_id, expected_property_id: req.relation_type_id, - expected_direction: &req.direction, + expected_direction: req.direction.as_deref(), created_by: user.id, origin: "person", }, diff --git a/crates/utopia-server/src/api/ontology_regression_routes_tests.rs b/crates/utopia-server/src/api/ontology_regression_routes_tests.rs index 6ba5da68b..fdcf9291b 100644 --- a/crates/utopia-server/src/api/ontology_regression_routes_tests.rs +++ b/crates/utopia-server/src/api/ontology_regression_routes_tests.rs @@ -16,23 +16,30 @@ async fn post( token: &str, statement: Uuid, property: Uuid, +) -> anyhow::Result { + post_body( + app, + kb, + token, + json!({ "statement_id": statement, + "relation_type_id": property, "direction": "forward" }), + ) + .await +} + +async fn post_body( + app: &Router, + kb: Uuid, + token: &str, + body: serde_json::Value, ) -> anyhow::Result { Ok(app .clone() .oneshot( - Request::builder() - .method("POST") - .uri(format!("/api/v1/kbs/{kb}/ontology/regressions")) + Request::post(format!("/api/v1/kbs/{kb}/ontology/regressions")) .header("content-type", "application/json") .header("authorization", format!("Bearer {token}")) - .body(Body::from( - json!({ - "statement_id": statement, - "relation_type_id": property, - "direction": "forward" - }) - .to_string(), - ))?, + .body(Body::from(body.to_string()))?, ) .await?) } @@ -134,12 +141,143 @@ async fn editor_adds_regression_case_while_viewer_and_cross_kb_statement_are_rej .await?; anyhow::ensure!(saved); + let negative = graph::insert_open_statement( + &pool, + kb, + subject, + "mentioned alongside", + graph::FactObject::Entity(object), + None, + 1.0, + ) + .await? + .0; + for fields in [ + json!({}), + json!({"expected_binding":"bound"}), + json!({"expected_binding":"unknown"}), + json!({"expected_binding":null}), + json!({"relation_type_id":property}), + json!({"direction":"forward"}), + json!({"expected_binding":"none","relation_type_id":property}), + json!({"expected_binding":"none","direction":"reverse"}), + json!({"relation_type_id":property,"direction":"sideways"}), + ] { + let mut body = fields; + body["statement_id"] = json!(negative); + // The JSON extractor rejects an explicit null enum before the handler. + let expected = if body.get("expected_binding") == Some(&serde_json::Value::Null) { + StatusCode::BAD_REQUEST + } else { + StatusCode::UNPROCESSABLE_ENTITY + }; + assert_eq!( + post_body(&app, kb, &editor_token, body.clone()) + .await? + .status(), + expected, + "{body}" + ); + } + let mut negative_id = serde_json::Value::Null; + for targets in [json!({}), json!({"relation_type_id":null,"direction":null})] { + let mut request = targets; + request["statement_id"] = json!(negative); + request["expected_binding"] = json!("none"); + let response = post_body(&app, kb, &editor_token, request).await?; + assert_eq!(response.status(), StatusCode::CREATED); + let body: serde_json::Value = + serde_json::from_slice(&to_bytes(response.into_body(), 65536).await?)?; + if negative_id.is_null() { + negative_id = body["id"].clone(); + } + assert_eq!(body["id"], negative_id); + } + for request in [ + json!({"statement_id":statement,"expected_binding":"none"}), + json!({"statement_id":negative,"relation_type_id":property,"direction":"forward"}), + ] { + let response = post_body(&app, kb, &editor_token, request).await?; + assert_eq!(response.status(), StatusCode::CREATED); + let body: serde_json::Value = + serde_json::from_slice(&to_bytes(response.into_body(), 65536).await?)?; + assert_ne!( + body["id"], negative_id, + "opposite cases do not overwrite a confirmation" + ); + } + let sig = utopia_store::phrase_bindings::signatures(&pool, kb) + .await? + .into_iter() + .find(|s| s.phrase == "mentioned alongside") + .unwrap(); + utopia_store::phrase_bindings::decide( + &pool, + kb, + &sig, + utopia_store::phrase_bindings::Decision { + relation_type_id: Some(property), + direction: Some("forward"), + status: "bound", + decided_by: "person", + votes: &serde_json::Value::Null, + basis: None, + marks: None, + marks_asked: false, + }, + ) + .await?; + // Negative expectations remain visible even though they have no property to join. + let response = app + .clone() + .oneshot( + Request::builder() + .uri(format!("/api/v1/kbs/{kb}/ontology/regressions")) + .header("authorization", format!("Bearer {viewer_token}")) + .body(Body::empty())?, + ) + .await?; + let body: serde_json::Value = + serde_json::from_slice(&to_bytes(response.into_body(), 65536).await?)?; + let case = body["cases"] + .as_array() + .unwrap() + .iter() + .find(|c| c["id"] == negative_id) + .unwrap(); + assert_eq!(case["expected_binding"], "none"); + for key in [ + "expected_property_id", + "expected_direction", + "expected_property_label", + "expected_property_key", + ] { + assert!(case[key].is_null(), "{key}: {case}"); + } + assert_eq!(case["last_result"]["conflict"], "human_binding"); + assert_eq!(case["last_result"]["passed"], false); + assert_eq!(case["last_result"]["actual_property_id"], json!(property)); + assert_eq!(case["actual_property_key"], "worksFor"); + assert_eq!(case["subject_label"], "Alice"); + assert_eq!(case["statement_id"], json!(negative)); + for (token, statement, expected) in [ (&viewer_token, statement, StatusCode::FORBIDDEN), (&editor_token, foreign_statement, StatusCode::NOT_FOUND), ] { let response = post(&app, kb, token, statement, property).await?; anyhow::ensure!(response.status() == expected); + assert_eq!( + post_body( + &app, + kb, + token, + json!({"statement_id":statement,"expected_binding":"none"}) + ) + .await? + .status(), + expected + ); } anyhow::Ok(()) } @@ -209,8 +347,8 @@ async fn viewer_reads_only_live_cases_in_the_authorized_kb() -> anyhow::Result<( ).await?.0; utopia_store::ontology_regressions::add(&pool, case_kb, utopia_store::ontology_regressions::NewCase { - statement_id: id, expected_property_id: expected, - expected_direction: "forward", created_by: viewer, origin: "person", + statement_id: id, expected_property_id: Some(expected), + expected_direction: Some("forward"), created_by: viewer, origin: "person", }, ).await?; if case_kb == kb { diff --git a/crates/utopia-server/src/ontology_agent.rs b/crates/utopia-server/src/ontology_agent.rs index 6b61a96b9..fe844de1d 100644 --- a/crates/utopia-server/src/ontology_agent.rs +++ b/crates/utopia-server/src/ontology_agent.rs @@ -1245,8 +1245,10 @@ async fn finish_adoption( } } for (statement_id, direction) in examples { - // Removed examples cannot be checked; never substitute another statement. - let live: Option = sqlx::query_scalar("SELECT id FROM facts WHERE kb_id=$1 AND id=$2 AND layer='open' AND invalidated_at IS NULL FOR SHARE") + // A negative case observes disagreement; it must not veto adoption or gain an + // automatic positive counterpart. Removed examples are skipped too, never replaced. + let live: Option = sqlx::query_scalar("SELECT f.id FROM facts f WHERE f.kb_id=$1 AND f.id=$2 AND f.layer='open' AND f.invalidated_at IS NULL + AND NOT EXISTS (SELECT 1 FROM ontology_regression_cases c WHERE c.kb_id=f.kb_id AND c.statement_id=f.id AND c.expected_property_id IS NULL) FOR SHARE OF f") .bind(kb_id).bind(statement_id).fetch_optional(&mut **tx).await?; if live.is_none() { continue; @@ -1256,8 +1258,8 @@ async fn finish_adoption( kb_id, utopia_store::ontology_regressions::NewCase { statement_id, - expected_property_id: property_id, - expected_direction: direction, + expected_property_id: Some(property_id), + expected_direction: Some(direction), created_by: actor, origin: "adoption", }, diff --git a/crates/utopia-server/src/ontology_agent_tests.rs b/crates/utopia-server/src/ontology_agent_tests.rs index e6073af9d..e7ac8ba9e 100644 --- a/crates/utopia-server/src/ontology_agent_tests.rs +++ b/crates/utopia-server/src/ontology_agent_tests.rs @@ -461,8 +461,8 @@ async fn an_unbound_shape_becomes_a_proposal_that_serves_a_question_and_adoption let cases = utopia_store::ontology_regressions::list(&f.pool, f.kb).await?; assert_eq!(cases.len(), 1); assert_eq!(cases[0].statement_id, example_id); - assert_eq!(cases[0].expected_property_id, id); - assert_eq!(cases[0].expected_direction, "forward"); + assert_eq!(cases[0].expected_property_id, Some(id)); + assert_eq!(cases[0].expected_direction.as_deref(), Some("forward")); assert_eq!(cases[0].created_by, Some(f.user)); assert_eq!(cases[0].origin, "adoption"); assert!( @@ -1214,3 +1214,130 @@ async fn a_class_deleted_during_adoption_answers_unknown_class() -> anyhow::Resu assert_eq!(f.left_by("city").await?, (false, "open".to_string(), 0, 0)); f.cleanup().await } + +#[tokio::test] +async fn adoption_skips_negative_examples_without_vetoing_creation_or_reuse() -> anyhow::Result<()> +{ + for (section, key) in [("relation_types", "founded_in"), ("map_to", "located_in")] { + for ordinary_example in [false, true] { + let Some(f) = Fx::new(Vec::new()).await? else { + return Ok(()); + }; + let result = async { + f.decide_none().await?; + let negative = f.shape("founded in").await?; + let statement_id = + serde_json::from_value(negative["example_statement_ids"][0].clone())?; + let case = utopia_store::ontology_regressions::add( + &f.pool, + f.kb, + utopia_store::ontology_regressions::NewCase { + statement_id, + expected_property_id: None, + expected_direction: None, + created_by: f.user, + origin: "person", + }, + ) + .await?; + let before = utopia_store::ontology_regressions::get(&f.pool, f.kb, case) + .await? + .unwrap(); + let mut shapes = vec![negative]; + if ordinary_example { + shapes.push(f.shape("located in").await?); + } + f.seed( + section, + key, + if section == "map_to" { + json!({"kind":"relation"}) + } else { + founded_in() + }, + json!({"phrases":shapes}), + ) + .await?; + let (status, body) = f + .post( + "/ontology/proposals/adopt", + json!({"section":section,"key":key}), + ) + .await?; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!( + f.left_by(key).await?, + ( + true, + "adopted".into(), + if section == "map_to" { 1 } else { 2 }, + 1 + i64::from(ordinary_example) + ) + ); + let positive: i64 = sqlx::query_scalar( + "SELECT count(*) FROM ontology_regression_cases + WHERE kb_id=$1 AND statement_id=$2 AND expected_property_id IS NOT NULL", + ) + .bind(f.kb) + .bind(statement_id) + .fetch_one(&f.pool) + .await?; + assert_eq!( + positive, 0, + "negative examples do not gain an automatic positive case" + ); + let property: Uuid = serde_json::from_value(body["id"].clone())?; + // Creation queues alignment; exercise that later committed decision without a model. + // Reuse has already written the person's binding in the adoption transaction. + if section != "map_to" { + let sig = phrase_bindings::signatures(&f.pool, f.kb) + .await? + .into_iter() + .find(|s| s.phrase == "founded in") + .unwrap(); + phrase_bindings::decide( + &f.pool, + f.kb, + &sig, + Decision { + relation_type_id: Some(property), + direction: Some("forward"), + status: "bound", + decided_by: "agent", + votes: &Value::Null, + basis: None, + marks: None, + marks_asked: false, + }, + ) + .await?; + } + let after = utopia_store::ontology_regressions::get(&f.pool, f.kb, case) + .await? + .unwrap(); + assert_eq!( + (after.created_by, after.created_at, after.origin), + (before.created_by, before.created_at, before.origin) + ); + let report = after.last_result.unwrap(); + assert_eq!(report["passed"], false); + assert_eq!(report["actual_property_id"], json!(property)); + assert_eq!( + report["conflict"], + if section == "map_to" { + json!("human_binding") + } else { + Value::Null + } + ); + assert!(f.model.requests.lock().unwrap().is_empty()); + assert_eq!(f.model.embeds.load(Ordering::SeqCst), 0); + anyhow::Ok(()) + } + .await; + f.cleanup().await?; + result?; + } + } + Ok(()) +} diff --git a/crates/utopia-server/src/phrase_alignment_tests.rs b/crates/utopia-server/src/phrase_alignment_tests.rs index 5d1a365dc..126ff6436 100644 --- a/crates/utopia-server/src/phrase_alignment_tests.rs +++ b/crates/utopia-server/src/phrase_alignment_tests.rs @@ -1470,6 +1470,47 @@ async fn asking_what_a_date_marks_never_reopens_the_binding() -> anyhow::Result< run } +#[tokio::test] +async fn a_negative_case_follows_ordinary_rejudgement_and_keeps_human_binding() -> anyhow::Result<()> +{ + let Some(f) = Fx::new().await? else { + return Ok(()); + }; + let run = async { + use utopia_store::ontology_regressions as cases; + let actor = f.editor().await?.id; + let statement_id = sqlx::query_scalar("SELECT id FROM facts WHERE kb_id=$1 AND layer='open'") + .bind(f.kb).fetch_one(&f.pool).await?; + let id = cases::add(&f.pool,f.kb,cases::NewCase { statement_id, expected_property_id: None, + expected_direction: None, created_by: actor, origin: "person" }).await?; + f.script(none()); f.run().await?; + assert_eq!(cases::get(&f.pool,f.kb,id).await?.unwrap().last_result.unwrap()["passed"],true); + assert_eq!(f.requests().len(),1); + f.clear_jobs().await?; + sqlx::query("UPDATE relation_types SET description='New definition',updated_at=clock_timestamp() WHERE id=$1") + .bind(f.based_in).execute(&f.pool).await?; + f.script(bound()); f.run().await?; + let result = cases::get(&f.pool,f.kb,id).await?.unwrap().last_result.unwrap(); + assert_eq!(result["status"],"bound"); assert_eq!(result["passed"],false); + assert_eq!(f.requests().len(),3,"cases do not add model calls"); + let sig = phrase_bindings::signatures(&f.pool,f.kb).await?.remove(0); + phrase_bindings::decide(&f.pool,f.kb,&sig,phrase_bindings::Decision { + relation_type_id:Some(f.based_in),direction:Some("forward"),status:"bound",decided_by:"person", + votes:&Value::Null,basis:None,marks:None,marks_asked:false, + }).await?; + let before = f.binding().await?; + f.run().await?; + let result = cases::get(&f.pool,f.kb,id).await?.unwrap().last_result.unwrap(); + assert_eq!(result["passed"],false); assert_eq!(result["conflict"],"human_binding"); + assert_eq!(f.requests().len(),3); + let after = f.binding().await?; + assert_eq!(before.decided_at,after.decided_at); assert_eq!(after.relation_type_id,Some(f.based_in)); + anyhow::Ok(()) + }.await; + f.cleanup().await?; + run +} + #[tokio::test] async fn a_base_with_no_properties_queues_propose_ontology_once_twenty_signatures_are_unbound( ) -> anyhow::Result<()> { diff --git a/crates/utopia-store/src/export_provenance_integrity.sql b/crates/utopia-store/src/export_provenance_integrity.sql index e6c85f32c..c4c894832 100644 --- a/crates/utopia-store/src/export_provenance_integrity.sql +++ b/crates/utopia-store/src/export_provenance_integrity.sql @@ -339,7 +339,7 @@ SELECT edge, kind, COUNT(*) AS rows FROM ( SELECT 'regression.property', 'cross_kb', p.kb_id IS DISTINCT FROM c.kb_id FROM ontology_regression_cases c LEFT JOIN relation_types p ON p.id = c.expected_property_id - WHERE c.kb_id = $1 + WHERE c.kb_id = $1 AND c.expected_property_id IS NOT NULL UNION ALL -- @edge phrase_readings.entity_id -> entities.id -- 读数缓存：答出的实体喂给蕴含——别库的实体等于把别库语义读进本库 diff --git a/crates/utopia-store/src/ontology_regressions.rs b/crates/utopia-store/src/ontology_regressions.rs index d546e625f..953f6c3c7 100644 --- a/crates/utopia-store/src/ontology_regressions.rs +++ b/crates/utopia-store/src/ontology_regressions.rs @@ -12,8 +12,9 @@ pub struct Case { pub id: Uuid, pub kb_id: Uuid, pub statement_id: Uuid, - pub expected_property_id: Uuid, - pub expected_direction: String, + pub expected_binding: String, + pub expected_property_id: Option, + pub expected_direction: Option, pub created_by: Option, pub created_at: DateTime, pub origin: String, @@ -23,8 +24,8 @@ pub struct Case { pub struct NewCase<'a> { pub statement_id: Uuid, - pub expected_property_id: Uuid, - pub expected_direction: &'a str, + pub expected_property_id: Option, + pub expected_direction: Option<&'a str>, pub created_by: Uuid, pub origin: &'a str, } @@ -40,8 +41,8 @@ pub struct CaseView { pub object_id: Option, pub object_label: Option, pub object_value: Option, - pub expected_property_label: String, - pub expected_property_key: String, + pub expected_property_label: Option, + pub expected_property_key: Option, pub actual_property_label: Option, pub actual_property_key: Option, pub created_by_label: Option, @@ -50,7 +51,8 @@ pub struct CaseView { /// The workbench reads saved results and live source labels without evaluating or writing. pub async fn views(pool: &PgPool, kb_id: Uuid) -> AppResult> { Ok(sqlx::query_as( - "SELECT c.*, f.subject_id, s.canonical_name AS subject_label, f.phrase, + "SELECT c.*, CASE WHEN c.expected_property_id IS NULL THEN 'none' ELSE 'bound' END AS expected_binding, + f.subject_id, s.canonical_name AS subject_label, f.phrase, f.object_id, o.canonical_name AS object_label, f.object_value, expected.label AS expected_property_label, expected.key AS expected_property_key, actual.label AS actual_property_label, actual.key AS actual_property_key, @@ -59,7 +61,7 @@ pub async fn views(pool: &PgPool, kb_id: Uuid) -> AppResult> { JOIN facts f ON f.kb_id=c.kb_id AND f.id=c.statement_id JOIN entities s ON s.kb_id=f.kb_id AND s.id=f.subject_id LEFT JOIN entities o ON o.kb_id=f.kb_id AND o.id=f.object_id - JOIN relation_types expected ON expected.kb_id=c.kb_id AND expected.id=c.expected_property_id + LEFT JOIN relation_types expected ON expected.kb_id=c.kb_id AND expected.id=c.expected_property_id LEFT JOIN relation_types actual ON actual.kb_id=c.kb_id AND actual.id=(c.last_result->>'actual_property_id')::uuid LEFT JOIN users u ON u.id=c.created_by @@ -73,7 +75,8 @@ pub async fn views(pool: &PgPool, kb_id: Uuid) -> AppResult> { pub async fn list(pool: &PgPool, kb_id: Uuid) -> AppResult> { Ok(sqlx::query_as( - "SELECT c.* FROM ontology_regression_cases c WHERE c.kb_id=$1 + "SELECT c.*, CASE WHEN c.expected_property_id IS NULL THEN 'none' ELSE 'bound' END AS expected_binding + FROM ontology_regression_cases c WHERE c.kb_id=$1 AND EXISTS(SELECT 1 FROM facts f WHERE f.id=c.statement_id AND f.kb_id=c.kb_id AND f.layer='open' AND f.invalidated_at IS NULL) ORDER BY c.created_at DESC,c.id", ) @@ -84,7 +87,8 @@ pub async fn list(pool: &PgPool, kb_id: Uuid) -> AppResult> { pub async fn get(pool: &PgPool, kb_id: Uuid, id: Uuid) -> AppResult> { Ok( - sqlx::query_as("SELECT * FROM ontology_regression_cases WHERE kb_id=$1 AND id=$2") + sqlx::query_as("SELECT *, CASE WHEN expected_property_id IS NULL THEN 'none' ELSE 'bound' END AS expected_binding + FROM ontology_regression_cases WHERE kb_id=$1 AND id=$2") .bind(kb_id) .bind(id) .fetch_optional(pool) @@ -102,10 +106,13 @@ pub async fn add(pool: &PgPool, kb_id: Uuid, case: NewCase<'_>) -> AppResult) -> AppResult { - if !matches!(case.expected_direction, "forward" | "reverse") { + if !matches!( + (case.expected_property_id, case.expected_direction), + (None, None) | (Some(_), Some("forward" | "reverse")) + ) { return Err(AppError::invalid( - "bad_direction", - "Direction is forward or reverse.", + "bad_case_expectation", + "A bound expectation needs a property and direction; a none expectation has neither.", )); } if !matches!(case.origin, "adoption" | "person") { @@ -124,33 +131,45 @@ pub async fn add_on(conn: &mut PgConnection, kb_id: Uuid, case: NewCase<'_>) -> .fetch_optional(&mut *conn) .await?; let value = value.ok_or(AppError::NotFound)?; - let kind: Option = - sqlx::query_scalar("SELECT kind FROM relation_types WHERE kb_id=$1 AND id=$2 FOR SHARE") - .bind(kb_id) - .bind(case.expected_property_id) - .fetch_optional(&mut *conn) - .await?; - let kind = kind.ok_or(AppError::NotFound)?; - if value != (kind == "attribute") || (value && case.expected_direction != "forward") { - return Err(AppError::invalid( - "bad_property_shape", - "The property's kind and direction must fit the statement's object.", - )); + if let Some(property) = case.expected_property_id { + let kind: Option = sqlx::query_scalar( + "SELECT kind FROM relation_types WHERE kb_id=$1 AND id=$2 FOR SHARE", + ) + .bind(kb_id) + .bind(property) + .fetch_optional(&mut *conn) + .await?; + let kind = kind.ok_or(AppError::NotFound)?; + if value != (kind == "attribute") || (value && case.expected_direction != Some("forward")) { + return Err(AppError::invalid( + "bad_property_shape", + "The property's kind and direction must fit the statement's object.", + )); + } } // DO NOTHING followed by SELECT observes a concurrent inserter after its commit, // without turning an idempotent retry into an edit of the original confirmation. - let inserted:Option=sqlx::query_scalar("INSERT INTO ontology_regression_cases + let inserted: Option = sqlx::query_scalar( + "INSERT INTO ontology_regression_cases (id,kb_id,statement_id,expected_property_id,expected_direction,created_by,origin) VALUES($1,$2,$3,$4,$5,$6,$7) - ON CONFLICT(kb_id,statement_id,expected_property_id,expected_direction) DO NOTHING RETURNING id") - .bind(Uuid::now_v7()).bind(kb_id).bind(case.statement_id).bind(case.expected_property_id) - .bind(case.expected_direction).bind(case.created_by).bind(case.origin).fetch_optional(&mut *conn).await?; + ON CONFLICT DO NOTHING RETURNING id", + ) + .bind(Uuid::now_v7()) + .bind(kb_id) + .bind(case.statement_id) + .bind(case.expected_property_id) + .bind(case.expected_direction) + .bind(case.created_by) + .bind(case.origin) + .fetch_optional(&mut *conn) + .await?; let id = match inserted { Some(id) => id, None => { sqlx::query_scalar( "SELECT id FROM ontology_regression_cases WHERE kb_id=$1 AND statement_id=$2 - AND expected_property_id=$3 AND expected_direction=$4", + AND expected_property_id IS NOT DISTINCT FROM $3 AND expected_direction IS NOT DISTINCT FROM $4", ) .bind(kb_id) .bind(case.statement_id) @@ -173,15 +192,18 @@ pub async fn add_on(conn: &mut PgConnection, kb_id: Uuid, case: NewCase<'_>) -> } // The live statement determines the signature, exactly as production alignment does. -// A person's binding remains authoritative even when it differs from the example; the -// human_bound flag distinguishes that protection from an independent machine success. +// Comparing a case never changes a binding. Only positive cases retain the human-bound +// exception; a negative case records that disagreement explicitly as a failure. fn record_sql(fresh: bool) -> String { let checked_at = if fresh { "now()" } else { "b.decided_at" }; format!("UPDATE ontology_regression_cases c SET last_checked_at={checked_at}, - last_result=jsonb_build_object('passed',coalesce((b.decided_by='person' AND b.status='bound') OR - (b.status='bound' AND b.relation_type_id=c.expected_property_id AND b.direction=c.expected_direction),false), - 'human_bound',b.decided_by='person' AND b.status='bound','actual_property_id',b.relation_type_id, + last_result=jsonb_build_object('passed',coalesce(CASE WHEN c.expected_property_id IS NULL THEN b.status='none' + ELSE (b.decided_by='person' AND b.status='bound') OR + (b.status='bound' AND b.relation_type_id=c.expected_property_id AND b.direction=c.expected_direction) END,false), + 'human_bound',c.expected_property_id IS NOT NULL AND b.decided_by='person' AND b.status='bound', + 'conflict',CASE WHEN c.expected_property_id IS NULL AND b.decided_by='person' AND b.status='bound' THEN 'human_binding' END, + 'actual_property_id',b.relation_type_id, 'actual_direction',b.direction,'status',b.status,'decided_at',b.decided_at) FROM facts f JOIN entities s ON s.id=f.subject_id AND s.kb_id=f.kb_id LEFT JOIN entities o ON o.id=f.object_id AND o.kb_id=f.kb_id diff --git a/crates/utopia-store/tests/store/ontology_regressions.rs b/crates/utopia-store/tests/store/ontology_regressions.rs index c9bb42848..c1c265ca4 100644 --- a/crates/utopia-store/tests/store/ontology_regressions.rs +++ b/crates/utopia-store/tests/store/ontology_regressions.rs @@ -39,7 +39,7 @@ async fn committed_decisions_report_matches_and_human_bindings() -> anyhow::Resu &pool, kb, subject, "acquired", graph::FactObject::Entity(object), None, 1.0, ).await?.0; let case_id = regressions::add(&pool, kb, regressions::NewCase { - statement_id, expected_property_id: property, expected_direction: "forward", + statement_id, expected_property_id: Some(property), expected_direction: Some("forward"), created_by: actor, origin: "person", }).await?; let signature = phrase_bindings::signatures(&pool, kb).await?.remove(0); @@ -68,3 +68,224 @@ async fn committed_decisions_report_matches_and_human_bindings() -> anyhow::Resu .await?; result } + +struct Fx { + pool: PgPool, + org: Uuid, + kb: Uuid, + actor: Uuid, + subject: Uuid, + object: Uuid, + property: Uuid, +} +impl Fx { + async fn new() -> anyhow::Result> { + let Some(url) = utopia_store::test_db::url() else { + return Ok(None); + }; + let pool = PgPool::connect(&url).await?; + utopia_store::db::migrate(&pool).await?; + let (org, workspace, kb, actor, subject, object, property) = ( + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + ); + sqlx::query("INSERT INTO organizations(id,name) VALUES($1,'regression-cases')") + .bind(org) + .execute(&pool) + .await?; + sqlx::query("INSERT INTO workspaces(id,org_id,name) VALUES($1,$2,'regression-cases')") + .bind(workspace) + .bind(org) + .execute(&pool) + .await?; + sqlx::query( + "INSERT INTO knowledge_bases(id,workspace_id,name) VALUES($1,$2,'regression-cases')", + ) + .bind(kb) + .bind(workspace) + .execute(&pool) + .await?; + sqlx::query("INSERT INTO users(id,org_id,email,password_hash,display_name) VALUES($1,$2,$3,'fixture','Reviewer')") + .bind(actor).bind(org).bind(format!("{actor}@example.test")).execute(&pool).await?; + for (id, name) in [(subject, "A"), (object, "B")] { + sqlx::query("INSERT INTO entities(id,kb_id,canonical_name) VALUES($1,$2,$3)") + .bind(id) + .bind(kb) + .bind(name) + .execute(&pool) + .await?; + } + sqlx::query( + "INSERT INTO relation_types(id,kb_id,key,label) VALUES($1,$2,'acquired','acquired')", + ) + .bind(property) + .bind(kb) + .execute(&pool) + .await?; + Ok(Some(Self { + pool, + org, + kb, + actor, + subject, + object, + property, + })) + } +} + +#[tokio::test] +async fn negative_cases_fail_on_human_bindings_and_pass_only_explicit_none() -> anyhow::Result<()> { + let Some(Fx { + pool, + org, + kb, + actor, + subject, + object, + property, + }) = Fx::new().await? + else { + return Ok(()); + }; + let result = async { + let statement_id = graph::insert_open_statement( + &pool, kb, subject, "acquired", graph::FactObject::Entity(object), None, 1.0, + ).await?.0; + let case_id = regressions::add(&pool, kb, regressions::NewCase { + statement_id, expected_property_id: Some(property), expected_direction: Some("forward"), + created_by: actor, origin: "person", + }).await?; + // Negative expectations compare the same committed ledger, including human choices. + let negative_statement = graph::insert_open_statement( + &pool, kb, subject, "reported alongside", graph::FactObject::Entity(object), None, 1.0, + ).await?.0; + let negative = || regressions::NewCase { statement_id: negative_statement, + expected_property_id: None, expected_direction: None, created_by: actor, origin: "person" }; + let negative_id = regressions::add(&pool, kb, negative()).await?; + let unchecked = regressions::get(&pool, kb, negative_id).await?.unwrap(); + assert_eq!(unchecked.expected_binding, "none"); + assert!(unchecked.last_result.is_none() && unchecked.last_checked_at.is_none()); + let sig = phrase_bindings::signatures(&pool, kb).await?.into_iter() + .find(|s| s.phrase == "reported alongside").unwrap(); + for (status, by, passed, conflict) in [ + ("undecided", "agent", false, false), ("bound", "agent", false, false), + ("none", "agent", true, false), ("bound", "person", false, true), + ("none", "person", true, false), + ] { + let bound = status == "bound"; + phrase_bindings::decide(&pool, kb, &sig, phrase_bindings::Decision { + relation_type_id: bound.then_some(property), direction: bound.then_some("forward"), + status, decided_by: by, votes: &serde_json::Value::Null, basis: None, + marks: None, marks_asked: false, + }).await?; + regressions::record_human(&pool, kb).await?; + let result = regressions::get(&pool, kb, negative_id).await?.unwrap().last_result.unwrap(); + assert_eq!(result["passed"], passed); + assert_eq!(result["human_bound"], false); + assert_eq!(result["conflict"], if conflict { serde_json::json!("human_binding") } else { serde_json::Value::Null }); + assert_eq!(result["actual_property_id"], serde_json::json!(bound.then_some(property))); + let stored: (String, Option, String) = sqlx::query_as( + "SELECT status,relation_type_id,decided_by FROM phrase_bindings WHERE kb_id=$1 AND phrase=$2" + ).bind(kb).bind(&sig.phrase).fetch_one(&pool).await?; + assert_eq!(stored, (status.into(), bound.then_some(property), by.into()), "comparison never rewrites a binding"); + } + // A new example of an already decided signature reports the original decision time. + let later_statement = graph::insert_open_statement( + &pool, kb, object, "reported alongside", graph::FactObject::Entity(subject), None, 1.0, + ).await?.0; + let later = regressions::add(&pool, kb, regressions::NewCase { statement_id: later_statement, ..negative() }).await?; + let original_time: bool = sqlx::query_scalar("SELECT c.last_checked_at=b.decided_at AND (c.last_result->>'decided_at')::timestamptz=b.decided_at + FROM ontology_regression_cases c JOIN phrase_bindings b ON b.kb_id=c.kb_id AND b.phrase='reported alongside' WHERE c.id=$1") + .bind(later).fetch_one(&pool).await?; + assert!(original_time); + let mut tx = pool.begin().await?; + utopia_store::export::provenance_integrity(&mut tx, kb).await?; + tx.rollback().await?; + sqlx::query("DELETE FROM relation_types WHERE id=$1").bind(property).execute(&pool).await?; + assert!(regressions::get(&pool, kb, case_id).await?.is_none()); + assert!(regressions::get(&pool, kb, negative_id).await?.is_some()); + anyhow::Ok(()) + }.await; + sqlx::query("DELETE FROM organizations WHERE id=$1") + .bind(org) + .execute(&pool) + .await?; + result +} + +#[tokio::test] +async fn confirmations_deduplicate_without_vetoing_opposite_expectations() -> anyhow::Result<()> { + let Some(Fx { + pool, + org, + kb, + actor, + subject, + object, + property, + }) = Fx::new().await? + else { + return Ok(()); + }; + let result = async { + + let statement = graph::insert_open_statement(&pool, kb, subject, "alongside", + graph::FactObject::Entity(object), None, 1.0).await?.0; + let other = Uuid::now_v7(); + sqlx::query("INSERT INTO users(id,org_id,email,password_hash,display_name) VALUES($1,$2,$3,'fixture','Other')") + .bind(other).bind(org).bind(format!("{other}@example.test")).execute(&pool).await?; + let case = |statement_id, negative: bool, created_by, origin| regressions::NewCase { + statement_id, expected_property_id: (!negative).then_some(property), + expected_direction: (!negative).then_some("forward"), created_by, origin, + }; + let first = regressions::add(&pool, kb, case(statement, true, actor, "person")).await?; + let (a,b) = tokio::join!(regressions::add(&pool,kb,case(statement,true,other,"person")), + regressions::add(&pool,kb,case(statement,true,actor,"person"))); + assert_eq!((a?, b?), (first, first)); + let saved = regressions::get(&pool,kb,first).await?.unwrap(); + assert_eq!(saved.created_by,Some(actor)); assert_eq!(saved.origin,"person"); + let fresh = graph::insert_open_statement(&pool,kb,subject,"first confirmations", + graph::FactObject::Entity(object),None,1.0).await?.0; + let (a,b) = tokio::join!(regressions::add(&pool,kb,case(fresh,true,actor,"person")), + regressions::add(&pool,kb,case(fresh,true,other,"person"))); + let a = a?; assert_eq!(a,b?); + let winner = regressions::get(&pool,kb,a).await?.unwrap(); + assert!((winner.created_by==Some(actor) && winner.origin=="person") || + (winner.created_by==Some(other) && winner.origin=="person")); + // Cases observe decisions: opposite expectations can coexist in either order + // and under concurrent confirmation, without rewriting either person's case. + for (i, first_negative) in [Some(true), Some(false), None].into_iter().enumerate() { + let statement = graph::insert_open_statement(&pool,kb,subject,&format!("case {i}"), + graph::FactObject::Entity(object),None,1.0).await?.0; + if let Some(negative) = first_negative { + regressions::add(&pool,kb,case(statement,negative,actor,"person")).await?; + regressions::add(&pool,kb,case(statement,!negative,other,"person")).await?; + } else { + let (a,b) = tokio::join!(regressions::add(&pool,kb,case(statement,true,actor,"person")), + regressions::add(&pool,kb,case(statement,false,other,"person"))); + assert_ne!(a?, b?); + } + let count: i64 = sqlx::query_scalar("SELECT count(*) FROM ontology_regression_cases WHERE statement_id=$1") + .bind(statement).fetch_one(&pool).await?; + assert_eq!(count,2); + } + // Storage itself rejects half targets and duplicate NULL targets, not just the route. + for (target,direction,code) in [(Some(property),None,"23514"),(None,Some("forward"),"23514"),(None,None,"23505")] { + let err = sqlx::query("INSERT INTO ontology_regression_cases(id,kb_id,statement_id,expected_property_id,expected_direction,origin) VALUES($1,$2,$3,$4,$5,'person')") + .bind(Uuid::now_v7()).bind(kb).bind(statement).bind(target).bind(direction).execute(&pool).await.unwrap_err(); + assert_eq!(err.as_database_error().and_then(|e|e.code()).as_deref(),Some(code)); + } + anyhow::Ok(()) + }.await; + sqlx::query("DELETE FROM organizations WHERE id=$1") + .bind(org) + .execute(&pool) + .await?; + result +} diff --git a/docs/design/ontology.md b/docs/design/ontology.md index 340ea8be1..cf1c3ae03 100644 --- a/docs/design/ontology.md +++ b/docs/design/ontology.md @@ -254,6 +254,14 @@ the prompt, a description is read by people and by the aligner. they would bind and the questions they serve; approval writes the element with its structure and re-decides those signatures; every approved element carries regression cases rerun when its definition changes; duplicate properties merged as governance. + A property regression case can also expect no binding (`expected_binding: "none"`, with + both target fields absent or null); omitting the mode keeps the existing bound request. + Only an explicit `none` decision passes a negative case. Bound or undecided decisions fail, + including a conflicting human binding, which remains authoritative in the formal graph. + With no decision the case stays unchecked. Reconfirming a case retains its first creator. + Adoption skips positive case registration for an example with a negative case; the + existing case reports any disagreement after alignment without blocking adoption. + - **Exploration proposes an alignment per table** through `ontology_proposals`, adopted as one thing; `Metric` / `Dimension` retire (#554, #556) [0036]. - Filtering reified-shell relations (`Action`, `Offer`) out of pack import [0012]; Chinese diff --git a/migrations/0118_a_regression_case_can_expect_no_binding.sql b/migrations/0118_a_regression_case_can_expect_no_binding.sql new file mode 100644 index 000000000..33ee09201 --- /dev/null +++ b/migrations/0118_a_regression_case_can_expect_no_binding.sql @@ -0,0 +1,12 @@ +-- 0061: no binding is an expectation, not a synthetic ontology property. +ALTER TABLE ontology_regression_cases + ALTER COLUMN expected_property_id DROP NOT NULL, + ALTER COLUMN expected_direction DROP NOT NULL, + ADD CONSTRAINT ontology_regression_cases_complete_target + CHECK ((expected_property_id IS NULL) = (expected_direction IS NULL)); + +-- The existing unique constraint still deduplicates bound expectations. NULLs need +-- their own uniqueness rule, including concurrent confirmations of the same case. +CREATE UNIQUE INDEX ontology_regression_cases_none + ON ontology_regression_cases(kb_id, statement_id) + WHERE expected_property_id IS NULL; diff --git a/web/src/api.ts b/web/src/api.ts index 6a66651f5..a1d1c650c 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -1196,8 +1196,9 @@ export interface OntologyRegressionCase { id: string; kb_id: string; statement_id: string; - expected_property_id: string; - expected_direction: "forward" | "reverse"; + expected_binding: "bound" | "none"; + expected_property_id: string | null; + expected_direction: "forward" | "reverse" | null; created_by: string | null; created_at: string; origin: "adoption" | "person"; @@ -1205,6 +1206,7 @@ export interface OntologyRegressionCase { last_result: { passed: boolean; human_bound: boolean; + conflict?: "human_binding" | null; actual_property_id: string | null; actual_direction: "forward" | "reverse" | null; status: "bound" | "none" | "undecided"; @@ -1216,8 +1218,8 @@ export interface OntologyRegressionCase { object_id: string | null; object_label: string | null; object_value: Record | null; - expected_property_label: string; - expected_property_key: string; + expected_property_label: string | null; + expected_property_key: string | null; actual_property_label: string | null; actual_property_key: string | null; created_by_label: string | null; @@ -2268,11 +2270,15 @@ export const api = { ), addOntologyRegression: ( kbId: string, - body: { - statement_id: string; + body: { statement_id: string } & ({ + expected_binding?: "bound"; relation_type_id: string; direction: "forward" | "reverse"; - }, + } | { + expected_binding: "none"; + relation_type_id?: null; + direction?: null; + }), ) => request<{ id: string }>(`/api/v1/kbs/${kbId}/ontology/regressions`, { method: "POST", diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts index bda621ff9..5f774207f 100644 --- a/web/src/i18n/en.ts +++ b/web/src/i18n/en.ts @@ -47,6 +47,8 @@ export const en = { /** 服务端校验错误的措辞。key = 服务端给的 code；缺一条就退回英文原句，不会崩。 契约守卫（调错接口才碰得到）刻意不在这里——它们的读者是开发者 */ err: { + bad_case_expectation: "Choose a property and direction, or expect no binding.", + bad_email: "That doesn't look like an email address.", password_too_short: "Password must be at least 8 characters.", bad_display_name: "Display name must be 1-64 characters.", @@ -1405,7 +1407,11 @@ export const en = { caseHideEvidence: "Hide source evidence", caseMake: "Add regression case", caseAddTitle: "Add a property regression case", - caseAddHint: "Save the property and direction this open statement should bind to. Its existing alignment decision will be compared with this expectation.", + caseExpectBound: "Bind to a property", + caseNoneHint: "Only an explicit no-binding decision passes. An undecided or bound result fails; no decision remains unchecked.", + caseHumanConflict: "Failed: manual binding conflict", + caseHumanConflictHint: "This case expects no binding, but a person bound the statement. The case fails; the manual binding is retained.", + caseAddHint: "Save whether this open statement should bind to a property or remain unbound. Its alignment decision is compared with that expectation.", caseExpectedProperty: "Expected property", caseDirection: "Direction", caseForward: "Subject → Object", diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.ts index f776ef233..1183a772c 100644 --- a/web/src/i18n/zh.ts +++ b/web/src/i18n/zh.ts @@ -49,6 +49,8 @@ export const zh: Strings = { docsUrl: "https://utopia.bi/docs", }, err: { + bad_case_expectation: "请选择属性及方向，或选择预期无绑定。", + bad_email: "这看起来不像一个邮箱地址。", password_too_short: "密码至少 8 位。", bad_display_name: "显示名需要 1–64 个字符。", @@ -1248,7 +1250,11 @@ export const zh: Strings = { caseHideEvidence: "收起原文证据", caseMake: "添加回归案例", caseAddTitle: "添加属性回归案例", - caseAddHint: "保存这条开放陈述应绑定的属性及方向，再将已有的对齐决定与此预期比较。", + caseExpectBound: "绑定到属性", + caseNoneHint: "只有明确判为无绑定才通过。未定或已绑定均失败；尚无判定时保持未检查。", + caseHumanConflict: "失败：与人工绑定冲突", + caseHumanConflictHint: "此案例预期无绑定，但人已将陈述绑定到属性。案例失败，正式人工绑定保留。", + caseAddHint: "保存这条开放陈述应绑定到属性，还是保持无绑定，再将对齐决定与此预期比较。", caseExpectedProperty: "预期属性", caseDirection: "方向", caseForward: "主语 → 宾语", diff --git a/web/src/pages/AddOntologyRegressionDialog.tsx b/web/src/pages/AddOntologyRegressionDialog.tsx index a07a6088a..4bd2ba9eb 100644 --- a/web/src/pages/AddOntologyRegressionDialog.tsx +++ b/web/src/pages/AddOntologyRegressionDialog.tsx @@ -21,12 +21,16 @@ export function AddOntologyRegressionDialog({ queryFn: () => api.ontology(kbId), }); const [property, setProperty] = useState(""); + const [expected, setExpected] = useState<"bound" | "none">("bound"); const [direction, setDirection] = useState<"forward" | "reverse">("forward"); const options = (ontology.data?.relation_types ?? []) .filter((p) => p.kind === (isValue ? "attribute" : "relation")) .map((p) => ({ value: p.id, label: p.label, hint: p.key })); const add = useMutation({ - mutationFn: () => api.addOntologyRegression(kbId, { + mutationFn: () => api.addOntologyRegression(kbId, expected === "none" ? { + statement_id: statementId, + expected_binding: "none", + } : { statement_id: statementId, relation_type_id: property, direction: isValue ? "forward" : direction, @@ -45,20 +49,28 @@ export function AddOntologyRegressionDialog({ closeLabel={S.ui.close} saveLabel={S.ontology.caseAdd} cancelLabel={S.ontology.cancel} - canSave={options.some((p) => p.value === property)} + canSave={expected === "none" || options.some((p) => p.value === property)} busy={add.isPending} onSave={() => add.mutate()} onCancel={onClose} >

{statement}

- {ontology.isError ? ( + + + + {expected === "none" ? ( +

{S.ontology.caseNoneHint}

+ ) : ontology.isError ? (

{(ontology.error as Error).message}

) : ontology.isPending ? ( {S.nav.loading} ) : options.length === 0 ? (

{S.ontology.caseNoProperties}

) : ( -
+
diff --git a/web/src/pages/OntologyRegressionsPanel.test.ts b/web/src/pages/OntologyRegressionsPanel.test.ts index d8622a822..649d9b207 100644 --- a/web/src/pages/OntologyRegressionsPanel.test.ts +++ b/web/src/pages/OntologyRegressionsPanel.test.ts @@ -8,7 +8,7 @@ import { OntologyRegressionsPanel } from "./OntologyRegressionsPanel"; const base: OntologyRegressionCase = { id: "case", kb_id: "kb", statement_id: "statement", - expected_property_id: "works-for", expected_direction: "forward", + expected_binding: "bound", expected_property_id: "works-for", expected_direction: "forward", created_by: "actor", created_at: "2026-10-10T00:00:00Z", origin: "adoption", last_checked_at: "2026-10-10T01:00:00Z", last_result: { @@ -63,3 +63,24 @@ describe("stored property regression comparisons", () => { expect(html).not.toContain(`>${S.ontology.casePassed}<`); }); }); + + +it("shows negative expectations and a failing manual conflict without a fabricated property or direction", () => { + const negative: OntologyRegressionCase = { + ...base, expected_binding: "none", expected_property_id: null, expected_direction: null, + expected_property_label: null, expected_property_key: null, actual_property_label: null, + last_result: { ...base.last_result!, status: "none", actual_property_id: null, actual_direction: null }, + }; + const passing = render([negative]); + expect(passing).toContain(S.ontology.caseUnbound); + expect(passing).toContain(S.ontology.casePassed); + expect(passing).not.toContain(S.ontology.caseForward); + const conflict = render([{ ...negative, actual_property_label: "Works for", last_result: { + ...base.last_result!, passed: false, human_bound: false, conflict: "human_binding", + } }]); + expect(conflict).toContain(S.ontology.caseHumanConflict); + expect(conflict).toContain(S.ontology.caseHumanConflictHint); + expect(conflict).toContain("Works for"); + expect(conflict).not.toContain(S.ontology.caseHumanHint); + expect(conflict).not.toContain(`>${S.ontology.casePassed}<`); +}); diff --git a/web/src/pages/OntologyRegressionsPanel.tsx b/web/src/pages/OntologyRegressionsPanel.tsx index 8a73aa8b8..51a05a252 100644 --- a/web/src/pages/OntologyRegressionsPanel.tsx +++ b/web/src/pages/OntologyRegressionsPanel.tsx @@ -100,8 +100,8 @@ function CaseRow({ kbId, item }: { kbId: string; item: OntologyRegressionCase })
-
{item.expected_property_label}
-
{directionText(item.expected_direction)}
+
{item.expected_binding === "none" ? S.ontology.caseUnbound : item.expected_property_label}
+ {item.expected_direction &&
{directionText(item.expected_direction)}
}
@@ -119,11 +119,14 @@ function CaseRow({ kbId, item }: { kbId: string; item: OntologyRegressionCase }) - {!result ? S.ontology.caseNotChecked : result.human_bound ? S.ontology.caseHumanBound : result.passed ? S.ontology.casePassed : S.ontology.caseMismatch} + {!result ? S.ontology.caseNotChecked : result.conflict === "human_binding" ? S.ontology.caseHumanConflict : result.human_bound ? S.ontology.caseHumanBound : result.passed ? S.ontology.casePassed : S.ontology.caseMismatch} {result?.human_bound && (

{S.ontology.caseHumanHint}

)} + {result?.conflict === "human_binding" && ( +

{S.ontology.caseHumanConflictHint}

+ )} {item.last_checked_at ? localDateTime(item.last_checked_at) : "—"} From 50904e9fdf2e6aef6eae07d6aa17e599da715f8c Mon Sep 17 00:00:00 2001 From: Szabo Bogdan Date: Fri, 23 Oct 2026 12:06:44 +0200 Subject: [PATCH 4/9] A statements source that declares its names identities attaches them by exact name and computes no name vectors for them (#1333) * A statements source that declares its names identities attaches them by exact name and computes no name vectors for them Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Bogdan Szabo * The identity record separates the class note from decision 6, and the may_attach comment is in Chinese Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Bogdan Szabo --------- Signed-off-by: Bogdan Szabo Co-authored-by: Claude Sonnet 5.5 Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- crates/utopia-core/src/models.rs | 26 ++++ crates/utopia-server/src/extraction.rs | 39 ++++-- crates/utopia-server/src/extraction_open.rs | 13 +- .../extraction_open_identity_names_tests.rs | 126 ++++++++++++++++++ crates/utopia-store/src/resolution.rs | 59 +++++++- ...dentity_name_attaches_by_its_exact_name.rs | 123 +++++++++++++++++ crates/utopia-store/tests/store/main.rs | 1 + ...source-may-declare-its-names-identities.md | 4 +- docs/decisions/README.md | 2 +- web/src/docs/ingest.md | 2 + 10 files changed, 382 insertions(+), 13 deletions(-) create mode 100644 crates/utopia-server/src/extraction_open_identity_names_tests.rs create mode 100644 crates/utopia-store/tests/store/an_identity_name_attaches_by_its_exact_name.rs diff --git a/crates/utopia-core/src/models.rs b/crates/utopia-core/src/models.rs index 15dbb2fbc..0178d60a6 100644 --- a/crates/utopia-core/src/models.rs +++ b/crates/utopia-core/src/models.rs @@ -181,6 +181,13 @@ impl Source { .and_then(serde_json::Value::as_bool) .unwrap_or(true) } + + /// statements 来源的 config 写了 `{"names": "identity"}`：它推来的每个名字各指一个 + /// 确定的东西（0071），按字面归并、不算名字向量。其余来源照旧。 + pub fn names_are_identities(&self) -> bool { + self.kind == "statements" + && self.config.get("names").and_then(serde_json::Value::as_str) == Some("identity") + } } #[cfg(test)] @@ -217,6 +224,25 @@ mod source_extracts_tests { assert!(with(serde_json::json!({ "extract": 0 })).extracts()); assert!(!with(serde_json::json!({ "extract": false })).extracts()); } + + fn of_kind(kind: &str, config: serde_json::Value) -> Source { + Source { + kind: kind.into(), + ..with(config) + } + } + + #[test] + fn only_a_statements_source_with_the_string_identity_has_identity_names() { + let identity = serde_json::json!({ "names": "identity" }); + assert!(of_kind("statements", identity.clone()).names_are_identities()); + assert!(!of_kind("api", identity).names_are_identities()); + assert!(!of_kind("statements", serde_json::json!({})).names_are_identities()); + assert!(!of_kind("statements", serde_json::json!({ "names": true })).names_are_identities()); + assert!( + !of_kind("statements", serde_json::json!({ "names": "prose" })).names_are_identities() + ); + } } /// 来源配置里**用来鉴权**的那几个键。凭据只进不出：列表与创建 / 更新的响应都剔掉， diff --git a/crates/utopia-server/src/extraction.rs b/crates/utopia-server/src/extraction.rs index 6b7ae44fb..993655e63 100644 --- a/crates/utopia-server/src/extraction.rs +++ b/crates/utopia-server/src/extraction.rs @@ -209,12 +209,23 @@ async fn resolve_uncached( name_vec: Option<&[f32]>, text: Option<&str>, exclude: &[Uuid], + identity: bool, needs_adjudication: &mut bool, ) -> anyhow::Result { - let r = utopia_store::resolution::resolve_mention( - pool, kb_id, type_id, name, ctx, name_vec, text, exclude, - ) - .await?; + let r = match identity { + true => { + utopia_store::resolution::resolve_identity_name( + pool, kb_id, type_id, name, ctx, text, exclude, + ) + .await? + } + false => { + utopia_store::resolution::resolve_mention( + pool, kb_id, type_id, name, ctx, name_vec, text, exclude, + ) + .await? + } + }; // 疑似重复对（画像灰区 / 类型漂移 / 同名并列）入审核队列。多数走批量裁决器， // 同名并列（`ReviewStage::Human`）分不出谁是谁，只能等人裁——它自己带着 stage。 for review in &r.reviews { @@ -268,6 +279,7 @@ pub(crate) async fn resolve_handle( ctx: Option<&[f32]>, name_vec: Option<&[f32]>, text: Option<&str>, + identity: bool, response_claims: &mut HashMap>, handled_by_name: &mut HashMap>, ambiguous_bare_cache: &mut HashMap, @@ -309,6 +321,7 @@ pub(crate) async fn resolve_handle( name_vec, text, &excluded, + identity, needs_adjudication, ) .await?; @@ -343,10 +356,14 @@ async fn run(state: &AppState, document_id: Uuid, proposer: Proposer) -> anyhow: } let kb = utopia_store::kbs::get(&state.pool, doc.kb_id).await?; // 推送来的陈述（0054）：块就是契约，抽取按契约解析、不问模型，没配对话模型也照抽 - let pushed = crate::pipeline::source_kind(state, doc.source_id) - .await? - .as_deref() - == Some("statements"); + let source = match doc.source_id { + Some(id) => Some(utopia_store::sources::get(&state.pool, id).await?), + None => None, + }; + let pushed = source.as_ref().is_some_and(|s| s.kind == "statements"); + let identity = source + .as_ref() + .is_some_and(utopia_core::models::Source::names_are_identities); // settings 有就传：推送路径不问对话模型，但名字向量的嵌入模型（#877）仍从它来 let settings = utopia_store::settings::get(&state.pool, kb.workspace_id).await?; let client = if pushed { @@ -376,6 +393,7 @@ async fn run(state: &AppState, document_id: Uuid, proposer: Proposer) -> anyhow: proposer, await_nod, pushed, + identity, ) .await } @@ -496,6 +514,7 @@ mod tests { None, None, None, + false, &mut response_claims, &mut document_claims, &mut bare_cache, @@ -510,6 +529,7 @@ mod tests { None, None, None, + false, &mut response_claims, &mut document_claims, &mut bare_cache, @@ -588,6 +608,7 @@ mod tests { None, None, None, + false, &mut later_response_claims, &mut document_claims, &mut bare_cache, @@ -616,6 +637,7 @@ mod tests { None, None, None, + false, &mut another_response_claims, &mut document_claims, &mut bare_cache, @@ -743,6 +765,7 @@ mod tests { Some(&ctx), None, None, + false, &mut response_claims, &mut document_claims, &mut bare_cache, diff --git a/crates/utopia-server/src/extraction_open.rs b/crates/utopia-server/src/extraction_open.rs index 9422929ef..4df943a94 100644 --- a/crates/utopia-server/src/extraction_open.rs +++ b/crates/utopia-server/src/extraction_open.rs @@ -286,6 +286,7 @@ async fn apply_open_extraction( attested_at: Option>, bound_types: &HashMap, pushed: bool, + identity: bool, await_nod: bool, proposer: Proposer, pending_count: &mut usize, @@ -330,6 +331,7 @@ async fn apply_open_extraction( ctx, name_vecs.get(&key).map(Vec::as_slice), evidence.get(&key).map(String::as_str), + identity, &mut response_claims, handled_by_name, ambiguous_bare_cache, @@ -1010,6 +1012,8 @@ pub(crate) async fn run_open( proposer: Proposer, await_nod: bool, pushed: bool, + // 0071: the source's names are identities, so they get no name vectors + identity: bool, ) -> anyhow::Result<()> { let pool = &state.pool; let document_id = doc.id; @@ -1044,7 +1048,9 @@ pub(crate) async fn run_open( // 名字向量的嵌入客户端（0041 决定 3 通道 2）。没配嵌入模型就是 None：召回退回 // 字面相等，抽取照常 - let embed = settings.and_then(crate::llm_util::embed_client); + let embed = settings + .and_then(crate::llm_util::embed_client) + .filter(|_| !identity); // 文档自己说到的日期（0064 决定 1）：每一块报上来的，连它所在的标题路径。块与块之间 // 不传——一条日期管到哪，看的是它自己那一块正文里的标题，块并行抽的时候照样成立（#588） let mut time_entries: Vec = Vec::new(); @@ -1140,6 +1146,7 @@ pub(crate) async fn run_open( attested_at, &bound_types, pushed, + identity, await_nod, proposer, &mut pending_count, @@ -1282,6 +1289,10 @@ pub(crate) async fn run_open( #[path = "extraction_open_known_flow_tests.rs"] mod known_flow_tests; +#[cfg(test)] +#[path = "extraction_open_identity_names_tests.rs"] +mod identity_names_tests; + #[cfg(test)] #[path = "extraction_open_out_of_credit_tests.rs"] mod out_of_credit_tests; diff --git a/crates/utopia-server/src/extraction_open_identity_names_tests.rs b/crates/utopia-server/src/extraction_open_identity_names_tests.rs new file mode 100644 index 000000000..352f0853f --- /dev/null +++ b/crates/utopia-server/src/extraction_open_identity_names_tests.rs @@ -0,0 +1,126 @@ +//! A statements source with `{"names": "identity"}` (0071) gets no embedding call and no +//! `name_vector|` pair. The fake endpoint returns one vector for every name. +use super::*; +use axum::{routing::post, Json, Router}; +use serde_json::{json, Value}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +/// Push `PUSHED` into a base holding `KNOWN` (with a name vector); answers (embedding calls, review reasons). +async fn push(config: Value) -> anyhow::Result)>> { + let Some(url) = utopia_store::test_db::url() else { + return Ok(None); + }; + let pool = sqlx::PgPool::connect(&url).await?; + utopia_store::db::migrate(&pool).await?; + let [org, ws, kb, source, doc, chunk, known_id]: [Uuid; 7] = + std::array::from_fn(|_| Uuid::now_v7()); + let text = serde_json::to_string(&json!({ + "external_id": "commit:identity", + "e": [[PUSHED, "SourceFile", true]], + "s": [[null, PUSHED, "path", null, "src/x.ts", null, null, null]], + "n": [] + }))?; + sqlx::raw_sql(&format!( + "INSERT INTO organizations(id,name) VALUES ('{org}','identity-names'); + INSERT INTO workspaces(id,org_id,name) VALUES ('{ws}','{org}','identity-names'); + INSERT INTO knowledge_bases(id,workspace_id,name) VALUES ('{kb}','{ws}','identity-names'); + INSERT INTO sources(id,kb_id,kind,name,config) VALUES + ('{source}','{kb}','statements','lore',$${config}$$::jsonb); + INSERT INTO documents(id,kb_id,filename,sha256,source_id) VALUES + ('{doc}','{kb}','commit.json','x{doc}','{source}'); + INSERT INTO chunks(id,kb_id,document_id,seq,text) VALUES + ('{chunk}','{kb}','{doc}',0,$${text}$$);" + )) + .execute(&pool) + .await?; + sqlx::query("INSERT INTO entities(id,kb_id,canonical_name) VALUES ($1,$2,$3)") + .bind(known_id) + .bind(kb) + .bind(KNOWN) + .execute(&pool) + .await?; + let fact = utopia_store::names::record(&pool, kb, known_id, KNOWN, None, None) + .await? + .expect("a named entity gets a name fact"); + utopia_store::name_vectors::set(&pool, kb, &[(fact, known_id, vec![1.0, 0.0, 0.0, 0.0])]) + .await?; + + let calls = Arc::new(AtomicUsize::new(0)); + let counted = calls.clone(); + let router = Router::new().route( + "/embeddings", + post(move |Json(body): Json| async move { + counted.fetch_add(1, Ordering::SeqCst); + let inputs = body["input"].as_array().map_or(0, Vec::len); + Json(json!({"data": vec![json!({"embedding": [1.0, 0.0, 0.0, 0.0]}); inputs]})) + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let base = format!("http://{}", listener.local_addr()?); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + utopia_store::settings::upsert( + &pool, + ws, + None, + None, + None, + Some(&base), + None, + Some("fake-embed"), + Some(4), + ) + .await?; + + let dir = tempfile::tempdir()?; + let cfg = utopia_core::config::AppConfig { + data_dir: dir.path().to_string_lossy().into_owned(), + ..Default::default() + }; + let search = Arc::new(utopia_search::SearchIndex::open( + &dir.path().join("search"), + )?); + let state = AppState::new(pool.clone(), &cfg, search, "test-only".into()); + crate::extraction::extract_document(&state, doc, utopia_core::models::Proposer::default()) + .await?; + + let reasons: Vec = sqlx::query_scalar( + "SELECT reason FROM resolution_reviews WHERE kb_id = $1 ORDER BY reason", + ) + .bind(kb) + .fetch_all(&pool) + .await?; + + server.abort(); + sqlx::query("DELETE FROM jobs WHERE payload->>'kb_id' = $1") + .bind(kb.to_string()) + .execute(&pool) + .await?; + sqlx::query("DELETE FROM organizations WHERE id = $1") + .bind(org) + .execute(&pool) + .await?; + Ok(Some((calls.load(Ordering::SeqCst), reasons))) +} + +const KNOWN: &str = "re-cinq/lore:src/a.ts"; +const PUSHED: &str = "re-cinq/lore:src/b.ts"; + +#[tokio::test] +async fn an_identity_source_adds_no_name_vectors_and_asks_no_neighbours() -> anyhow::Result<()> { + let Some(outcome) = push(json!({"names": "identity"})).await? else { + return Ok(()); + }; + assert_eq!(outcome, (0, Vec::::new())); + Ok(()) +} + +#[tokio::test] +async fn a_statements_source_without_the_flag_still_proposes_by_name_vector() -> anyhow::Result<()> +{ + let Some((calls, reasons)) = push(json!({})).await? else { + return Ok(()); + }; + assert!(calls > 0 && reasons.iter().any(|r| r.starts_with("name_vector|"))); + Ok(()) +} diff --git a/crates/utopia-store/src/resolution.rs b/crates/utopia-store/src/resolution.rs index d120d2f41..8c7a9e217 100644 --- a/crates/utopia-store/src/resolution.rs +++ b/crates/utopia-store/src/resolution.rs @@ -364,6 +364,58 @@ async fn kept_apart_from(pool: &PgPool, kb_id: Uuid, entity_id: Uuid) -> AppResu Ok(rows.into_iter().map(|(id,)| id).collect()) } +/// 0071 决定 2、3：恰有一个活着的同名持有者（类为声明的类或无类）时，不看画像，直接归并； +/// 没有就新建，哪怕该名字在别的类下已有（类型漂移审核照留）；两个及以上走原来的画像路径。 +/// 持有者按本名或名字事实认（合并会把输家的名字作为 known_as 移给存活者）。 +pub async fn resolve_identity_name( + pool: &PgPool, + kb_id: Uuid, + type_id: Option, + raw_name: &str, + context: Option<&[f32]>, + text: Option<&str>, + exclude: &[Uuid], +) -> AppResult { + let normalized = normalize_name(raw_name); + let name = fold_markdown_markers(&normalized); + let spellings = vec![normalized.to_lowercase(), name.to_lowercase()]; + let holders: Vec = sqlx::query_as(&format!( + "SELECT e.id, e.canonical_name, e.type_id, e.profile_embedding, e.profile_n, + TRUE AS exact_name + FROM entities e + WHERE e.kb_id = $1 AND e.merged_into IS NULL AND e.description IS NULL + AND (e.type_id IS NULL OR e.type_id = $2) + AND (lower(e.canonical_name) = ANY($3) OR {named})", + named = crate::names::has_name_in("e", 1, 3), + )) + .bind(kb_id) + .bind(type_id) + .bind(&spellings) + .fetch_all(pool) + .await? + .into_iter() + .filter(|holder: &Candidate| !exclude.contains(&holder.id)) + .collect(); + match holders.as_slice() { + [holder] => { + match context { + Some(ctx) => update_profile(pool, holder.id, holder.profile_n, ctx).await?, + None => touch_entity(pool, holder.id).await?, + } + Ok(Resolution { + entity_id: holder.id, + created: false, + reviews: Vec::new(), + }) + } + [] => { + let keys = recall_keys(&normalized); + resolve_type_drift(pool, kb_id, type_id, name, &keys, context, exclude, false).await + } + _ => resolve_by_name(pool, kb_id, type_id, raw_name, context, text, exclude).await, + } +} + /// 通道 2：mention 名字向量的近邻，各排一对（规矩见 [`resolve_mention`]） async fn propose_by_name_vector( pool: &PgPool, @@ -523,7 +575,7 @@ async fn resolve_by_name( if candidates.is_empty() { // 同类型无候选 ≠ 新名字：类型标签会漂（同一团队被抽成 organization/project/ // concept），先查其它类型下的同名实体，按类型对的互斥强度分流。 - return resolve_type_drift(pool, kb_id, type_id, name, &keys, context, exclude).await; + return resolve_type_drift(pool, kb_id, type_id, name, &keys, context, exclude, true).await; } let Some(ctx) = context else { @@ -1077,6 +1129,7 @@ fn confusable_reviews( /// concept 兜底型候选跑既有画像分层：高相似直接 ATTACH（漂移的召回修复，concept /// 侧类型升格为具体类型），灰区/无从判别 → 宁分勿合新建 + 审核对；易混具体类型 /// 一律新建 + 审核对（同名 + 类型摇摆本身就是信号，不设相似度门槛）；硬互斥忽略。 +#[allow(clippy::too_many_arguments)] async fn resolve_type_drift( pool: &PgPool, kb_id: Uuid, @@ -1086,6 +1139,8 @@ async fn resolve_type_drift( keys: &[String], context: Option<&[f32]>, exclude: &[Uuid], + // false = 不归并到概念兜底：身份名按自己声明的类新建，不能被兜底吞掉 + may_attach: bool, ) -> AppResult { // 这一侧也可能还没判出类型（0009），那时没有 key 可查 let mention_key: Option = match type_id { @@ -1142,7 +1197,7 @@ async fn resolve_type_drift( } } - if let Some(ctx) = context { + if let Some(ctx) = context.filter(|_| may_attach) { let best = recall_cands .iter() .filter_map(|c| { diff --git a/crates/utopia-store/tests/store/an_identity_name_attaches_by_its_exact_name.rs b/crates/utopia-store/tests/store/an_identity_name_attaches_by_its_exact_name.rs new file mode 100644 index 000000000..647606416 --- /dev/null +++ b/crates/utopia-store/tests/store/an_identity_name_attaches_by_its_exact_name.rs @@ -0,0 +1,123 @@ +//! 0071 decisions 2 and 3: an identity mention attaches to the exact-name holder whatever the +//! profile says, and is created otherwise. Skipped without `UTOPIA_DATABASE_URL`. + +use sqlx::PgPool; +use utopia_store::resolution::resolve_identity_name; +use uuid::Uuid; + +/// Orthogonal to every holder's profile, so the profile never supports an attach. +const UNRELATED: [f32; 3] = [0.0, 1.0, 0.0]; + +/// A kb with a project and an organization class, a body to run in it, then teardown. +async fn in_kb(body: F) -> anyhow::Result<()> +where + F: FnOnce(PgPool, Uuid, Uuid, Uuid) -> Fut, + Fut: std::future::Future>, +{ + let Some(url) = utopia_store::test_db::url() else { + return Ok(()); + }; + let pool = PgPool::connect(&url).await?; + let ids: [Uuid; 5] = std::array::from_fn(|_| Uuid::now_v7()); + let [org, ws, kb, project, organization] = ids; + sqlx::raw_sql(&format!( + "INSERT INTO organizations (id, name) VALUES ('{org}', 'identity'); + INSERT INTO workspaces (id, org_id, name) VALUES ('{ws}', '{org}', 'identity'); + INSERT INTO knowledge_bases (id, workspace_id, name) VALUES ('{kb}', '{ws}', 'identity'); + INSERT INTO entity_types (id, kb_id, key, label) VALUES + ('{project}', '{kb}', 'project', 'Project'), + ('{organization}', '{kb}', 'organization', 'Organization');" + )) + .execute(&pool) + .await?; + let run = body(pool.clone(), kb, project, organization).await; + sqlx::query("DELETE FROM organizations WHERE id = $1") + .bind(org) + .execute(&pool) + .await?; + run +} + +/// A live entity whose profile points along the first axis. +async fn holder( + pool: &PgPool, + kb: Uuid, + type_id: Option, + name: &str, +) -> anyhow::Result { + let id = Uuid::now_v7(); + sqlx::query( + "INSERT INTO entities (id, kb_id, type_id, canonical_name, profile_embedding, profile_n) + VALUES ($1, $2, $3, $4, '[1,0,0]'::vector, 1)", + ) + .bind(id) + .bind(kb) + .bind(type_id) + .bind(name) + .execute(pool) + .await?; + utopia_store::names::record(pool, kb, id, name, None, None).await?; + Ok(id) +} + +#[tokio::test] +async fn one_exact_holder_is_attached_and_otherwise_the_name_is_created() -> anyhow::Result<()> { + in_kb(|pool, kb, project, _| async move { + let only = holder(&pool, kb, Some(project), "acme/a.ts").await?; + let (twin, twin_too) = ( + holder(&pool, kb, Some(project), "acme/b.ts").await?, + holder(&pool, kb, Some(project), "acme/b.ts").await?, + ); + let resolve = |name: &'static str| { + resolve_identity_name(&pool, kb, Some(project), name, Some(&UNRELATED), None, &[]) + }; + let attached = resolve("ACME/a.ts").await?; + assert_eq!((attached.entity_id, attached.created), (only, false)); + assert!(resolve("acme/c.ts").await?.created); + let split = resolve("acme/b.ts").await?; + assert!(split.created && ![twin, twin_too].contains(&split.entity_id)); + // known only by a known_as fact, as after a merge + let renamed = holder(&pool, kb, Some(project), "acme/old.ts").await?; + utopia_store::names::record(&pool, kb, renamed, "acme/e.ts", None, None).await?; + assert_eq!(resolve("acme/e.ts").await?.entity_id, renamed); + // untyped mention, same profile as a typed holder: still no concept-fallback attach + let typed = holder(&pool, kb, Some(project), "acme/d.ts").await?; + let same = [1.0, 0.0, 0.0]; + let untyped = + resolve_identity_name(&pool, kb, None, "acme/d.ts", Some(&same), None, &[]).await?; + assert!(untyped.created && untyped.entity_id != typed); + Ok(()) + }) + .await +} + +#[tokio::test] +async fn a_name_under_another_class_is_created_with_a_drift_review_unless_disjoint( +) -> anyhow::Result<()> { + in_kb(|pool, kb, project, organization| async move { + holder(&pool, kb, Some(organization), "acme/a.ts").await?; + holder(&pool, kb, Some(organization), "acme/b.ts").await?; + let reviews = |name: &'static str| async { + let r = resolve_identity_name(&pool, kb, Some(project), name, None, None, &[]).await?; + anyhow::Ok(( + r.created, + r.reviews.into_iter().map(|v| v.reason).collect::>(), + )) + }; + assert_eq!( + reviews("acme/a.ts").await?, + (true, vec!["type_drift|project vs organization".to_string()]) + ); + for (a, b) in [(project, organization), (organization, project)] { + sqlx::query("INSERT INTO entity_type_disjoint (kb_id, a_id, b_id) VALUES ($1, $2, $3)") + .bind(kb) + .bind(a) + .bind(b) + .execute(&pool) + .await?; + } + assert_eq!(reviews("acme/b.ts").await?, (true, vec![])); + Ok(()) + }) + .await +} diff --git a/crates/utopia-store/tests/store/main.rs b/crates/utopia-store/tests/store/main.rs index 0f6e26a16..0b0e0bb90 100644 --- a/crates/utopia-store/tests/store/main.rs +++ b/crates/utopia-store/tests/store/main.rs @@ -119,6 +119,7 @@ mod an_end_date_closes_the_open_span; mod an_event_holds_at_the_moment_it_names; mod an_exploration_says_what_it_covered; mod an_export_carries_the_whole_ledger; +mod an_identity_name_attaches_by_its_exact_name; mod an_instance_list_names_what_a_type_lacks; mod an_open_statement_keeps_the_documents_words; mod an_sso_login_finds_or_opens_its_account; diff --git a/docs/decisions/0071-a-source-may-declare-its-names-identities.md b/docs/decisions/0071-a-source-may-declare-its-names-identities.md index e5a2622f5..c0c76194b 100644 --- a/docs/decisions/0071-a-source-may-declare-its-names-identities.md +++ b/docs/decisions/0071-a-source-may-declare-its-names-identities.md @@ -1,6 +1,6 @@ # 0071 · A source may declare its names identities -- **Status**: Proposed · 2026-10-19 · [#1293](https://github.com/deeplethe/utopia/issues/1293) · open: all seven decisions +- **Status**: In progress · 2026-10-19 · [#1293](https://github.com/deeplethe/utopia/issues/1293) · open: decisions 5, 6 and 7, and the `name_vectors::pending` skip of decision 4 - **Written**: 2026-10-19 (conventions in the [README](README.md)) - **Related**: [0054](0054-a-source-may-push-statements-in-the-open-contract.md) decision 3 sends pushed statements down the same resolution path as prose, and this record narrows that for one kind of source; [0041](0041-a-name-is-a-claim-about-an-entity.md) owns name facts and name-vector recall; [#1248](https://github.com/deeplethe/utopia/issues/1248) meets the same problem from the merge side and is decided here as decision 7; [#1296](https://github.com/deeplethe/utopia/issues/1296) measured what the recall costs; [#1308](https://github.com/deeplethe/utopia/pull/1308) is the disjointness fix that stands on its own @@ -47,6 +47,8 @@ One name fact can carry evidence from both an identity source and a prose docume Why a person and not a stricter threshold: an identity source has already said what its names are. Decisions 2 to 4 remove the pairs that name vectors and the single-candidate split would raise, so what is left are containment and type-drift pairs and the cross-source duplicate (one thing pushed under two names by two producers). Those are exactly the cases where the model's `same` is a guess about a claim the producer made, and a wrong merge of identities is what #1248 measured: on a small local model, 40 automatic merges with governance off, nearly all wrong (a method into its file, two unrelated types). A person sees few of them because the vectors are off. Raising `AUTO_CONF` for these pairs would still let a confident wrong verdict through, and a per-base switch is a dead end (below). +**A NULL-typed holder keeps its class for now.** A typed mention that attaches to a holder with `type_id IS NULL` does not upgrade the holder's class, as the type-drift path does. PR 2 decides that. + ## Not doing - **Containment for identity names.** `containment_reviews` still runs for them. #1104's next cut rewrites that predicate, and whether an identity source skips it is a separate decision after that. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 2d0b81998..bda0b9c92 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -55,7 +55,7 @@ By domain; the domains are the files of [../design/](../design/README.md). **Sta | | Record | Status | Overtaken | |---|---|---|---| | 0041 | [A name is a claim about an entity](0041-a-name-is-a-claim-about-an-entity.md) | In progress | | -| 0071 | [A source may declare its names identities](0071-a-source-may-declare-its-names-identities.md) | Proposed | | +| 0071 | [A source may declare its names identities](0071-a-source-may-declare-its-names-identities.md) | In progress | | ### [time](../design/time.md) diff --git a/web/src/docs/ingest.md b/web/src/docs/ingest.md index e2611fade..35bc59479 100644 --- a/web/src/docs/ingest.md +++ b/web/src/docs/ingest.md @@ -153,6 +153,8 @@ Content-Type: application/json Any other key is refused with `422`, so a mistaken `predicate` or `class` field cannot pass as a typed fact: the contract has no slot for one. A `subject`, or the entity of an `n` row, that is not listed in `e` is refused the same way, rather than dropped later in silence. A body over 64 KiB or with more than 200 statements is refused too. +If every name your system pushes is already unique, such as `owner/repo:src/a.ts#parseDate`, set `{"names": "identity"}` in the source's config. A name from that source attaches to the entity with the same name (ignoring case, in the same class or none) and is created when there is none, whatever the surrounding text. It gets no name vector, so it queues no near-name pairs. A name that exists only under another class is created in this source's class, and still queues the type-drift review unless the two classes are declared disjoint. + Two things to know before wiring a system to it: - **Send events, not state.** A row that *is* the current state of a system belongs on a mounted database, read at query time. Pushing it here copies state into the ledger and the two drift. From 8ea0612e1e95e11e3badf85e5fa47f8a7a50a8e4 Mon Sep 17 00:00:00 2001 From: Quintin Botes Date: Fri, 23 Oct 2026 12:12:40 +0200 Subject: [PATCH 5/9] A deployment can ask its identity provider for only the scopes its admin approved, so sign-in no longer stops at a consent page (#1355) (#1362) * A deployment can ask its identity provider for only the scopes its admin approved, so sign-in no longer stops at a consent page (#1355) Signed-off-by: Quintin Botes * Keep each startup warning's doc comment on its own function, and say in 0069 that the domain list needs the email scope (#1355) Signed-off-by: Quintin Botes * Name the revising PR in 0069's status line (#1355) Signed-off-by: Quintin Botes --------- Signed-off-by: Quintin Botes Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- .env.example | 4 ++ crates/utopia-server/src/api/oidc_routes.rs | 50 ++++++++++++++++++- crates/utopia-server/src/main.rs | 1 + ...ity-provider-vouches-for-listed-domains.md | 4 +- 4 files changed, 57 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index cf0fd51d1..7536fb174 100644 --- a/.env.example +++ b/.env.example @@ -25,6 +25,10 @@ UTOPIA_OPEN_REGISTRATION=true # UTOPIA_OIDC_CLIENT_ID= # UTOPIA_OIDC_CLIENT_SECRET= # UTOPIA_OIDC_REDIRECT_URI=https://your-deployment/api/v1/auth/oidc/callback +# 授权请求要的 scope，空格分隔，默认 openid profile email；openid 总会带上。身份提供方的管理员 +# 只批了 openid profile 时写这两个，免得每个人登录都卡在同意页。设了 UTOPIA_OIDC_ALLOWED_DOMAINS +# 就要留着 email：按域名关联靠 ID token 里的 email。 +# UTOPIA_OIDC_SCOPES=openid profile email # 受信的另一个应用可以拿它用户在同一个身份提供方刚登录到的 ID token，到 # POST /api/v1/auth/oidc/exchange 换一个 Utopia 会话（0066）。这里列它的 client id，逗号分隔； # 与 Utopia 共用一个 client 的就写 Utopia 自己的 client id。留空 = 不开放这个端点。 diff --git a/crates/utopia-server/src/api/oidc_routes.rs b/crates/utopia-server/src/api/oidc_routes.rs index 8cfdf7034..b218fd276 100644 --- a/crates/utopia-server/src/api/oidc_routes.rs +++ b/crates/utopia-server/src/api/oidc_routes.rs @@ -69,6 +69,27 @@ struct Config { /// 本地开发：允许 issuer 与各端点走回环地址上的明文 HTTP。只认 localhost / /// 127.0.0.1 / ::1，公网地址照旧只能 HTTPS loopback_http: bool, + /// 授权请求的 scope（`UTOPIA_OIDC_SCOPES`，默认 `openid profile email`）。有的身份提供方 + /// 的管理员只批了 `openid profile`，多要一个 `email` 就让每个人登录时都卡在同意页（#1355） + scopes: String, +} + +/// `openid` 必须在：少了它就不是 OIDC，拿不到 ID token。没写就补在最前面，而不是报错 +fn scopes(raw: Option<&str>) -> String { + let mut out: Vec<&str> = Vec::new(); + for s in raw + .filter(|r| !r.trim().is_empty()) + .unwrap_or("openid profile email") + .split_whitespace() + { + if !out.contains(&s) { + out.push(s); + } + } + if !out.contains(&"openid") { + out.insert(0, "openid"); + } + out.join(" ") } fn config() -> Result { @@ -100,6 +121,7 @@ fn config() -> Result { .collect(), loopback_http: std::env::var("UTOPIA_OIDC_ALLOW_LOOPBACK_HTTP") .is_ok_and(|v| matches!(v.trim(), "1" | "true")), + scopes: scopes(std::env::var("UTOPIA_OIDC_SCOPES").ok().as_deref()), }; secure_url(&c.issuer, c.loopback_http)?; secure_url(&c.redirect, true)?; @@ -292,6 +314,16 @@ pub fn password_login(s: &AppState) -> bool { s.password_login || config().is_err() } +/// 启动时说出来：允许名单按 ID token 里核实过的 email 关联账号，scope 里没有 `email` +/// 时多数身份提供方根本不发这个声明，名单就形同虚设 +pub fn warn_about_scopes() { + if let Ok(c) = config() { + if !c.allowed_domains.is_empty() && !c.scopes.split_whitespace().any(|s| s == "email") { + tracing::warn!("UTOPIA_OIDC_SCOPES 里没有 email，而 UTOPIA_OIDC_ALLOWED_DOMAINS 已设：身份提供方多半不发 email，按域名关联和开户不会发生") + } + } +} + /// 启动时把关掉密码登录的两种后果说出来：没配 SSO 时这个开关不生效； /// 没有允许名单时只有绑定过身份的人进得来，新人既注册不了也开不了户 pub fn warn_about_password_login(s: &AppState) { @@ -387,7 +419,7 @@ async fn begin( .await?; url.query_pairs_mut().extend_pairs([ ("response_type", "code"), - ("scope", "openid profile email"), + ("scope", c.scopes.as_str()), ("client_id", &c.client_id), ("redirect_uri", &c.redirect), ("state", &state), @@ -1100,9 +1132,25 @@ mod tests { exchange_audiences: vec!["sibling-app".into(), "utopia-test".into()], allowed_domains: vec![], loopback_http: false, + scopes: super::scopes(None), } } + #[test] + fn scopes_default_to_openid_profile_email_and_always_keep_openid() { + assert_eq!(super::scopes(None), "openid profile email"); + assert_eq!(super::scopes(Some(" ")), "openid profile email"); + assert_eq!(super::scopes(Some("openid profile")), "openid profile"); + assert_eq!( + super::scopes(Some("profile email")), + "openid profile email" + ); + assert_eq!( + super::scopes(Some("profile openid profile")), + "profile openid" + ); + } + // This key is generated solely for these tests and is never used by the app. #[test] fn verifies_signature_issuer_audience_nonce_and_expiry() { diff --git a/crates/utopia-server/src/main.rs b/crates/utopia-server/src/main.rs index a08b47d71..ddd8248c0 100644 --- a/crates/utopia-server/src/main.rs +++ b/crates/utopia-server/src/main.rs @@ -173,6 +173,7 @@ async fn main() -> anyhow::Result<()> { let state = AppState::new(pool.clone(), &cfg, search, jwt_secret); api::oidc_routes::warn_about_password_login(&state); + api::oidc_routes::warn_about_scopes(); // worker 并发数：系统设置持久化，启动时装载；运行中经同一 AtomicUsize 热调 let n = utopia_store::access::worker_concurrency(&pool) diff --git a/docs/decisions/0069-the-identity-provider-vouches-for-listed-domains.md b/docs/decisions/0069-the-identity-provider-vouches-for-listed-domains.md index 22c220ce7..18b0468da 100644 --- a/docs/decisions/0069-the-identity-provider-vouches-for-listed-domains.md +++ b/docs/decisions/0069-the-identity-provider-vouches-for-listed-domains.md @@ -1,6 +1,6 @@ # 0069 · The identity provider vouches for listed domains -- **Status**: Implemented · 2026-10-16 · #1237, #1258 · open: no per-domain role; an SSO-opened account has no password and no reset; no global case-insensitive email uniqueness +- **Status**: Implemented · 2026-10-16 · #1237, #1258, #1362 · open: no per-domain role; an SSO-opened account has no password and no reset; no global case-insensitive email uniqueness - **Written**: 2026-10-16 (conventions in the [README](README.md)) - **Related**: migration `0056` (an identity is linked by its owner); [0066](0066-another-app-may-trade-a-fresh-id-token-for-a-session.md) (the exchange, whose mapping this widens); [0014](0014-identity-from-the-person-scope-from-the-token.md) (identity comes from the person); [#1237](https://github.com/deeplethe/utopia/pull/1237) @@ -14,6 +14,8 @@ Migration 0056 rejected linking by email for a reason that still holds in genera 1. **`UTOPIA_OIDC_ALLOWED_DOMAINS` lists the domains the identity provider is authoritative for.** On a login or an exchange whose subject is not linked, a token whose `email` is in a listed domain and whose `email_verified` is true is mapped by email: to the active account with that address (case-insensitive), or to a new one. The subject is then linked to it, so later sign-ins go by subject as before. The list is empty by default, and an empty list keeps 0056's rule exactly. + **Revision 2026-10-22 (#1355).** The `email` claim arrives only when the authorization request asks for the `email` scope. The scopes are now `UTOPIA_OIDC_SCOPES` (default `openid profile email`), so a deployment that lists domains must keep `email` in them, or no address reaches this rule and nobody is linked or opened by domain; the server warns at startup when the list is set and `email` is missing. + **Revision 2026-10-16 (#1258).** This assumed the active match was unique. Checking `users_email_active_idx` showed that it compares the stored spelling, while registration and administrator-created accounts preserve casing: two active accounts can differ only in email case. `account_for_sso` now refuses multiple active case-insensitive matches as `unlinked` before creating an account or writing an identity, rather than choosing one. The check covers the query snapshot; a unique index on `lower(email)`, normalization at registration and administrator creation, and a decision about existing duplicates remain separate work. 2. **The trust assumption.** Listing a domain says: every token this provider signs with a verified address in that domain belongs to the person who owns that address. That holds for a provider that only knows the company's directory. It does not hold for a plain Google OAuth client: Google also signs verified tokens for consumer accounts someone registered with a company address, and that person need not be, or still be, an employee. A deployment on such a client should put a gate in front that admits only the company's Workspace accounts, or restrict the client to its organization at the provider. From 7eff6ad5e839e8b8c226a575c4af37e87b64d460 Mon Sep 17 00:00:00 2001 From: ShryukGrandhi <85529990+ShryukGrandhi@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:51:03 -0700 Subject: [PATCH 6/9] An empty balance during phrase and kind-word alignment fails the run once instead of re-queueing it every few seconds (#1381) * An empty balance during phrase and kind-word alignment fails the run once instead of re-queueing it every few seconds (#1360) Signed-off-by: ShryukGrandhi <85529990+ShryukGrandhi@users.noreply.github.com> * The candidates_for doc comment says an empty balance is the exception to the whole-list fallback Signed-off-by: ShryukGrandhi <85529990+ShryukGrandhi@users.noreply.github.com> --------- Signed-off-by: ShryukGrandhi <85529990+ShryukGrandhi@users.noreply.github.com> Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- crates/utopia-server/src/phrase_alignment.rs | 8 +++ .../src/phrase_alignment_tests.rs | 71 +++++++++++++++++-- crates/utopia-server/src/type_alignment.rs | 5 ++ .../utopia-server/src/type_alignment_tests.rs | 43 +++++++++-- 4 files changed, 115 insertions(+), 12 deletions(-) diff --git a/crates/utopia-server/src/phrase_alignment.rs b/crates/utopia-server/src/phrase_alignment.rs index 1b1b236a7..6b018a90b 100644 --- a/crates/utopia-server/src/phrase_alignment.rs +++ b/crates/utopia-server/src/phrase_alignment.rs @@ -500,6 +500,9 @@ async fn shortlist( tracing::warn!(%kb_id, sent = batch.len(), got = v.len(), "签名向量数量对不上，这一批不开短名单"); continue; } + // 欠费不是这一批的事：跳过它，后面每批、下一份对齐都撞同一个 429，而任务 + // 照样 done、告警不响、马上再排一份（#1360）。带着类型出去，`alerting::hopeless` 才看得见 + Err(e) if utopia_llm::out_of_credit(&e).is_some() => return Err(e), Err(e) => { tracing::warn!(%kb_id, error = %e, "签名向量没算出来，这一批不开短名单"); continue; @@ -600,6 +603,8 @@ async fn shortlist_kind_words( let _permit = llm_util::acquire_embed(state, settings).await; match client.embed(&texts).await { Ok(v) if v.len() == batch.len() => v, + // 同 `shortlist`：欠费停下整轮 + Err(e) if utopia_llm::out_of_credit(&e).is_some() => return Err(e), Ok(_) | Err(_) => { tracing::warn!(%kb_id, "类别词向量没算出来，这一批看全部候选"); continue; @@ -960,6 +965,9 @@ async fn align_phrases_locked( .await { Ok(r) => r, + // 欠费：这一批的 future 带着错误返回，下面的 `r?` 让整轮失败一次、告警响起， + // 不按 failed 记——记了只会自己再排一份，撞同一个 429（#1360） + Err(e) if utopia_llm::out_of_credit(&e).is_some() => return Err(e), Err(e) => { tracing::warn!(%kb_id, error = %e, "短语对齐调用失败，这一批留到下次"); failed += 1; diff --git a/crates/utopia-server/src/phrase_alignment_tests.rs b/crates/utopia-server/src/phrase_alignment_tests.rs index 126ff6436..888955cb0 100644 --- a/crates/utopia-server/src/phrase_alignment_tests.rs +++ b/crates/utopia-server/src/phrase_alignment_tests.rs @@ -2,6 +2,7 @@ //! 无候选与超限各自落库、端点类换了旧行不再循环、请求途中的编辑留下可见的过期、 //! 人的判定不被覆盖。模型是脚本化的 HTTP 端点，库是真的 PostgreSQL。 use super::*; +use anyhow::Context as _; use axum::{extract::State, response::IntoResponse, routing::post, Json, Router}; use serde_json::{json, Value}; use std::sync::{Arc, Mutex}; @@ -16,8 +17,17 @@ struct Model { release: Arc, /// 嵌入端点被叫了几次：短名单该不该嵌签名（#1097）看的是这个数 embeds: Arc, + /// 嵌入端点回余额耗尽（#1360） + embeds_broke: Arc, } -async fn reply(State(m): State, Json(body): Json) -> impl IntoResponse { +/// 脚本里放这个字符串，对话端点就回 OpenAI 余额耗尽的 429 +const OUT_OF_CREDIT: &str = "out_of_credit"; +fn out_of_credit() -> axum::response::Response { + let body = json!({"error": {"message": "You have no credits remaining.", + "type": "insufficient_quota", "code": "insufficient_quota"}}); + (axum::http::StatusCode::TOO_MANY_REQUESTS, Json(body)).into_response() +} +async fn reply(State(m): State, Json(body): Json) -> axum::response::Response { let n = { let mut seen = m.requests.lock().unwrap(); seen.push(body); @@ -32,19 +42,26 @@ async fn reply(State(m): State, Json(body): Json) -> impl IntoResp if replies.is_empty() { panic!("unexpected model request #{n}"); } - replies.remove(0).to_string() + replies.remove(0) }; - let frame = json!({"choices":[{"delta":{"content":text}}]}); + if text == OUT_OF_CREDIT { + return out_of_credit(); + } + let frame = json!({"choices":[{"delta":{"content":text.to_string()}}]}); ( [("content-type", "text/event-stream")], format!("data: {frame}\n\ndata: [DONE]\n\n"), ) + .into_response() } /// 脚本化的嵌入端点：每段文字一个由字节算出的四维向量。只要求确定、条数对得上—— /// 这里测的是「向量有没有」，不是近不近 -async fn embed(State(m): State, Json(body): Json) -> impl IntoResponse { +async fn embed(State(m): State, Json(body): Json) -> axum::response::Response { m.embeds.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + if m.embeds_broke.load(std::sync::atomic::Ordering::SeqCst) { + return out_of_credit(); + } let data: Vec = body["input"] .as_array() .cloned() @@ -66,7 +83,7 @@ async fn embed(State(m): State, Json(body): Json) -> impl IntoResp json!({ "embedding": v }) }) .collect(); - Json(json!({ "data": data })) + Json(json!({ "data": data })).into_response() } struct Fx { @@ -134,6 +151,7 @@ impl Fx { entered: Arc::new(tokio::sync::Notify::new()), release: Arc::new(tokio::sync::Notify::new()), embeds: Arc::new(std::sync::atomic::AtomicUsize::new(0)), + embeds_broke: Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; let endpoint = format!("http://{}", listener.local_addr()?); @@ -731,6 +749,49 @@ async fn signatures_are_not_embedded_while_no_property_has_a_vector() -> anyhow: run } +/// 余额空了（#1360）：签名嵌入或对话的第一个欠费 429 让整轮带着类型失败，任务失败一次、 +/// 告警响起。从前它按「这一批留到下次」记，任务照样 done，马上再排一份，同一个 429 每几秒 +/// 来一遍，直到有人充值 +#[tokio::test] +async fn an_empty_balance_fails_the_run_instead_of_queueing_another() -> anyhow::Result<()> { + let Some(f) = Fx::new().await? else { + return Ok(()); + }; + let run = async { + f.enable_embeddings().await?; + // 13 > SHORTLIST 条候选、属性都有向量：这条签名要嵌入 + for i in 0..12 { + sqlx::query("INSERT INTO relation_types(id,kb_id,key,label,kind,temporal) VALUES($1,$2,$3,$3,'relation','state')") + .bind(Uuid::now_v7()).bind(f.kb).bind(format!("filler_{i}")).execute(&f.pool).await?; + } + sqlx::query("UPDATE relation_types SET embedding = '[1,0,0,0]' WHERE kb_id=$1") + .bind(f.kb) + .execute(&f.pool) + .await?; + + f.model + .embeds_broke + .store(true, std::sync::atomic::Ordering::SeqCst); + let err = f.run().await.err().context("a refused embedding fails the run")?; + anyhow::ensure!(crate::alerting::hopeless(&err), "not hopeless: {err:#}"); + assert!(f.requests().is_empty(), "nothing is asked after the refusal"); + assert!(!f.requeued().await?, "no follow-up alignment is queued"); + + f.model + .embeds_broke + .store(false, std::sync::atomic::Ordering::SeqCst); + f.script(vec![json!(OUT_OF_CREDIT)]); + let err = f.run().await.err().context("a refused chat call fails the run")?; + anyhow::ensure!(crate::alerting::hopeless(&err), "not hopeless: {err:#}"); + assert_eq!(f.requests().len(), 1); + assert!(!f.requeued().await?, "no follow-up alignment is queued"); + anyhow::Ok(()) + } + .await; + f.cleanup().await?; + run +} + /// 签名的向量存下来（#1097）：库没变，第二轮一次嵌入也不叫，也不再问模型。换了嵌入模型， /// 旧向量与新的属性向量不在一个空间里，重嵌一次，旧模型的行删掉 #[tokio::test] diff --git a/crates/utopia-server/src/type_alignment.rs b/crates/utopia-server/src/type_alignment.rs index 7052acff6..3f0a5c6e3 100644 --- a/crates/utopia-server/src/type_alignment.rs +++ b/crates/utopia-server/src/type_alignment.rs @@ -43,6 +43,8 @@ type Vote = Option; /// /// 检索报错也退回整表，同短语那边：候选算在判定的指纹里，报错这一轮判的词恢复以后 /// 指纹对不上、再问一轮。不退回的话，嵌入端点一直坏着时新来的词就永远没有类 +/// +/// 例外是余额用完：不退回整表，原样返回这个错，让这一轮失败一次并报警，而不是接着问、接着排队。 async fn candidates_for( state: &AppState, kb_id: Uuid, @@ -63,6 +65,7 @@ async fn candidates_for( .await { Ok(nearest) => nearest, + Err(e) if utopia_llm::out_of_credit(&e).is_some() => return Err(e), Err(e) => { tracing::warn!(%kb_id, error = %e, "类别词对齐检索候选类失败，退回整表"); Vec::new() @@ -294,6 +297,8 @@ async fn align_types_locked( .await { Ok(r) => r, + // 欠费让整轮失败一次、告警响起，不按 failed 记了再排（同短语对齐，#1360） + Err(e) if utopia_llm::out_of_credit(&e).is_some() => return Err(e), Err(e) => { tracing::warn!(%kb_id, error = %e, "类别词对齐调用失败，这一批留到下次"); failed += 1; diff --git a/crates/utopia-server/src/type_alignment_tests.rs b/crates/utopia-server/src/type_alignment_tests.rs index ede11c5f4..63f7b72d2 100644 --- a/crates/utopia-server/src/type_alignment_tests.rs +++ b/crates/utopia-server/src/type_alignment_tests.rs @@ -1,5 +1,6 @@ //! A disputed kind-word binding no longer projects its old class onto entities. use super::*; +use anyhow::Context as _; use axum::{extract::State, response::IntoResponse, routing::post, Json, Router}; use serde_json::{json, Value}; use std::sync::{Arc, Mutex}; @@ -12,7 +13,7 @@ struct Model { entered: Arc, release: Arc, } -async fn reply(State(m): State, Json(body): Json) -> impl IntoResponse { +async fn reply(State(m): State, Json(body): Json) -> axum::response::Response { let n = { let mut seen = m.requests.lock().unwrap(); seen.push(body); @@ -22,17 +23,21 @@ async fn reply(State(m): State, Json(body): Json) -> impl IntoResp m.entered.notify_one(); m.release.notified().await; } - let text = m - .replies - .get(n) - .expect("unexpected model request") - .to_string(); - let frame = json!({"choices":[{"delta":{"content":text}}]}); + let text = m.replies.get(n).expect("unexpected model request"); + if text == OUT_OF_CREDIT { + let body = json!({"error": {"message": "You have no credits remaining.", + "type": "insufficient_quota", "code": "insufficient_quota"}}); + return (axum::http::StatusCode::TOO_MANY_REQUESTS, Json(body)).into_response(); + } + let frame = json!({"choices":[{"delta":{"content":text.to_string()}}]}); ( [("content-type", "text/event-stream")], format!("data: {frame}\n\ndata: [DONE]\n\n"), ) + .into_response() } +/// 脚本里放这个字符串，对话端点就回 OpenAI 余额耗尽的 429 +const OUT_OF_CREDIT: &str = "out_of_credit"; /// 嵌入端点一直坏着：只有配了嵌入模型的测试会走到这里 async fn embeddings_down() -> impl IntoResponse { ( @@ -1037,3 +1042,27 @@ async fn a_retrieval_error_falls_back_to_the_whole_class_list() -> anyhow::Resul f.cleanup().await?; run } + +/// 余额空了（#1360）：第一个欠费 429 让整轮带着类型失败，不按「这一批留到下次」再排一份 +#[tokio::test] +async fn an_empty_balance_fails_the_run_instead_of_queueing_another() -> anyhow::Result<()> { + let Some(f) = Fx::new(vec![json!(OUT_OF_CREDIT)]).await? else { + return Ok(()); + }; + let run = async { + f.seed_bound().await?; + let err = f + .run() + .await + .err() + .context("a refused chat call fails the run")?; + anyhow::ensure!(crate::alerting::hopeless(&err), "not hopeless: {err:#}"); + anyhow::ensure!(f.requests().len() == 1); + let jobs = align_types_jobs(&f).await?; + anyhow::ensure!(jobs.is_empty(), "nothing is queued, got {jobs:?}"); + anyhow::Ok(()) + } + .await; + f.cleanup().await?; + run +} From a742eee6c1bab63a1b16de54376d1c1d51d81bc4 Mon Sep 17 00:00:00 2001 From: Quintin Botes Date: Fri, 23 Oct 2026 12:56:30 +0200 Subject: [PATCH 7/9] A settings probe that runs out of time reads as timed out, not refused (#1354) (#1373) * A settings probe that runs out of time reads as timed out, not refused (#1354) Signed-off-by: Quintin Botes * A probe that hits the client's read timeout before the probe limit also reads as timed out. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Quintin Botes --------- Signed-off-by: Quintin Botes Co-authored-by: Claude Opus 5.5 Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- .../utopia-server/src/api/settings_routes.rs | 93 +++++++++++++++++-- web/src/api.ts | 2 +- web/src/i18n/en.ts | 1 + web/src/i18n/zh.ts | 1 + web/src/pages/Settings.test.ts | 15 +++ web/src/pages/Settings.tsx | 8 +- 6 files changed, 107 insertions(+), 13 deletions(-) diff --git a/crates/utopia-server/src/api/settings_routes.rs b/crates/utopia-server/src/api/settings_routes.rs index d837c5bda..ff265d9ae 100644 --- a/crates/utopia-server/src/api/settings_routes.rs +++ b/crates/utopia-server/src/api/settings_routes.rs @@ -259,10 +259,6 @@ async fn chat_checks(client: &utopia_llm::LlmClient) -> serde_json::Value { limited(async { client.chat_at_streaming(&msg, Some(0.0)).await.map(|_| ()) }), limited(async { client.chat_tools(&ask, &tool).await.map(|_| ()) }), ); - let check = |r: &anyhow::Result<()>| match r { - Ok(()) => json!({ "ok": true }), - Err(e) => json!({ "ok": false, "error": e.to_string() }), - }; let failed: Vec = [("streaming", &streaming), ("tools", &tools)] .into_iter() .filter_map(|(name, r)| r.as_ref().err().map(|e| format!("{name}: {e}"))) @@ -282,16 +278,50 @@ async fn chat_checks(client: &utopia_llm::LlmClient) -> serde_json::Value { const PROBE_LIMIT: std::time::Duration = std::time::Duration::from_secs(45); async fn limited(f: impl std::future::Future>) -> anyhow::Result { - tokio::time::timeout(PROBE_LIMIT, f) + limited_for(PROBE_LIMIT, f).await +} + +async fn limited_for( + limit: std::time::Duration, + f: impl std::future::Future>, +) -> anyhow::Result { + tokio::time::timeout(limit, f) .await - .unwrap_or_else(|_| { - Err(anyhow::anyhow!( - "no answer within {} s", - PROBE_LIMIT.as_secs() - )) + .unwrap_or_else(|_| Err(TimedOut(limit).into())) +} + +/// 一问没在上限里答完。与被拒分开报（#1363 评审）：被拒说的是端点不收这个形状，超时说的是 +/// 它没说话，读的人该去查的东西不一样 +#[derive(Debug)] +struct TimedOut(std::time::Duration); + +impl std::fmt::Display for TimedOut { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "no answer within {} s", self.0.as_secs()) + } +} + +impl std::error::Error for TimedOut {} + +/// 没答完：撞上探测上限，或者更早撞上探测客户端自己的读超时（`chat_probe_client` 是 30 s，比上限短） +fn timed_out(e: &anyhow::Error) -> bool { + e.downcast_ref::().is_some() + || e.chain().any(|c| { + c.downcast_ref::() + .is_some_and(|u| u.0.is_timeout()) }) } +fn check(r: &anyhow::Result<()>) -> serde_json::Value { + match r { + Ok(()) => json!({ "ok": true }), + Err(e) if timed_out(e) => { + json!({ "ok": false, "timed_out": true, "error": e.to_string() }) + } + Err(e) => json!({ "ok": false, "error": e.to_string() }), + } +} + /// 连通性测试：对话按任务的三种请求形状各试一次（见 [`chat_checks`]）；embedding 试算一条并 /// 返回维度；版面识别服务问一声健康。 pub async fn test( @@ -428,6 +458,49 @@ mod tests { assert_eq!(tooled["tools"][0]["function"]["name"], "ping"); } + #[tokio::test] + async fn a_probe_that_runs_out_of_time_reads_as_timed_out_not_refused() { + let slow = async { + tokio::time::sleep(std::time::Duration::from_secs(5)).await; + anyhow::Ok(()) + }; + let r = super::limited_for(std::time::Duration::from_millis(20), slow).await; + let c = super::check(&r); + assert_eq!(c["timed_out"], true, "{c}"); + assert!( + c["error"].as_str().unwrap().starts_with("no answer within"), + "{c}" + ); + let refused = super::check(&Err(anyhow::anyhow!("HTTP 400"))); + assert!(refused.get("timed_out").is_none(), "{refused}"); + } + + #[tokio::test] + async fn a_read_timeout_before_the_limit_reads_as_timed_out() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(200).set_delay(std::time::Duration::from_secs(2))) + .mount(&server) + .await; + let client = utopia_llm::LlmClient::with_timeouts( + &server.uri(), + None, + "m", + std::time::Duration::from_secs(1), + std::time::Duration::from_millis(100), + ); + let msg = [utopia_llm::ChatMessage { + role: "user".into(), + content: "x".into(), + }]; + let r = super::limited_for(std::time::Duration::from_secs(10), async { + client.chat(&msg).await.map(|_| ()) + }) + .await; + let c = super::check(&r); + assert_eq!(c["timed_out"], true, "{c}"); + } + #[tokio::test] async fn an_endpoint_that_is_down_is_reported_once() { let server = endpoint(true).await; diff --git a/web/src/api.ts b/web/src/api.ts index a1d1c650c..6a997ba74 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -2766,7 +2766,7 @@ export const api = { ok: boolean; reply?: string; error?: string; - checks?: Record<"plain" | "streaming" | "tools", { ok: boolean; error?: string }>; + checks?: Record<"plain" | "streaming" | "tools", { ok: boolean; timed_out?: boolean; error?: string }>; }; embed: { ok: boolean; dim?: number; error?: string }; ocr?: { ok: boolean; version?: string | null; error?: string }; diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts index 5f774207f..8c5310367 100644 --- a/web/src/i18n/en.ts +++ b/web/src/i18n/en.ts @@ -1357,6 +1357,7 @@ export const en = { }, chatShapeOk: "ok", chatShapeRefused: (error: string) => `refused: ${error}`, + chatShapeTimedOut: (error: string) => `timed out (${error})`, okDim: (dim: number) => `Reachable and authenticated (dim ${dim})`, unsaved: "Unsaved changes. Save this card to test them.", readersTitle: "Reading scans and recordings", diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.ts index 1183a772c..e63b50fc8 100644 --- a/web/src/i18n/zh.ts +++ b/web/src/i18n/zh.ts @@ -1203,6 +1203,7 @@ export const zh: Strings = { }, chatShapeOk: "通过", chatShapeRefused: (error: string) => `被拒：${error}`, + chatShapeTimedOut: (error: string) => `超时（${error}）`, okDim: (dim: number) => `已连通，认证通过（维度 ${dim}）`, unsaved: "有未保存的修改。先保存这张卡，再测试。", readersTitle: "读扫描件与录音", diff --git a/web/src/pages/Settings.test.ts b/web/src/pages/Settings.test.ts index 91d8d7b47..0e73b4fbc 100644 --- a/web/src/pages/Settings.test.ts +++ b/web/src/pages/Settings.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from "vitest"; +import { S } from "../i18n"; import { chatTestMessage, modelCardStatus } from "./Settings"; describe("model card status", () => { @@ -83,6 +84,20 @@ describe("chat test message", () => { expect(lines[0]).not.toContain("refused"); }); + it("says a shape timed out rather than that it was refused", () => { + const text = chatTestMessage({ + ok: false, + checks: { + plain: { ok: true }, + streaming: { ok: false, timed_out: true, error: "no answer within 45 s" }, + tools: { ok: false, error: "HTTP 400" }, + }, + }); + const [, streaming, tools] = text.split("\n"); + expect(streaming).toContain(S.settings.chatShapeTimedOut("no answer within 45 s")); + expect(tools).toContain(S.settings.chatShapeRefused("HTTP 400")); + }); + it("falls back to one sentence when the server sends no checks", () => { expect(chatTestMessage({ ok: false, error: "Not configured" })).toBe("Not configured"); }); diff --git a/web/src/pages/Settings.tsx b/web/src/pages/Settings.tsx index 554c33db6..466e2edce 100644 --- a/web/src/pages/Settings.tsx +++ b/web/src/pages/Settings.tsx @@ -61,13 +61,17 @@ export function chatTestMessage(chat: { ok: boolean; reply?: string; error?: string; - checks?: Record<"plain" | "streaming" | "tools", { ok: boolean; error?: string }>; + checks?: Record<"plain" | "streaming" | "tools", { ok: boolean; timed_out?: boolean; error?: string }>; }): string { if (!chat.checks) return chat.ok ? S.settings.ok(chat.reply ?? "OK") : (chat.error ?? ""); return (["plain", "streaming", "tools"] as const) .map((shape) => { const c = chat.checks![shape]; - const verdict = c.ok ? S.settings.chatShapeOk : S.settings.chatShapeRefused(c.error ?? ""); + const verdict = c.ok + ? S.settings.chatShapeOk + : c.timed_out + ? S.settings.chatShapeTimedOut(c.error ?? "") + : S.settings.chatShapeRefused(c.error ?? ""); return `${S.settings.chatShapes[shape]}: ${verdict}`; }) .join("\n"); From 63b8fd3019c42e77689ccee29f9fd590506ca9f1 Mon Sep 17 00:00:00 2001 From: Quintin Botes Date: Fri, 23 Oct 2026 13:01:57 +0200 Subject: [PATCH 8/9] GitHub and Jira tickets render through one Ticket document, byte for byte as before, so GitLab issues can use it too (#1327) (#1378) * Record what every fixture ticket renders today, so moving GitHub and Jira onto one renderer can be checked byte for byte (#1327) Signed-off-by: Quintin Botes * GitHub and Jira tickets render through one Ticket document, byte for byte as before, so GitLab issues can use it too (#1327) Signed-off-by: Quintin Botes * The ticket stamp imports sit with the other imports, and decision 0013's status names the Ticket document as built. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Quintin Botes --------- Signed-off-by: Quintin Botes Co-authored-by: Claude Opus 5.5 Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- crates/utopia-server/src/github_issues.rs | 99 +++++++------------ crates/utopia-server/src/jira_issues.rs | 95 ++++++------------ crates/utopia-server/src/main.rs | 3 + crates/utopia-server/src/ticket.rs | 72 ++++++++++++++ .../utopia-server/src/ticket_golden_tests.rs | 95 ++++++++++++++++++ .../tests/fixtures/ticket_render_golden.json | 11 +++ ...3-a-source-should-hand-over-its-history.md | 2 +- 7 files changed, 249 insertions(+), 128 deletions(-) create mode 100644 crates/utopia-server/src/ticket.rs create mode 100644 crates/utopia-server/src/ticket_golden_tests.rs create mode 100644 crates/utopia-server/tests/fixtures/ticket_render_golden.json diff --git a/crates/utopia-server/src/github_issues.rs b/crates/utopia-server/src/github_issues.rs index 0b7254662..2dee2a118 100644 --- a/crates/utopia-server/src/github_issues.rs +++ b/crates/utopia-server/src/github_issues.rs @@ -37,10 +37,11 @@ //! `pull_request` 字段区分。默认排除：问"工单系统"要的是工单。但留了开关—— //! 有些仓库（包括本仓库）的决策记录实际写在 PR 描述里。 -use crate::time_text; use chrono::{DateTime, Utc}; use serde::Deserialize; +use crate::ticket::stamp; + /// 一次同步最多取多少页（每页 100）。GitHub 的分页没有天然终点， /// 一个活跃仓库能翻很久；这里沿用单端点的分页预算。 const MAX_PAGES: u32 = 10; @@ -101,95 +102,67 @@ fn issue_number_from_url(url: &str) -> Option { url.rsplit('/').next()?.parse().ok() } -/// 工单上的一个时刻，写进正文的样子：到秒、带 `Z`（`2026-08-18T16:18:27Z`） -fn stamp(t: DateTime) -> String { - time_text::world(t, Some("second")) -} - -/// 把一张工单连同它的评论与事件排成一篇文档。 +/// 把一张工单连同它的评论与事件排成一篇文档（[`crate::ticket::render`]）。 /// /// **纯函数，不联网**——取回与组织分开，于是组织这一半测得动。 /// 三次分页的拼装逻辑（谁归谁、按什么排序）恰恰是最容易出错的部分。 pub fn render(issue: &Issue, comments: &[&Comment], events: &[&Event]) -> String { - let mut out = String::new(); - out.push_str(&format!("# #{} {}\n\n", issue.number, issue.title)); - // 抬头写成带日期的陈述句，而不是键值对：抽取器读的是句子。 // "opened by X on 2026-08-18T16:18:27Z" 能抽出带 valid_from 的事实， - // "created_at: 2026-08-18" 则要它自己去猜这是什么意思。 - // - // **时间写到秒**（#691，0024 第 3 节）：GitHub 给的是带 `Z` 的时刻，抽取器读回来就是 - // second 精度。从前截到 UTC 的那一天，起点最多提前 24 小时，同一天的两件事还会被判成 - // 同时发生的冲突。这不是开关——精度是数据自己的，截掉它等于替来源说「不知道几点」 + // "created_at: 2026-08-18" 则要它自己去猜这是什么意思 + let mut sentences = Vec::new(); if let Some(u) = &issue.user { - out.push_str(&format!( - "Opened by {} on {}.\n", + sentences.push(format!( + "Opened by {} on {}.", u.login, stamp(issue.created_at) )); } - out.push_str(&format!("Currently {}.\n", issue.state)); + sentences.push(format!("Currently {}.", issue.state)); if let Some(c) = issue.closed_at { - out.push_str(&format!("Closed on {}.\n", stamp(c))); + sentences.push(format!("Closed on {}.", stamp(c))); } if !issue.labels.is_empty() { let names: Vec<&str> = issue.labels.iter().map(|l| l.name.as_str()).collect(); - out.push_str(&format!("Labelled {}.\n", names.join(", "))); + sentences.push(format!("Labelled {}.", names.join(", "))); } if !issue.assignees.is_empty() { let names: Vec<&str> = issue.assignees.iter().map(|a| a.login.as_str()).collect(); - out.push_str(&format!("Assigned to {}.\n", names.join(", "))); + sentences.push(format!("Assigned to {}.", names.join(", "))); } - - if let Some(b) = issue - .body - .as_deref() - .map(str::trim) - .filter(|b| !b.is_empty()) - { - out.push_str("\n## Description\n\n"); - out.push_str(b); - out.push('\n'); - } - // **状态变更史是这个来源存在的理由。** 每一行都带日期， // 于是账本拿到的是"何时变成什么"，而不是一个静止的当前值 - if !events.is_empty() { - out.push_str("\n## History\n\n"); - for e in events { + let history = events + .iter() + .map(|e| { let who = e.actor.as_ref().map(|a| a.login.as_str()).unwrap_or("?"); let detail = match (e.event.as_str(), &e.label, &e.assignee) { ("labeled" | "unlabeled", Some(l), _) => format!(" ({})", l.name), ("assigned" | "unassigned", _, Some(a)) => format!(" ({})", a.login), _ => String::new(), }; - out.push_str(&format!( - "- {} — {} by {}{}\n", - stamp(e.created_at), - e.event, - who, - detail - )); - } - } - - if !comments.is_empty() { - out.push_str("\n## Comments\n\n"); - for c in comments { - let who = c.user.as_ref().map(|a| a.login.as_str()).unwrap_or("?"); - let body = c.body.as_deref().unwrap_or("").trim(); - if body.is_empty() { - continue; - } - out.push_str(&format!( - "### {} on {}\n\n{}\n\n", - who, - stamp(c.created_at), - body - )); - } - } - out + (e.created_at, format!("{} by {}{}", e.event, who, detail)) + }) + .collect(); + let comments = comments + .iter() + .map(|c| crate::ticket::Comment { + author: c + .user + .as_ref() + .map(|a| a.login.clone()) + .unwrap_or_else(|| "?".into()), + at: Some(c.created_at), + body: c.body.clone().unwrap_or_default(), + }) + .collect(); + crate::ticket::render(&crate::ticket::Ticket { + heading: format!("#{} {}", issue.number, issue.title), + sentences, + description: issue.body.clone(), + history, + comments, + }) } /// 只保留这张工单的评论，按时间升序；响应里归不到它的评论不能混进正文。 diff --git a/crates/utopia-server/src/jira_issues.rs b/crates/utopia-server/src/jira_issues.rs index e71fc57c5..4ade51412 100644 --- a/crates/utopia-server/src/jira_issues.rs +++ b/crates/utopia-server/src/jira_issues.rs @@ -41,6 +41,8 @@ use chrono::{DateTime, Utc}; use serde::Deserialize; +use crate::ticket::stamp; + /// 一次同步最多翻多少页。Jira 的 `total` 常常是几万，全量拉回来没有意义—— /// 增量窗口之外的等下一次 JQL 取。 const MAX_PAGES: u32 = 10; @@ -159,20 +161,13 @@ fn who(u: &Option) -> &str { /// 把一张工单排成一篇文档。**纯函数，不联网**——取回与组织分开，组织这一半测得动。 pub fn render(issue: &Issue) -> String { let f = &issue.fields; - let mut out = String::new(); - out.push_str(&format!( - "# {} {}\n\n", - issue.key, - f.summary.as_deref().unwrap_or("") - )); - // 抬头写成带日期的陈述句，不是键值对：抽取器读的是句子， // "Reported by X on 2026-08-24T11:11:52Z" 能抽出带 valid_from 的事实。 - // 时间写到秒、带 `Z`（#691，0024 第 3 节），与 GitHub 那边同一个写法； // Jira 给的毫秒不写，抽取器也只认到秒 + let mut sentences = Vec::new(); if let (Some(r), Some(c)) = (&f.reporter, &f.created) { - out.push_str(&format!( - "Reported by {} on {}.\n", + sentences.push(format!( + "Reported by {} on {}.", who(&Some(User { display_name: r.display_name.clone() })), @@ -180,41 +175,30 @@ pub fn render(issue: &Issue) -> String { )); } if let Some(t) = name(&f.issuetype) { - out.push_str(&format!("Type {t}.\n")); + sentences.push(format!("Type {t}.")); } if let Some(s) = name(&f.status) { - out.push_str(&format!("Currently {s}.\n")); + sentences.push(format!("Currently {s}.")); } if let Some(p) = name(&f.priority) { - out.push_str(&format!("Priority {p}.\n")); + sentences.push(format!("Priority {p}.")); } if let Some(a) = &f.assignee { - out.push_str(&format!( - "Assigned to {}.\n", + sentences.push(format!( + "Assigned to {}.", a.display_name.as_deref().unwrap_or("?") )); } if let Some(r) = &f.resolutiondate { - out.push_str(&format!("Resolved on {}.\n", stamp(r.0))); + sentences.push(format!("Resolved on {}.", stamp(r.0))); } if !f.labels.is_empty() { - out.push_str(&format!("Labelled {}.\n", f.labels.join(", "))); - } - - if let Some(d) = f - .description - .as_deref() - .map(str::trim) - .filter(|d| !d.is_empty()) - { - out.push_str("\n## Description\n\n"); - out.push_str(d); - out.push('\n'); + sentences.push(format!("Labelled {}.", f.labels.join(", "))); } // **字段级变更史。** Jira 比 GitHub 多给的正是这个：不只是"发生了什么事件"， // 而是"哪个字段从什么变成了什么"——from/to 本身就是一次认知变更的两端 - let mut lines: Vec<(DateTime, String)> = Vec::new(); + let mut history = Vec::new(); for h in issue.changelog.iter().flat_map(|c| c.histories.iter()) { let Some(at) = h.created else { continue }; for item in &h.items { @@ -223,42 +207,30 @@ pub fn render(issue: &Issue) -> String { }; let from = item.from_string.as_deref().unwrap_or("(empty)"); let to = item.to_string.as_deref().unwrap_or("(empty)"); - lines.push(( + history.push(( at.0, - format!( - "- {} — {} changed {field}: {from} → {to}\n", - stamp(at.0), - who(&h.author), - ), + format!("{} changed {field}: {from} → {to}", who(&h.author)), )); } } - if !lines.is_empty() { - // 端点的顺序不是契约，排序自己做——排错了的后果是历史倒着讲 - lines.sort_by_key(|(t, _)| *t); - out.push_str("\n## History\n\n"); - for (_, l) in &lines { - out.push_str(l); - } - } - let comments: Vec<&Comment> = f + let comments = f .comment - .as_ref() - .map(|c| c.comments.iter().collect()) - .unwrap_or_default(); - if !comments.is_empty() { - out.push_str("\n## Comments\n\n"); - for c in comments { - let body = c.body.as_deref().unwrap_or("").trim(); - if body.is_empty() { - continue; - } - let at = c.created.map(|t| stamp(t.0)).unwrap_or_else(|| "?".into()); - out.push_str(&format!("### {} on {}\n\n{}\n\n", who(&c.author), at, body)); - } - } - out + .iter() + .flat_map(|c| c.comments.iter()) + .map(|c| crate::ticket::Comment { + author: who(&c.author).to_string(), + at: c.created.map(|t| t.0), + body: c.body.clone().unwrap_or_default(), + }) + .collect(); + crate::ticket::render(&crate::ticket::Ticket { + heading: format!("{} {}", issue.key, f.summary.as_deref().unwrap_or("")), + sentences, + description: f.description.clone(), + history, + comments, + }) } /// 增量用的 JQL。**时间格式是 Jira 自己的**（`yyyy-MM-dd HH:mm`），不是 RFC3339， @@ -329,11 +301,6 @@ pub async fn fetch_all( Ok((out, total)) } -/// 工单上的一个时刻，写进正文的样子：到秒、带 `Z` -fn stamp(t: DateTime) -> String { - crate::time_text::world(t, Some("second")) -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/utopia-server/src/main.rs b/crates/utopia-server/src/main.rs index ddd8248c0..918a5027a 100644 --- a/crates/utopia-server/src/main.rs +++ b/crates/utopia-server/src/main.rs @@ -39,6 +39,9 @@ mod retrieval; mod rss_full_content; mod search_recovery; mod state; +mod ticket; +#[cfg(test)] +mod ticket_golden_tests; mod time_resolution; mod time_text; mod type_alignment; diff --git a/crates/utopia-server/src/ticket.rs b/crates/utopia-server/src/ticket.rs new file mode 100644 index 000000000..edb2a5282 --- /dev/null +++ b/crates/utopia-server/src/ticket.rs @@ -0,0 +1,72 @@ +//! 一张工单一篇文档，三个工单来源共用这一份排版（0013 决定 7）。 +//! +//! 每个来源把自己的数据装进 [`Ticket`]、自己去取；这里只管把它写成文档。措辞（GitHub 写 +//! "Opened by"、Jira 写 "Reported by"）由来源成句后放进来，这里不改一个字：文档的字节就是 +//! 它的身份，改一个字就是每张工单下次同步都多一个版本。`ticket_golden_tests` 钉着这一点 + +use chrono::{DateTime, Utc}; + +pub struct Ticket { + /// `# ` 后面那一行，如 `#12 Fix it`、`KAFKA-1 Summary` + pub heading: String, + /// 抬头下的句子，已经成句、按来源自己的顺序 + pub sentences: Vec, + /// `## Description` 下的正文；空白的不写这一节 + pub description: Option, + /// `(时刻, 文字)`，写成 `- <时刻> — <文字>`，按时刻稳定排序 + pub history: Vec<(DateTime, String)>, + /// 空正文的评论不写，但只要有评论就有 `## Comments` 这一节，与从前一样 + pub comments: Vec, +} + +pub struct Comment { + pub author: String, + /// 没有时刻的写成 `?` + pub at: Option>, + pub body: String, +} + +/// 工单上的一个时刻，写进正文的样子：到秒、带 `Z`（#691，0024 第 3 节） +pub fn stamp(t: DateTime) -> String { + crate::time_text::world(t, Some("second")) +} + +pub fn render(t: &Ticket) -> String { + let mut out = format!("# {}\n\n", t.heading); + for s in &t.sentences { + out.push_str(s); + out.push('\n'); + } + if let Some(d) = t + .description + .as_deref() + .map(str::trim) + .filter(|d| !d.is_empty()) + { + out.push_str("\n## Description\n\n"); + out.push_str(d); + out.push('\n'); + } + // 端点的顺序不是契约，排序自己做——排错了的后果是历史倒着讲。稳定排序：同一刻的几条 + // 保持来源给的先后 + if !t.history.is_empty() { + let mut history: Vec<&(DateTime, String)> = t.history.iter().collect(); + history.sort_by_key(|(at, _)| *at); + out.push_str("\n## History\n\n"); + for (at, text) in history { + out.push_str(&format!("- {} — {}\n", stamp(*at), text)); + } + } + if !t.comments.is_empty() { + out.push_str("\n## Comments\n\n"); + for c in &t.comments { + let body = c.body.trim(); + if body.is_empty() { + continue; + } + let at = c.at.map(stamp).unwrap_or_else(|| "?".into()); + out.push_str(&format!("### {} on {}\n\n{}\n\n", c.author, at, body)); + } + } + out +} diff --git a/crates/utopia-server/src/ticket_golden_tests.rs b/crates/utopia-server/src/ticket_golden_tests.rs new file mode 100644 index 000000000..f0e2110d0 --- /dev/null +++ b/crates/utopia-server/src/ticket_golden_tests.rs @@ -0,0 +1,95 @@ +//! Every ticket in the GitHub and Jira fixtures renders exactly the text it rendered before the +//! two connectors shared one renderer (0013 decision 7). A changed byte re-versions every ticket +//! on its next sync, so this compares whole documents, not fragments. +//! +//! Regenerate only on purpose: `UTOPIA_WRITE_TICKET_GOLDEN=1 cargo test -p utopia-server ticket_golden`. + +use std::collections::BTreeMap; + +const GOLDEN: &str = "tests/fixtures/ticket_render_golden.json"; + +fn rendered() -> BTreeMap { + use crate::{github_issues as gh, jira_issues as jira}; + #[derive(serde::Deserialize)] + struct Fixture { + issues: Vec, + comments: Vec, + events_by_issue: std::collections::HashMap>, + } + let mut out = BTreeMap::new(); + // The fixtures never label a ticket, assign a GitHub one, leave a comment empty or a Jira comment + // undated, so one variant of each first ticket adds those branches + let mut raw: serde_json::Value = + serde_json::from_str(include_str!("../tests/fixtures/github_issues.json")).unwrap(); + let mut extra = raw["issues"][0].clone(); + let n = extra["number"].as_i64().unwrap() + 1000; + extra["number"] = n.into(); + extra["labels"] = serde_json::json!([{ "name": "bug" }, { "name": "docs" }]); + extra["assignees"] = serde_json::json!([{ "login": "ann" }, { "login": "bo" }]); + let url = raw["comments"][0]["issue_url"] + .as_str() + .unwrap() + .rsplit_once('/') + .unwrap() + .0 + .to_string(); + let at = extra["created_at"].clone(); + raw["comments"].as_array_mut().unwrap().extend([ + serde_json::json!({ "body": " ", "created_at": at, "issue_url": format!("{url}/{n}"), "user": { "login": "ann" } }), + serde_json::json!({ "body": "Looks right.", "created_at": at, "issue_url": format!("{url}/{n}"), "user": null }), + ]); + raw["events_by_issue"][n.to_string()] = serde_json::json!([ + { "event": "labeled", "created_at": at, "actor": { "login": "ann" }, "label": { "name": "bug" } }, + { "event": "assigned", "created_at": at, "actor": null, "assignee": { "login": "bo" } }, + ]); + raw["issues"].as_array_mut().unwrap().push(extra); + let f: Fixture = serde_json::from_value(raw).unwrap(); + for issue in &f.issues { + // the same calls the sync makes (`ingest_sources`) + let cs = gh::comments_for_issue(issue, &f.comments); + let events = gh::sort_events( + f.events_by_issue + .get(&issue.number.to_string()) + .cloned() + .unwrap_or_default(), + ); + let es: Vec<&gh::Event> = events.iter().collect(); + out.insert( + format!("github #{}", issue.number), + gh::render(issue, &cs, &es), + ); + } + let mut raw: serde_json::Value = + serde_json::from_str(include_str!("../tests/fixtures/jira_issues.json")).unwrap(); + let mut extra = raw["issues"][0].clone(); + extra["key"] = format!("{}-VARIANT", extra["key"].as_str().unwrap()).into(); + extra["fields"]["labels"] = serde_json::json!(["backend", "flaky"]); + extra["fields"]["comment"]["comments"] = serde_json::json!([ + { "author": { "displayName": "Ann" }, "body": "Undated.", "created": null }, + { "author": null, "body": " ", "created": null }, + ]); + raw["issues"].as_array_mut().unwrap().push(extra); + let page: jira::SearchPage = serde_json::from_value(raw).unwrap(); + for issue in &page.issues { + out.insert(format!("jira {}", issue.key), jira::render(issue)); + } + out +} + +#[test] +fn ticket_golden_every_fixture_ticket_renders_its_recorded_text() { + let now = rendered(); + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join(GOLDEN); + if std::env::var("UTOPIA_WRITE_TICKET_GOLDEN").is_ok() { + std::fs::write(&path, serde_json::to_string_pretty(&now).unwrap() + "\n").unwrap(); + } + let golden: BTreeMap = + serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap(); + assert_eq!( + now.keys().collect::>(), + golden.keys().collect::>() + ); + for (key, text) in &now { + assert_eq!(text, &golden[key], "{key} renders differently"); + } +} diff --git a/crates/utopia-server/tests/fixtures/ticket_render_golden.json b/crates/utopia-server/tests/fixtures/ticket_render_golden.json new file mode 100644 index 000000000..75650d03d --- /dev/null +++ b/crates/utopia-server/tests/fixtures/ticket_render_golden.json @@ -0,0 +1,11 @@ +{ + "github #1018": "# #1018 Deleting an agent conversation permanently removes all turns and events\n\nOpened by Danmushu on 2026-08-18T16:18:27Z.\nCurrently closed.\nClosed on 2026-08-20T22:31:47Z.\nLabelled bug, docs.\nAssigned to ann, bo.\n\n## Description\n\n# Description\n\nDeleting an agent conversation immediately removes the conversation and all related turns and events from the database.\n\nThe deletion cannot be recovered, and no independent audit record is created to identify who deleted the conversation or when.\n\n# Steps to Reproduce\n\n1. Create an a\n\n## History\n\n- 2026-08-18T16:18:27Z — labeled by ann (bug)\n- 2026-08-18T16:18:27Z — assigned by ? (bo)\n\n## Comments\n\n### ? on 2026-08-18T16:18:27Z\n\nLooks right.\n\n", + "github #15": "# #15 Re-parsing a document can create duplicate ABox entities\n\nOpened by Danmushu on 2026-08-18T16:01:49Z.\nCurrently closed.\nClosed on 2026-08-20T22:31:47Z.\n\n## Description\n\n## Summary\n\nRe-parsing a document replaces its chunks without reconciling the existing entity resolution records and RDF facts associated with those chunks.\n\nAfter reparse, a subsequent ABox extraction may no longer reuse the previous resolution decision. If the resolver selects `new`, another IRI i\n\n## History\n\n- 2026-08-20T22:31:47Z — closed by WaylandYang\n- 2026-08-21T08:12:51Z — referenced by Danmushu\n\n## Comments\n\n### WaylandYang on 2026-08-20T22:31:46Z\n\nCompleted by #20 and merged into dev in commit 6007547.\n\n", + "github #16": "# #16 Expand ABox entity resolution beyond Match and Create\n\nOpened by Danmushu on 2026-08-18T16:07:01Z.\nCurrently closed.\nClosed on 2026-08-20T22:31:47Z.\n\n## Description\n\n## Summary\n\nThe ABox entity resolution workflow currently allows reviewers to match a mention to a suggested candidate or create a new entity.\n\nIt does not support rejecting an invalid mention, explicitly deferring a decision, searching for an entity outside the candidate list, or merging entities t\n\n## History\n\n- 2026-08-20T22:31:47Z — closed by WaylandYang\n- 2026-08-21T08:12:51Z — referenced by Danmushu\n\n## Comments\n\n### WaylandYang on 2026-08-20T22:31:46Z\n\nCompleted by #20 and merged into dev in commit 6007547.\n\n", + "github #17": "# #17 Generic HTTP 409 error is shown when an extraction job is active\n\nOpened by Danmushu on 2026-08-18T16:14:13Z.\nCurrently closed.\nClosed on 2026-08-20T22:31:47Z.\n\n## Description\n\n# Description\n\nWhen a document parse, reparse, or extraction request is blocked by an active extraction job, the UI may display a generic HTTP 409 error without explaining the conflict.\n\n# Steps to Reproduce\n\n1. Start a TBox, ABox, or combined extraction.\n2. Confirm that the extraction job is `pendi\n\n## History\n\n- 2026-08-20T22:31:47Z — closed by WaylandYang\n- 2026-08-21T08:12:51Z — referenced by Danmushu\n\n## Comments\n\n### WaylandYang on 2026-08-20T22:31:46Z\n\nCompleted by #20 and merged into dev in commit 6007547.\n\n", + "github #18": "# #18 Deleting an agent conversation permanently removes all turns and events\n\nOpened by Danmushu on 2026-08-18T16:18:27Z.\nCurrently closed.\nClosed on 2026-08-20T22:31:47Z.\n\n## Description\n\n# Description\n\nDeleting an agent conversation immediately removes the conversation and all related turns and events from the database.\n\nThe deletion cannot be recovered, and no independent audit record is created to identify who deleted the conversation or when.\n\n# Steps to Reproduce\n\n1. Create an a\n\n## History\n\n- 2026-08-20T22:31:47Z — closed by WaylandYang\n- 2026-08-21T08:12:51Z — referenced by Danmushu\n\n## Comments\n\n### WaylandYang on 2026-08-20T22:31:46Z\n\nCompleted by #20 and merged into dev in commit 6007547.\n\n", + "jira KAFKA-20920": "# KAFKA-20920 Document new group coordinator executor metrics\n\nReported by Sean Quah on 2026-08-11T05:45:35Z.\nType Sub-task.\nCurrently Resolved.\nPriority Major.\nAssigned to Sean Quah.\nResolved on 2026-08-28T16:31:21Z.\n\n## Description\n\nWe forgot to update docs/operations/monitoring.md in KAFKA-20211.\n\n## History\n\n- 2026-08-11T05:46:35Z — Sean Quah changed Fix Version: (empty) → 4.4.0\n- 2026-08-11T05:46:45Z — Sean Quah changed Fix Version: (empty) → 4.3.2\n- 2026-08-11T06:08:10Z — ASF GitHub Bot changed RemoteIssueLink: (empty) → This issue links to \"GitHub Pull Request #23127 (Web Link)\"\n- 2026-08-11T06:10:34Z — ASF GitHub Bot changed RemoteIssueLink: This issue links to \"GitHub Pull Request #23127 (Web Link)\" → This issue links to \"GitHub Pull Request #23127 (Web Link)\"\n- 2026-08-11T06:37:48Z — Sean Quah changed assignee: (empty) → Sean Quah\n- 2026-08-13T14:31:36Z — Omnia Ibrahim changed Fix Version: (empty) → 4.5.0\n- 2026-08-13T14:31:36Z — Omnia Ibrahim changed Fix Version: 4.4.0 → (empty)\n- 2026-08-13T14:31:36Z — Omnia Ibrahim changed Fix Version: 4.3.2 → (empty)\n\n## Comments\n\n### Omnia Ibrahim on 2026-08-13T14:31:26Z\n\nMoving to 4.5 as we are now in 4.4.0 code freeze\n\n", + "jira KAFKA-20979": "# KAFKA-20979 Potential data loss caused by time index truncation\n\nReported by Mickael Maison on 2026-08-24T10:49:20Z.\nType Bug.\nCurrently Resolved.\nPriority Major.\nAssigned to Mickael Maison.\nResolved on 2026-08-28T10:45:31Z.\n\n## Description\n\nOn broker startup, I see these lines for many partitions:\r\n\r\n{noformat}\r\n[LocalLog partition=TOPIC-0, dir=LOG_DIR/TOPIC-0] Rolled new log segment at offset 83461 in 0 ms.\r\n[UnifiedLog partition=TOPIC-0, dir=LOG_DIR] Incremented log start offset to 83461 due to segment deletion\r\n[UnifiedLog partition\n\n## History\n\n- 2026-08-24T11:11:52Z — Mickael Maison changed Version: (empty) → 4.1.0\n- 2026-08-24T13:47:12Z — ASF GitHub Bot changed RemoteIssueLink: (empty) → This issue links to \"GitHub Pull Request #23258 (Web Link)\"\n- 2026-08-24T19:52:18Z — ASF GitHub Bot changed RemoteIssueLink: This issue links to \"GitHub Pull Request #23258 (Web Link)\" → This issue links to \"GitHub Pull Request #23258 (Web Link)\"\n- 2026-08-25T01:44:17Z — ASF GitHub Bot changed RemoteIssueLink: This issue links to \"GitHub Pull Request #23258 (Web Link)\" → This issue links to \"GitHub Pull Request #23258 (Web Link)\"\n- 2026-08-26T09:16:22Z — ASF GitHub Bot changed RemoteIssueLink: This issue links to \"GitHub Pull Request #23258 (Web Link)\" → This issue links to \"GitHub Pull Request #23258 (Web Link)\"\n- 2026-08-26T10:41:20Z — ASF GitHub Bot changed RemoteIssueLink: This issue links to \"GitHub Pull Request #23258 (Web Link)\" → This issue links to \"GitHub Pull Request #23258 (Web Link)\"\n\n## Comments\n\n### Luke Chen on 2026-08-24T11:00:50Z\n\nGiven this issue will cause data loss for the impacted partitions, I think we should include this fix in v4.4.0.\n\n### Gaurav Narula on 2026-08-24T11:01:08Z\n\n[~mimaison] Which version was this observed in? In the past, IOException emitted by {{TimeIndex#close}} were swallowed and this was fixed with KAFKA-19221\r\n\r\n \r\n\r\nI've also raised KAFKA-19200 before t\n\n", + "jira KAFKA-20979-VARIANT": "# KAFKA-20979-VARIANT Potential data loss caused by time index truncation\n\nReported by Mickael Maison on 2026-08-24T10:49:20Z.\nType Bug.\nCurrently Resolved.\nPriority Major.\nAssigned to Mickael Maison.\nResolved on 2026-08-28T10:45:31Z.\nLabelled backend, flaky.\n\n## Description\n\nOn broker startup, I see these lines for many partitions:\r\n\r\n{noformat}\r\n[LocalLog partition=TOPIC-0, dir=LOG_DIR/TOPIC-0] Rolled new log segment at offset 83461 in 0 ms.\r\n[UnifiedLog partition=TOPIC-0, dir=LOG_DIR] Incremented log start offset to 83461 due to segment deletion\r\n[UnifiedLog partition\n\n## History\n\n- 2026-08-24T11:11:52Z — Mickael Maison changed Version: (empty) → 4.1.0\n- 2026-08-24T13:47:12Z — ASF GitHub Bot changed RemoteIssueLink: (empty) → This issue links to \"GitHub Pull Request #23258 (Web Link)\"\n- 2026-08-24T19:52:18Z — ASF GitHub Bot changed RemoteIssueLink: This issue links to \"GitHub Pull Request #23258 (Web Link)\" → This issue links to \"GitHub Pull Request #23258 (Web Link)\"\n- 2026-08-25T01:44:17Z — ASF GitHub Bot changed RemoteIssueLink: This issue links to \"GitHub Pull Request #23258 (Web Link)\" → This issue links to \"GitHub Pull Request #23258 (Web Link)\"\n- 2026-08-26T09:16:22Z — ASF GitHub Bot changed RemoteIssueLink: This issue links to \"GitHub Pull Request #23258 (Web Link)\" → This issue links to \"GitHub Pull Request #23258 (Web Link)\"\n- 2026-08-26T10:41:20Z — ASF GitHub Bot changed RemoteIssueLink: This issue links to \"GitHub Pull Request #23258 (Web Link)\" → This issue links to \"GitHub Pull Request #23258 (Web Link)\"\n\n## Comments\n\n### Ann on ?\n\nUndated.\n\n", + "jira KAFKA-20992": "# KAFKA-20992 Remove hamcrest from org.apache.kafka.test package\n\nReported by Chia-Ping Tsai on 2026-08-26T16:07:58Z.\nType Sub-task.\nCurrently Resolved.\nPriority Major.\nAssigned to Hung-Yau Su.\nResolved on 2026-08-28T19:46:33Z.\n\n## Description\n\nMockApiFixedKeyProcessor\n\n## History\n\n- 2026-08-26T16:27:00Z — Hung-Yau Su changed assignee: Chia-Ping Tsai → HungYau Su\n- 2026-08-26T21:02:35Z — ASF GitHub Bot changed RemoteIssueLink: (empty) → This issue links to \"GitHub Pull Request #23287 (Web Link)\"\n- 2026-08-27T01:37:22Z — ASF GitHub Bot changed RemoteIssueLink: This issue links to \"GitHub Pull Request #23287 (Web Link)\" → This issue links to \"GitHub Pull Request #23287 (Web Link)\"\n- 2026-08-27T01:59:34Z — ASF GitHub Bot changed RemoteIssueLink: This issue links to \"GitHub Pull Request #23287 (Web Link)\" → This issue links to \"GitHub Pull Request #23287 (Web Link)\"\n- 2026-08-27T12:26:39Z — ASF GitHub Bot changed RemoteIssueLink: This issue links to \"GitHub Pull Request #23287 (Web Link)\" → This issue links to \"GitHub Pull Request #23287 (Web Link)\"\n- 2026-08-27T18:21:14Z — ASF GitHub Bot changed RemoteIssueLink: This issue links to \"GitHub Pull Request #23287 (Web Link)\" → This issue links to \"GitHub Pull Request #23287 (Web Link)\"\n" +} diff --git a/docs/decisions/0013-a-source-should-hand-over-its-history.md b/docs/decisions/0013-a-source-should-hand-over-its-history.md index a8c1631f4..70e4e99e9 100644 --- a/docs/decisions/0013-a-source-should-hand-over-its-history.md +++ b/docs/decisions/0013-a-source-should-hand-over-its-history.md @@ -1,6 +1,6 @@ # 0013 · A source hands over its history -- **Status**: Implemented for GitHub, Jira and Notion (#134, #135, #213) · open: Feishu, Confluence, decision 7 (#1327) +- **Status**: Implemented for GitHub, Jira and Notion (#134, #135, #213); decision 7's shared ticket document (#1378) · open: Feishu, Confluence, GitLab issues on the ticket document (#1327) - **Written**: 2026-08-31 · condensed into English 2026-09-03 - **Related**: the bitemporal ground of [0001](0001-ontology-import-and-governance.md); the same judgment on the corpus side in `scripts/bench/fetch-wiki-history.mjs` (#122); [0012](0012-the-ontology-is-a-contract-not-a-suggestion.md) is the other end of the line — this record is about how things come in, that one about the rules they land by; the grant layer added afterwards (#142, `data_source_grants`) decides who may mount a source: provenance visible, destination governed From 2402938061540677063c2be6ac398cdec74aec80 Mon Sep 17 00:00:00 2001 From: Szabo Bogdan Date: Fri, 23 Oct 2026 13:07:28 +0200 Subject: [PATCH 9/9] Chat tests that make an answer fail to save share one trigger installed once, so no test locks the messages table while another holds a row lock and reads it (#1391) Signed-off-by: Bogdan Szabo Co-authored-by: Claude Opus 5.5 Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- .../src/api/chat_empty_reply_tests.rs | 50 ++++++++++++++++--- .../src/api/chat_error_code_tests.rs | 18 +------ .../src/api/chat_persistence_tests.rs | 10 +--- .../src/api/chat_registry_tests.rs | 13 ++--- .../utopia-server/src/api/chat_retry_tests.rs | 1 + 5 files changed, 49 insertions(+), 43 deletions(-) diff --git a/crates/utopia-server/src/api/chat_empty_reply_tests.rs b/crates/utopia-server/src/api/chat_empty_reply_tests.rs index b1baf0ba5..028a0acd6 100644 --- a/crates/utopia-server/src/api/chat_empty_reply_tests.rs +++ b/crates/utopia-server/src/api/chat_empty_reply_tests.rs @@ -301,8 +301,50 @@ impl Fx { let _ = std::fs::remove_dir_all(&self.dir); Ok(()) } + + /// From now on an assistant answer in this base fails to save. One trigger, created once + /// and never dropped, rejects marked bases: creating and dropping a trigger per test locks + /// `conversation_messages` under every other test, and a queued `DROP TRIGGER` deadlocks + /// with a test that holds a row lock while it reads messages. + pub(super) async fn reject_answer_saves(&self) -> anyhow::Result<()> { + static INSTALLED: tokio::sync::OnceCell<()> = tokio::sync::OnceCell::const_new(); + INSTALLED + .get_or_try_init(|| async { + sqlx::raw_sql( + "SELECT pg_advisory_xact_lock(hashtext('test_reject_answer_save')); + CREATE OR REPLACE FUNCTION test_reject_answer_save() RETURNS trigger + LANGUAGE plpgsql AS $$ BEGIN + IF NEW.role = 'assistant' AND EXISTS( + SELECT 1 FROM conversations c JOIN knowledge_bases k ON k.id = c.kb_id + WHERE c.id = NEW.conversation_id AND k.name = 'reject-answer-saves') + THEN RAISE EXCEPTION 'private persistence diagnostic'; END IF; + RETURN NEW; + END $$; + DO $$ BEGIN + IF NOT EXISTS(SELECT 1 FROM pg_trigger WHERE tgname = 'test_reject_answer_save' + AND tgrelid = 'conversation_messages'::regclass) THEN + CREATE TRIGGER test_reject_answer_save BEFORE INSERT ON conversation_messages + FOR EACH ROW EXECUTE FUNCTION test_reject_answer_save(); + END IF; + END $$;", + ) + .execute(&self.pool) + .await + .map(drop) + }) + .await?; + sqlx::query("UPDATE knowledge_bases SET name = 'reject-answer-saves' WHERE id = $1") + .bind(self.kb) + .execute(&self.pool) + .await?; + Ok(()) + } } +/// Held by tests that keep a database lock open while they wait on the chat under test, so +/// two such tests never queue behind each other's locks. +pub(super) static HELD_DB_LOCKS: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + /// 发给模型的请求里，最后一条消息的正文 fn last_message(request: &serde_json::Value) -> String { request["messages"] @@ -867,14 +909,8 @@ async fn save_failure_is_an_error_without_publishing_the_buffered_answer_or_retr let Some(f) = fixture(Scripted::new(replies)).await? else { return Ok(()); }; - let name = format!("reject_assistant_{}", f.kb.simple()); - sqlx::raw_sql(&format!("CREATE FUNCTION {name}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.role='assistant' AND EXISTS(SELECT 1 FROM conversations WHERE id=NEW.conversation_id AND kb_id='{}') THEN RAISE EXCEPTION 'injected persistence failure'; END IF; RETURN NEW; END $$; CREATE TRIGGER {name} BEFORE INSERT ON conversation_messages FOR EACH ROW EXECUTE FUNCTION {name}();",f.kb)).execute(&f.pool).await?; + f.reject_answer_saves().await?; let sse = f.ask("What happened?").await?; - sqlx::raw_sql(&format!( - "DROP TRIGGER {name} ON conversation_messages; DROP FUNCTION {name}();" - )) - .execute(&f.pool) - .await?; assert!(sse.contains("event: error"), "{sse}"); assert!(!sse.contains("event: done"), "{sse}"); if budget { diff --git a/crates/utopia-server/src/api/chat_error_code_tests.rs b/crates/utopia-server/src/api/chat_error_code_tests.rs index 6d0106dc0..6a6d8abeb 100644 --- a/crates/utopia-server/src/api/chat_error_code_tests.rs +++ b/crates/utopia-server/src/api/chat_error_code_tests.rs @@ -73,24 +73,8 @@ async fn an_answer_that_cannot_be_saved_says_so_with_its_code() -> anyhow::Resul let Some(f) = fixture(Scripted::new(vec![TOOL, Reply::Text("Accepted answer.")])).await? else { return Ok(()); }; - let name = format!("reject_assistant_{}", f.kb.simple()); - sqlx::raw_sql(&format!( - "CREATE FUNCTION {name}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN \ - IF NEW.role='assistant' AND EXISTS(SELECT 1 FROM conversations \ - WHERE id=NEW.conversation_id AND kb_id='{}') THEN \ - RAISE EXCEPTION 'injected persistence failure'; END IF; RETURN NEW; END $$; \ - CREATE TRIGGER {name} BEFORE INSERT ON conversation_messages \ - FOR EACH ROW EXECUTE FUNCTION {name}();", - f.kb - )) - .execute(&f.pool) - .await?; + f.reject_answer_saves().await?; let sse = f.ask("What happened?").await?; - sqlx::raw_sql(&format!( - "DROP TRIGGER {name} ON conversation_messages; DROP FUNCTION {name}();" - )) - .execute(&f.pool) - .await?; assert_eq!(failure(&sse).0, "answer_not_saved"); f.cleanup().await } diff --git a/crates/utopia-server/src/api/chat_persistence_tests.rs b/crates/utopia-server/src/api/chat_persistence_tests.rs index fcee030ae..31d8ab59b 100644 --- a/crates/utopia-server/src/api/chat_persistence_tests.rs +++ b/crates/utopia-server/src/api/chat_persistence_tests.rs @@ -29,9 +29,8 @@ async fn exercise(deny: bool, disconnect: bool) -> anyhow::Result<()> { .respond_with(model.clone()) .mount(&f._server) .await; - let trigger = format!("reject_chat_{}", f.kb.simple()); if deny { - sqlx::raw_sql(&format!("CREATE FUNCTION {trigger}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.role='assistant' AND EXISTS(SELECT 1 FROM conversations WHERE id=NEW.conversation_id AND kb_id='{}') THEN RAISE EXCEPTION 'private persistence diagnostic'; END IF; RETURN NEW; END $$; CREATE TRIGGER {trigger} BEFORE INSERT ON conversation_messages FOR EACH ROW EXECUTE FUNCTION {trigger}();",f.kb)).execute(&f.pool).await?; + f.reject_answer_saves().await?; } let result = tokio::time::timeout(std::time::Duration::from_secs(20), async { let sse = if disconnect { @@ -89,13 +88,6 @@ async fn exercise(deny: bool, disconnect: bool) -> anyhow::Result<()> { Ok::<_, anyhow::Error>(()) }) .await; - if deny { - sqlx::raw_sql(&format!( - "DROP TRIGGER {trigger} ON conversation_messages; DROP FUNCTION {trigger}();" - )) - .execute(&f.pool) - .await?; - } f.cleanup().await?; result??; Ok(()) diff --git a/crates/utopia-server/src/api/chat_registry_tests.rs b/crates/utopia-server/src/api/chat_registry_tests.rs index ede365b36..2d653accb 100644 --- a/crates/utopia-server/src/api/chat_registry_tests.rs +++ b/crates/utopia-server/src/api/chat_registry_tests.rs @@ -477,6 +477,7 @@ async fn disconnect_keeps_running_busy_does_not_append_and_stale_stop_spares_fol #[tokio::test] async fn stop_after_save_starts_keeps_one_complete_answer_and_holds_busy_until_commit( ) -> anyhow::Result<()> { + let _held = HELD_DB_LOCKS.lock().await; let Some((f, gates, server)) = setup(Pause::AfterTool).await? else { return Ok(()); }; @@ -552,8 +553,7 @@ async fn stopped_answer_save_failure_emits_error_and_releases_the_conversation( let Some((f, gates, server)) = setup(Pause::AfterTool).await? else { return Ok(()); }; - let trigger = format!("reject_stopped_{}", f.kb.simple()); - sqlx::raw_sql(&format!("CREATE FUNCTION {trigger}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.role='assistant' AND EXISTS(SELECT 1 FROM conversations WHERE id=NEW.conversation_id AND kb_id='{}') THEN RAISE EXCEPTION 'private stop persistence diagnostic'; END IF; RETURN NEW; END $$; CREATE TRIGGER {trigger} BEFORE INSERT ON conversation_messages FOR EACH ROW EXECUTE FUNCTION {trigger}();", f.kb)).execute(&f.pool).await?; + f.reject_answer_saves().await?; let result = tokio::time::timeout(std::time::Duration::from_secs(20), async { let id = utopia_store::conversations::create(&f.pool, f.kb, f.user.id, "save failure").await?; @@ -574,9 +574,7 @@ async fn stopped_answer_save_failure_emits_error_and_releases_the_conversation( ); let error: serde_json::Value = serde_json::from_str(&terminals[0].data)?; anyhow::ensure!(error["code"] == "answer_not_saved"); - anyhow::ensure!(!terminals[0] - .data - .contains("private stop persistence diagnostic")); + anyhow::ensure!(!terminals[0].data.contains("private persistence diagnostic")); anyhow::ensure!(f.state.live.attach(id).await.is_none()); anyhow::ensure!( utopia_store::conversations::messages(&f.pool, id) @@ -591,11 +589,6 @@ async fn stopped_answer_save_failure_emits_error_and_releases_the_conversation( gates.release[0].notify_one(); server.abort(); let _ = server.await; - sqlx::raw_sql(&format!( - "DROP TRIGGER {trigger} ON conversation_messages; DROP FUNCTION {trigger}();" - )) - .execute(&f.pool) - .await?; f.cleanup().await?; result??; Ok(()) diff --git a/crates/utopia-server/src/api/chat_retry_tests.rs b/crates/utopia-server/src/api/chat_retry_tests.rs index 50490f059..9c54d4367 100644 --- a/crates/utopia-server/src/api/chat_retry_tests.rs +++ b/crates/utopia-server/src/api/chat_retry_tests.rs @@ -287,6 +287,7 @@ async fn every_question_tells_the_interface_its_stored_id() -> anyhow::Result<() /// 重试资格尚未读完时，另一标签页不能订阅这次随后会被拒绝的占位。 #[tokio::test] async fn a_rejected_retry_does_not_publish_a_stream_to_another_tab() -> anyhow::Result<()> { + let _held = HELD_DB_LOCKS.lock().await; let Some(f) = fixture(Scripted::new(vec![])).await? else { return Ok(()); };