From 92800db074c88a3f98127834dda2d90725066959 Mon Sep 17 00:00:00 2001 From: Szabo Bogdan Date: Sun, 18 Oct 2026 05:20:07 +0200 Subject: [PATCH 01/19] A write token creates an api source as well as a statements source, since both are push-only and a backend mirrors code and docs into a base as plain documents (#1274) Signed-off-by: Bogdan Szabo Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- .../utopia-server/src/api/sources_routes.rs | 22 ++++++++------- .../src/api/token_setup_tests.rs | 27 +++++++++++++++++++ ...ty-from-the-person-scope-from-the-token.md | 11 +++++--- docs/design/access-and-audit.md | 2 +- 4 files changed, 48 insertions(+), 14 deletions(-) diff --git a/crates/utopia-server/src/api/sources_routes.rs b/crates/utopia-server/src/api/sources_routes.rs index 52db41256..2c786394d 100644 --- a/crates/utopia-server/src/api/sources_routes.rs +++ b/crates/utopia-server/src/api/sources_routes.rs @@ -84,11 +84,12 @@ pub struct CreateBody { pub sync_cron: Option, } -/// 写令牌也能建（0014 决定 6）：给一个镜像仓库配 `statements` 来源，不该要人 +/// 写令牌也能建（0014 决定 6）：给一个镜像仓库配 `statements` 或 `api` 来源，不该要人 /// 一个个手点。 /// -/// 令牌只建 `statements`：`url`/`rss`/`custom` 一建好服务器就按时去拉（`custom` -/// 还带着存下的凭据），漏出去的 CI 令牌不该能让服务器定时去敲任意地址。会话照旧建什么都行 +/// 令牌只建推送型（`statements`/`api`，服务器不替它们拉任何东西）：`url`/`rss`/`custom` +/// 一建好服务器就按时去拉（`custom` 还带着存下的凭据），漏出去的 CI 令牌不该能让服务器定时去敲 +/// 任意地址。会话照旧建什么都行 pub async fn create( State(state): State, caller: SessionOrToken, @@ -96,10 +97,11 @@ pub async fn create( Json(body): Json, ) -> ApiResult> { caller.require_write(Some(kb_id))?; - if caller.pat.is_some() && body.kind != "statements" { + if caller.pat.is_some() && !is_push_kind(&body.kind) { return Err(utopia_core::AppError::CodedForbidden { code: "token_source_kind", - message: "A token creates only statements sources; other kinds take a session".into(), + message: "A token creates only statements or api sources; other kinds take a session" + .into(), } .into()); } @@ -130,7 +132,7 @@ pub async fn create( } // api 来源：生成专属推送密钥（此后可随时经 get_token 查看） let mut ingest_token: Option = None; - if has_push_token(&source) { + if is_push_kind(&source.kind) { let token = new_ingest_token(); utopia_store::sources::set_ingest_token(&state.pool, source.id, &token).await?; ingest_token = Some(token); @@ -156,8 +158,8 @@ pub async fn create( /// 有推送密钥的来源：`api` 推文档，`statements` 推陈述（0054）。两种的密钥同一套 /// 生成、查看、轮换 -fn has_push_token(source: &utopia_core::models::Source) -> bool { - matches!(source.kind.as_str(), "api" | "statements") +fn is_push_kind(kind: &str) -> bool { + matches!(kind, "api" | "statements") } /// 查看推送密钥（Editor；列表响应从不携带，查看走这里）。 @@ -171,7 +173,7 @@ pub async fn get_token( caller.require_write(Some(kb_id))?; require_kb(&state, &caller.user, kb_id, Role::Editor).await?; let source = source_in_kb(&state, kb_id, source_id).await?; - if !has_push_token(&source) { + if !is_push_kind(&source.kind) { return Err(utopia_core::AppError::NotFound.into()); } Ok(Json(json!({ "ingest_token": source.ingest_token }))) @@ -185,7 +187,7 @@ pub async fn rotate_token( ) -> ApiResult> { require_kb(&state, &user, kb_id, Role::Editor).await?; let source = source_in_kb(&state, kb_id, source_id).await?; - if !has_push_token(&source) { + if !is_push_kind(&source.kind) { return Err(utopia_core::AppError::NotFound.into()); } let token = new_ingest_token(); diff --git a/crates/utopia-server/src/api/token_setup_tests.rs b/crates/utopia-server/src/api/token_setup_tests.rs index c1b8da788..bf794868b 100644 --- a/crates/utopia-server/src/api/token_setup_tests.rs +++ b/crates/utopia-server/src/api/token_setup_tests.rs @@ -296,6 +296,33 @@ async fn a_live_write_token_of_a_workspace_admin_sets_up_a_base_and_a_dead_one_d result } +#[tokio::test] +async fn a_write_token_creates_an_api_source_and_reads_its_push_key() -> anyhow::Result<()> { + let Some(f) = Fixture::new().await? else { + return Ok(()); + }; + let result = async { + let kb = f.seeded_kb("docs-mirror").await?; + let token = f.token(f.admin, "write", None).await; + + let (status, created) = f.create_source_of_kind(kb, &token, "api").await?; + anyhow::ensure!(status == StatusCode::OK, "{status} {created}"); + let source: Uuid = serde_json::from_value(created["source"]["id"].clone())?; + + let (status, key) = f.push_key(kb, source, &token).await?; + anyhow::ensure!( + status == StatusCode::OK + && key["ingest_token"] == created["ingest_token"] + && key["ingest_token"].is_string(), + "{status} {key} vs {created}" + ); + anyhow::Ok(()) + } + .await; + f.cleanup().await?; + result +} + #[tokio::test] async fn a_write_token_creating_a_url_source_is_refused_with_token_source_kind( ) -> anyhow::Result<()> { diff --git a/docs/decisions/0014-identity-from-the-person-scope-from-the-token.md b/docs/decisions/0014-identity-from-the-person-scope-from-the-token.md index 8c83b8643..ed3082aec 100644 --- a/docs/decisions/0014-identity-from-the-person-scope-from-the-token.md +++ b/docs/decisions/0014-identity-from-the-person-scope-from-the-token.md @@ -70,9 +70,11 @@ push documents in. it cannot create one, since a base that does not exist yet is in no list (`token_kb_out_of_scope`); revocation and expiry are judged in SQL on every request. Nothing here is a base-level machine token or a service account. A token creates only `statements` - sources: a `url`, `rss` or `custom` source starts the server fetching on a schedule, with - stored credentials for `custom`, which would turn a leaked CI token into a way to make the - server call any address; other kinds from a token are refused with `token_source_kind`, and a + and `api` sources, both push-only: the server fetches nothing for them and the client pushes + with the source key, so a backend mirroring code and docs as plain documents sets up its `api` + source the same way. A `url`, `rss` or `custom` source starts the server fetching on a + schedule, with stored credentials for `custom`, which would turn a leaked CI token into a way + to make the server call any address; other kinds from a token are refused with `token_source_kind`, and a session still creates every kind. The audit rows these routes write (`source.created`, `alignment.kind_word_decided`, `ontology.imported`) carry the acting token's id as `detail.token_id`, so before revoking a leaked token an admin can see what it did (decision @@ -104,6 +106,9 @@ push documents in. - 2026-10-15: the status line still said `can_write` was hard-coded false. #318 had already computed it from the token's scope and the person's role, and `remember` has been reachable over MCP with a write token since then. The line now names what is open. +- 2026-10-16: decision 6 let a token create only `statements` sources, though an `api` source is + push-only as well; a backend mirroring several kinds of documents pushes them through one. A + token now creates `api` sources too. ## Open questions diff --git a/docs/design/access-and-audit.md b/docs/design/access-and-audit.md index 4f83985db..cc331f69d 100644 --- a/docs/design/access-and-audit.md +++ b/docs/design/access-and-audit.md @@ -16,7 +16,7 @@ SHA-256 hash, `scope` read or write, `kb_ids` or every base, `expires_at` with 9 default, `last_used_at`, `revoked_at` as a trace); effective permission is the person's role intersected with the token's scope; every MCP call re-runs authentication, scope and role in SQL and writes an audit row `mcp.tool_called`; the tokens page shows the plaintext once beside a client -snippet [0014]. A `write` token also sets up a base as its owner: creating a base and a `statements` source, +snippet [0014]. A `write` token also sets up a base as its owner: creating a base and a `statements` or `api` source, reading the source's push key, importing an ontology and deciding a kind word, with the owner's role unchanged as the ceiling; any token lists a workspace's bases; a token limited by `kb_ids` creates no base. These routes and document content take `SessionOrToken`; apart from MCP, every From e5eba4ba6a14728f6d2f5777b295efe59bea81c2 Mon Sep 17 00:00:00 2001 From: Maya Kid Date: Sun, 18 Oct 2026 14:31:42 +1100 Subject: [PATCH 02/19] Preserve ordinary XLSX imports while bounding sparse grid storage (#1255) * Keep sparse XLSX imports within the grid memory budget Signed-off-by: dada-yan * Preserve ordinary XLSX imports while bounding sparse grid storage Signed-off-by: dada-yan * Preserve readable worksheets when another sheet cannot be decoded Signed-off-by: dada-yan --------- Signed-off-by: dada-yan Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- crates/utopia-ingest/src/parsers.rs | 379 +++++++++++++++--- .../tests/spreadsheet_truncation.rs | 209 +++++++++- 2 files changed, 526 insertions(+), 62 deletions(-) diff --git a/crates/utopia-ingest/src/parsers.rs b/crates/utopia-ingest/src/parsers.rs index 686b59b3c..a80d11ce5 100644 --- a/crates/utopia-ingest/src/parsers.rs +++ b/crates/utopia-ingest/src/parsers.rs @@ -7,6 +7,11 @@ use std::io::{Cursor, Read, Write}; use std::process::Command; const SPREADSHEET_ROW_LIMIT: usize = 2_000; +// Each retained cell passes through a 40-byte GridRow entry and two 48-byte renderer +// Cells, besides source values, strings and Vec spare capacity (~370 B/cell in a +// full dense import). Bound actual records/segments, not their empty column spans: +// a 2,000 x 150 sheet uses 300,000 entries and fits without losing any rows. +const XLSX_SEGMENT_LIMIT: usize = 1_000_000; const CSV_RECORD_LIMIT: usize = 10_000; /// 文本解码：chardetng 探测编码（覆盖 GBK/GB18030/BIG5 等中文常见编码）。 @@ -773,53 +778,200 @@ pub fn spreadsheet(bytes: &[u8]) -> anyhow::Result<(String, Vec { + let Ok(sheet) = xlsx_sheet(book, &sheet_name, XLSX_SEGMENT_LIMIT) else { + continue; + }; + sheet + } + _ => { + let Ok(range) = workbook.worksheet_range(&sheet_name) else { + continue; + }; + let merges = match &mut workbook { + calamine::Sheets::Xls(book) => book + .merge_cells_by_sheet_name(&sheet_name) + .unwrap_or_default(), + _ => Vec::new(), + }; + let (rows, headers) = spreadsheet_grid(&range, &merges); + let text = crate::table::render_grid_with_headers(&rows, headers) + .map(|text| format!("{text}\n")) + .unwrap_or_else(|| { + let mut text = String::new(); + for row in &rows { + let cells: Vec<&str> = row.iter().map(|(s, _, _)| s.as_str()).collect(); + text.push_str(&cells.join("\t")); + text.push('\n'); + } + text + }); + ( + text, + range.height().min(SPREADSHEET_ROW_LIMIT), + range.height(), + ) + } }; - if range.is_empty() { + if rows_read == 0 { continue; } - // xlsx/xls 把 merge 单独给出；xlsb/ods 的 calamine reader 没有这一步，按无 merge 读。 - let merges = match &mut workbook { - calamine::Sheets::Xlsx(book) => book - .merge_cells_by_sheet_name(&sheet_name) - .unwrap_or_default(), - calamine::Sheets::Xls(book) => book - .merge_cells_by_sheet_name(&sheet_name) - .unwrap_or_default(), - _ => Vec::new(), - }; out.push_str(&format!("\n# Sheet: {sheet_name}\n\n")); - let rows_total = range.height(); - if rows_total > SPREADSHEET_ROW_LIMIT { - let rows_read = SPREADSHEET_ROW_LIMIT; + if rows_total > rows_read { warnings.push(crate::ParseWarning { kind: crate::ParseWarning::SPREADSHEET_ROWS_TRUNCATED, detail: serde_json::json!({ - "sheet": sheet_name.clone(), + "sheet": sheet_name, "rows_read": rows_read, "rows_total": rows_total, "rows_omitted": rows_total - rows_read, }), }); } - // 每张表按网格渲染成带列头的 Markdown 表；一格字都没有的表退回制表符分隔 - let (rows, headers) = spreadsheet_grid(&range, &merges); - match crate::table::render_grid_with_headers(&rows, headers) { - Some(md) => { - out.push_str(&md); - out.push('\n'); + out.push_str(&text); + } + Ok((out, warnings)) +} + +fn xlsx_sheet( + book: &mut calamine::Xlsx>>, + sheet: &str, + budget: usize, +) -> anyhow::Result<(String, usize, usize)> { + let mut start_row = u32::MAX; + let mut end_row = 0; + let mut row_counts = vec![0usize; SPREADSHEET_ROW_LIMIT]; + let mut reader = book.worksheet_cells_reader(sheet)?; + // Keep only the first 2,000 physical rows' counts, even if XML cells arrive + // out of order. The full row bounds survive so truncation remains honest. + while let Some(cell) = reader.next_cell()? { + if cell.get_value() == &calamine::DataRef::Empty { + continue; + } + let (row, col) = cell.get_position(); + anyhow::ensure!( + row < 1_048_576 && col < 16_384, + "Cell exceeds the Excel worksheet limits" + ); + if row < start_row { + let shift = (start_row - row).min(SPREADSHEET_ROW_LIMIT as u32) as usize; + row_counts.rotate_right(shift); + row_counts[..shift].fill(0); + start_row = row; + } + end_row = end_row.max(row); + let offset = (row - start_row) as usize; + if let Some(count) = row_counts.get_mut(offset) { + // Repeated coordinates overwrite values below, so even malformed + // XML cannot retain more than one physical row's worth of cells. + *count = (*count + 1).min(16_384); + } + } + drop(reader); + if start_row == u32::MAX { + return Ok((String::new(), 0, 0)); + } + let rows_total = (end_row - start_row + 1) as usize; + let mut height = rows_total.min(SPREADSHEET_ROW_LIMIT); + let mut count = 0; + for (row, &cells) in row_counts[..height].iter().enumerate() { + if cells > budget - count { + height = row; + break; + } + count += cells; + } + // Counts bound this allocation before any source values are retained. A + // distant note adds one entry, not thousands of empty Data/String objects. + let mut values = vec![std::collections::BTreeMap::::new(); height]; + let mut reader = book.worksheet_cells_reader(sheet)?; + while let Some(cell) = reader.next_cell()? { + let (row, col) = cell.get_position(); + let offset = row.saturating_sub(start_row) as usize; + if row >= start_row && offset < height && cell.get_value() != &calamine::DataRef::Empty { + let text = spreadsheet_cell_text(Some(&calamine::Data::from(cell.get_value().clone()))); + values[offset].insert(col, text); + } + } + drop(reader); + let merges = book.merge_cells_by_sheet_name(sheet).unwrap_or_default(); + loop { + let start_col = values + .iter() + .filter_map(|row| row.keys().next()) + .min() + .copied() + .unwrap(); + let end_col = values + .iter() + .filter_map(|row| row.keys().next_back()) + .max() + .copied() + .unwrap(); + let width = (end_col - start_col + 1) as usize; + let text_at = |row: usize, col: usize| { + values[row] + .get(&(start_col + col as u32)) + .cloned() + .unwrap_or_default() + }; + let regions = spreadsheet_regions( + (start_row, start_col), + (start_row + height as u32 - 1, end_col), + &merges, + text_at, + ); + let mut owner = vec![usize::MAX; width]; + let mut rows = Vec::with_capacity(height); + let mut segments = 0; + for row in 0..height { + let cells = spreadsheet_row( + row, + width, + ®ions, + &mut owner, + |col| text_at(row, col), + true, + ); + if cells.len() > budget - segments { + break; } + segments += cells.len(); + rows.push(cells); + } + if rows.len() != height { + height = rows.len(); + values.truncate(height); + // Dropped rows cannot leave a distant column in the kept window's + // geometry. Reproject merges too, retaining their physical spans. + continue; + } + let headers = spreadsheet_headers(®ions, &rows); + let text = match crate::table::render_grid_with_headers(&rows, headers) { + Some(text) => format!("{text}\n"), None => { - for line in &rows { - let cells: Vec<&str> = line.iter().map(|(s, _, _)| s.as_str()).collect(); - out.push_str(&cells.join("\t")); - out.push('\n'); + // Preserve the old TSV fallback exactly, but materialize only + // one dense row at a time rather than the whole rectangle. + let mut text = String::new(); + for row in 0..height { + let cells = spreadsheet_row( + row, + width, + ®ions, + &mut owner, + |col| text_at(row, col), + false, + ); + let cells: Vec<&str> = cells.iter().map(|(s, _, _)| s.as_str()).collect(); + text.push_str(&cells.join("\t")); + text.push('\n'); } + text } - } + }; + return Ok((text, height, rows_total)); } - Ok((out, warnings)) } #[derive(Clone)] @@ -843,9 +995,36 @@ fn spreadsheet_grid( if height == 0 || width == 0 { return (Vec::new(), None); } - let end_row = start_row + height as u32 - 1; - let end_col = start_col + width as u32 - 1; + let text_at = |row, col| spreadsheet_cell_text(range.get((row, col))); + let regions = spreadsheet_regions( + (start_row, start_col), + (start_row + height as u32 - 1, start_col + width as u32 - 1), + merges, + text_at, + ); + let mut owner = vec![usize::MAX; width]; + let rows: Vec<_> = (0..height) + .map(|row| { + spreadsheet_row( + row, + width, + ®ions, + &mut owner, + |col| text_at(row, col), + false, + ) + }) + .collect(); + let headers = spreadsheet_headers(®ions, &rows); + (rows, headers) +} +fn spreadsheet_regions( + (start_row, start_col): (u32, u32), + (end_row, end_col): (u32, u32), + merges: &[calamine::Dimensions], + text_at: impl Fn(usize, usize) -> String, +) -> Vec { let mut regions = Vec::new(); for merge in merges { if merge.start.0 > merge.end.0 @@ -868,30 +1047,40 @@ fn spreadsheet_grid( regions.push(GridMerge { start, end, - text: spreadsheet_cell_text(range.get(start)), + text: text_at(start.0, start.1), }); } + regions +} - let mut owner = vec![usize::MAX; height * width]; +fn spreadsheet_row( + row: usize, + width: usize, + regions: &[GridMerge], + owner: &mut [usize], + text_at: impl Fn(usize) -> String, + compact_empty: bool, +) -> GridRow { + // One owner row keeps the old last-merge-wins projection and anchor jumps, + // without allocating height * width indices. XLSX bounds width at XFD. + owner.fill(usize::MAX); for (index, region) in regions.iter().enumerate() { - for row in region.start.0..=region.end.0 { - for col in region.start.1..=region.end.1 { - owner[row * width + col] = index; - } + if region.start.0 <= row && row <= region.end.0 { + owner[region.start.1..=region.end.1].fill(index); } } - - let mut rows: Vec = Vec::with_capacity(height); - for row in 0..height { - let mut cells = GridRow::new(); - let mut col = 0usize; - while col < width { - let index = owner[row * width + col]; - if index == usize::MAX { - cells.push((spreadsheet_cell_text(range.get((row, col))), 1, 0)); - col += 1; - continue; + let mut cells: GridRow = Vec::new(); + let mut col = 0; + while col < width { + let index = owner[col]; + if index == usize::MAX { + let text = text_at(col); + match (compact_empty && text.is_empty(), cells.last_mut()) { + (true, Some((previous, span, _))) if previous.is_empty() => *span += 1, + _ => cells.push((text, 1, 0)), } + col += 1; + } else { let region = ®ions[index]; if col == region.start.1 { cells.push((region.text.clone(), region.end.1 - region.start.1 + 1, 0)); @@ -900,11 +1089,8 @@ fn spreadsheet_grid( col += 1; } } - rows.push(cells); } - - let headers = spreadsheet_headers(®ions, &rows); - (rows, headers) + cells } fn spreadsheet_cell_text(cell: Option<&calamine::Data>) -> String { @@ -1205,6 +1391,95 @@ fn pptx_xml_to_text(xml: &str) -> anyhow::Result { mod tests { use super::*; + fn budget_book(sheet_data: &str, merges: &str) -> calamine::Xlsx>> { + let worksheet = format!( + r#"{sheet_data}{merges}"# + ); + let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new())); + for (name, xml) in [ + ( + "[Content_Types].xml", + r#""#, + ), + ( + "_rels/.rels", + r#""#, + ), + ( + "xl/workbook.xml", + r#""#, + ), + ( + "xl/_rels/workbook.xml.rels", + r#""#, + ), + ("xl/worksheets/sheet1.xml", &worksheet), + ] { + zip.start_file(name, zip::write::SimpleFileOptions::default()) + .unwrap(); + zip.write_all(xml.as_bytes()).unwrap(); + } + calamine::open_workbook_from_rs(Cursor::new(zip.finish().unwrap().into_inner())).unwrap() + } + + #[test] + fn a_segment_budget_keeps_only_complete_rows() { + // Four source cells fit; two rows of three emitted segments do not. + let rows = r#"leftright12"#; + let (_, rows_read, rows_total) = + xlsx_sheet(&mut budget_book(rows, ""), "Budget", 5).unwrap(); + assert_eq!((rows_read, rows_total), (1, 2)); + } + + #[test] + fn dropping_a_row_reprojects_the_retained_title_and_merged_span() { + let title = r#"Title"#; + let rest = r#" note"#; + let merges = r#""#; + // Four source cells fit, but the first two rows use 3 + 4 segments. + let (text, rows_read, rows_total) = xlsx_sheet( + &mut budget_book(&format!("{title}{rest}"), merges), + "Budget", + 5, + ) + .unwrap(); + let (expected, _, _) = xlsx_sheet(&mut budget_book(title, merges), "Budget", 5).unwrap(); + assert_eq!((rows_read, rows_total), (1, 3)); + assert!(text.contains("Title")); + assert_eq!(text, expected); + } + + #[test] + fn empty_runs_keep_the_physical_cell_and_merge_coordinates() { + let regions = [GridMerge { + start: (0, 2), + end: (0, 4), + text: "Merged".into(), + }]; + let row = spreadsheet_row( + 0, + 10, + ®ions, + &mut [usize::MAX; 10], + |col| match col { + 0 => "Item".into(), + 7 => "3".into(), + _ => String::new(), + }, + true, + ); + let expected = [ + ("Item", 1), + ("", 1), + ("Merged", 3), + ("", 2), + ("3", 1), + ("", 2), + ] + .map(|(text, span)| (text.to_owned(), span, 0)); + assert_eq!(row, expected); + } + const DOC: &str = r#" Segment results SegmentRevenue diff --git a/crates/utopia-ingest/tests/spreadsheet_truncation.rs b/crates/utopia-ingest/tests/spreadsheet_truncation.rs index e4f8b7cde..d4df2fd24 100644 --- a/crates/utopia-ingest/tests/spreadsheet_truncation.rs +++ b/crates/utopia-ingest/tests/spreadsheet_truncation.rs @@ -4,7 +4,7 @@ use std::fmt::Write as _; use std::io::{Cursor, Write}; -fn workbook(data_rows: usize) -> Vec { +fn ledger_rows(data_rows: usize) -> String { let mut sheet_data = String::from( r#"idname"#, ); @@ -14,17 +14,69 @@ fn workbook(data_rows: usize) -> Vec { r#"{id}customer-{id}"# )); } - let worksheet = format!( - r#"{sheet_data}"# + sheet_data +} + +fn workbook(data_rows: usize) -> Vec { + workbook_xml(&ledger_rows(data_rows)) +} + +fn workbook_xml(sheet_data: &str) -> Vec { + workbook_sheets(&[("Ledger", sheet_data)]) +} + +fn workbook_sheets(sheets: &[(&str, &str)]) -> Vec { + workbook_sheets_with_chart(sheets, false, "") +} + +fn workbook_sheets_with_chart( + sheets: &[(&str, &str)], + chart: bool, + worksheet_suffix: &str, +) -> Vec { + let mut content_types = String::from( + r#""#, ); + let mut sheet_entries = String::new(); + let mut sheet_rels = String::new(); + let mut parts = Vec::new(); + for (i, (name, sheet_data)) in sheets.iter().enumerate() { + let id = i + 1; + write!(content_types, r#""#).unwrap(); + write!( + sheet_entries, + r#""# + ) + .unwrap(); + write!(sheet_rels, r#""#).unwrap(); + parts.push((format!("xl/worksheets/sheet{id}.xml"), format!(r#"{sheet_data}{worksheet_suffix}"#))); + } + if chart { + let id = sheets.len() + 1; + write!(content_types, r#""#).unwrap(); + write!( + sheet_entries, + r#""# + ) + .unwrap(); + write!(sheet_rels, r#""#).unwrap(); + parts.extend([ + (format!("xl/chartsheets/sheet{id}.xml"), r#""#.into()), + (format!("xl/chartsheets/_rels/sheet{id}.xml.rels"), r#""#.into()), + ("xl/drawings/drawing1.xml".into(), r#""#.into()), + ("xl/drawings/_rels/drawing1.xml.rels".into(), r#""#.into()), + ("xl/charts/chart1.xml".into(), r#"Series 1ItemGeneral1"#.into()), + ]); + } + content_types.push_str(""); + parts.extend([ + ("[Content_Types].xml".into(), content_types), + ("_rels/.rels".into(), r#""#.into()), + ("xl/workbook.xml".into(), format!(r#"{sheet_entries}"#)), + ("xl/_rels/workbook.xml.rels".into(), format!(r#"{sheet_rels}"#)), + ]); let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new())); - for (part, xml) in [ - ("[Content_Types].xml", r#""#.to_string()), - ("_rels/.rels", r#""#.to_string()), - ("xl/workbook.xml", r#""#.to_string()), - ("xl/_rels/workbook.xml.rels", r#""#.to_string()), - ("xl/worksheets/sheet1.xml", worksheet), - ] { + for (part, xml) in parts { zip.start_file(part, zip::write::SimpleFileOptions::default()) .unwrap(); zip.write_all(xml.as_bytes()).unwrap(); @@ -32,6 +84,23 @@ fn workbook(data_rows: usize) -> Vec { zip.finish().unwrap().into_inner() } +fn words(cell: &str, text: &str) -> String { + format!(r#"{text}"#) +} + +fn column(mut col: usize) -> String { + let mut letters = Vec::new(); + loop { + letters.push(b'A' + (col % 26) as u8); + if col < 26 { + break; + } + col = col / 26 - 1; + } + letters.reverse(); + String::from_utf8(letters).unwrap() +} + fn csv(records: usize) -> String { let mut csv = String::from("id,name\n"); for id in 1..=records { @@ -78,6 +147,126 @@ fn a_spreadsheet_at_the_row_cap_has_no_warning() { assert!(parsed.warnings.is_empty(), "{:?}", parsed.warnings); } +#[test] +fn a_distant_cell_past_the_row_window_does_not_shrink_the_retained_ledger() { + let mut rows = ledger_rows(1_999); + rows.push_str(&format!( + r#"{}"#, + words("P1000000", "omitted-distant-note") + )); + let parsed = utopia_ingest::parse("ledger.xlsx", &workbook_xml(&rows)).unwrap(); + assert!(parsed.text.contains("| 1999 | customer-1999 |")); + assert!(!parsed.text.contains("omitted-distant-note")); + assert_eq!(parsed.warnings.len(), 1); + assert_eq!(parsed.warnings[0].kind, "spreadsheet.rows_truncated"); + assert_eq!( + parsed.warnings[0].detail, + serde_json::json!({ + "sheet": "Ledger", "rows_read": 2000, + "rows_total": 1000000, "rows_omitted": 998000, + }) + ); +} + +#[test] +fn a_header_note_beyond_empty_columns_does_not_truncate_a_ledger() { + let rows = ledger_rows(1_500); + let expected = utopia_ingest::parse("ledger.xlsx", &workbook_xml(&rows)).unwrap(); + assert!(expected.text.contains("| 1500 | customer-1500 |")); + for reference in ["AZZ1", "XFD1"] { + let with_note = rows.replacen( + "", + &format!("{}", words(reference, "header-only-note")), + 1, + ); + let parsed = utopia_ingest::parse("ledger.xlsx", &workbook_xml(&with_note)).unwrap(); + assert!( + parsed.text == expected.text, + "note at {reference} changed the ledger" + ); + assert!( + parsed.warnings.is_empty(), + "{reference}: {:?}", + parsed.warnings + ); + } +} + +#[test] +fn an_ordinary_dense_sheet_retains_two_thousand_rows_and_one_hundred_fifty_columns() { + let columns: Vec<_> = (0..150).map(column).collect(); + let mut rows = String::from(r#""#); + for (col, name) in columns.iter().enumerate() { + rows.push_str(&words(&format!("{name}1"), &format!("column-{col}"))); + } + rows.push_str(""); + for row in 2..=2_000 { + write!( + rows, + r#"{}"#, + words(&format!("A{row}"), &format!("customer-{}", row - 1)) + ) + .unwrap(); + for (col, name) in columns.iter().enumerate().skip(1) { + write!( + rows, + r#"{}"#, + row * 1_000 + col + ) + .unwrap(); + } + rows.push_str(""); + } + let parsed = utopia_ingest::parse("ledger.xlsx", &workbook_xml(&rows)).unwrap(); + assert!(parsed.warnings.is_empty(), "{:?}", parsed.warnings); + let last = parsed + .text + .lines() + .find(|line| line.starts_with("| customer-1999 |")) + .expect("the final row is retained"); + assert!(last.ends_with("| 2000149 |"), "the last column is retained"); + assert_eq!(last.split('|').count(), 152, "all 150 columns are present"); +} + +#[test] +fn a_broken_sheet_does_not_prevent_reading_the_other_sheets() { + let good = ledger_rows(1); + let expected = utopia_ingest::parse("ledger.xlsx", &workbook_xml(&good)).unwrap(); + let broken = r#"not-a-number"#; + let bytes = workbook_sheets(&[("Ledger", &good), ("Broken", broken)]); + let parsed = utopia_ingest::parse("ledger.xlsx", &bytes).unwrap(); + assert_eq!(parsed.text, expected.text); + assert!(parsed.warnings.is_empty(), "{:?}", parsed.warnings); +} + +#[test] +fn unreadable_merges_do_not_prevent_reading_the_worksheet_cells() { + use calamine::Reader as _; + + let rows = ledger_rows(1); + let expected = utopia_ingest::parse("ledger.xlsx", &workbook_xml(&rows)).unwrap(); + let merges = r#""#; + let bytes = workbook_sheets_with_chart(&[("Ledger", &rows)], false, merges); + let mut book: calamine::Xlsx<_> = + calamine::open_workbook_from_rs(Cursor::new(bytes.clone())).unwrap(); + assert_eq!(book.worksheet_range("Ledger").unwrap().height(), 2); + assert!(book.merge_cells_by_sheet_name("Ledger").is_err()); + + let parsed = utopia_ingest::parse("ledger.xlsx", &bytes).unwrap(); + assert_eq!(parsed.text, expected.text); + assert!(parsed.warnings.is_empty(), "{:?}", parsed.warnings); +} + +#[test] +fn a_chartsheet_does_not_prevent_reading_the_worksheets() { + let rows = ledger_rows(1); + let expected = utopia_ingest::parse("ledger.xlsx", &workbook_xml(&rows)).unwrap(); + let bytes = workbook_sheets_with_chart(&[("Ledger", &rows)], true, ""); + let parsed = utopia_ingest::parse("ledger.xlsx", &bytes).unwrap(); + assert_eq!(parsed.text, expected.text); + assert!(parsed.warnings.is_empty(), "{:?}", parsed.warnings); +} + #[test] fn a_csv_past_the_record_cap_reports_the_records_it_left_out() { let parsed = utopia_ingest::parse("customers.csv", csv(10_000).as_bytes()).unwrap(); From ff52e45fd1ae7303390849aecd68d9e7217c5b57 Mon Sep 17 00:00:00 2001 From: Daniel <118035379+Floating-Y@users.noreply.github.com> Date: Sun, 18 Oct 2026 11:36:20 +0800 Subject: [PATCH 03/19] Reject ambiguous email matches so SSO cannot link the wrong account (#1258) * Reject ambiguous email matches so SSO cannot link the wrong account Signed-off-by: Floating-Y <118035379+Floating-Y@users.noreply.github.com> * Classify ambiguous SSO accounts as unlinked and record the decision Signed-off-by: Floating-Y <118035379+Floating-Y@users.noreply.github.com> --------- Signed-off-by: Floating-Y <118035379+Floating-Y@users.noreply.github.com> Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- .../src/api/oidc_exchange_tests.rs | 18 +++++++ crates/utopia-store/src/accounts.rs | 17 +++++-- ...an_sso_login_finds_or_opens_its_account.rs | 49 +++++++++++++++++++ ...ity-provider-vouches-for-listed-domains.md | 4 +- 4 files changed, 83 insertions(+), 5 deletions(-) diff --git a/crates/utopia-server/src/api/oidc_exchange_tests.rs b/crates/utopia-server/src/api/oidc_exchange_tests.rs index 817c68dfa..d2b0cf35d 100644 --- a/crates/utopia-server/src/api/oidc_exchange_tests.rs +++ b/crates/utopia-server/src/api/oidc_exchange_tests.rs @@ -211,6 +211,24 @@ async fn exchange_maps_only_linked_active_people_and_spends_each_token_once() -> let (status, body) = exchange(sign_email(&format!("sub-{ann}"), &format!("{ann}@EXAMPLE.test"), true)?).await?; anyhow::ensure!(status == StatusCode::OK && user_of(&body)? == ann, "existing by email: {status} {body}"); + let duplicate = Uuid::now_v7(); + sqlx::query("INSERT INTO users(id,org_id,email,password_hash,display_name) VALUES($1,$2,$3,'unused','Duplicate Ann')") + .bind(duplicate) + .bind(org) + .bind(format!("{ann}@EXAMPLE.TEST")) + .execute(&pool) + .await?; + let (status, body) = exchange(sign_email("ambiguous-ann", &format!("{ann}@example.test"), true)?).await?; + anyhow::ensure!( + status == StatusCode::UNPROCESSABLE_ENTITY + && body["code"] == "oidc_unavailable" + && body["error"] == "The identity could not be mapped to an account", + "ambiguous email: {status} {body}" + ); + let linked: i64 = sqlx::query_scalar("SELECT count(*) FROM oidc_identities WHERE issuer = $1 AND (subject = 'ambiguous-ann' OR user_id = $2)") + .bind(&issuer).bind(duplicate).fetch_one(&pool).await?; + anyhow::ensure!(linked == 0); + let bea_sub = format!("sub-{}", Uuid::now_v7()); let (status, body) = exchange(sign_email(&bea_sub, &bea_email, true)?).await?; anyhow::ensure!(status == StatusCode::OK, "opened: {status} {body}"); diff --git a/crates/utopia-store/src/accounts.rs b/crates/utopia-store/src/accounts.rs index b665ed6c2..b55da1264 100644 --- a/crates/utopia-store/src/accounts.rs +++ b/crates/utopia-store/src/accounts.rs @@ -377,6 +377,8 @@ pub async fn set_system_admin( /// subject 绑上。开户与绑定同一个事务：绑定撞车（`taken` / `already_linked`）时 /// 不留下一个没人能登进去的账号。 /// +/// 大小写不敏感的邮箱匹配必须只有一个活动账号；多个匹配返回冲突，不开户也不绑定。 +/// /// 这个 email 只剩一个停用的账号时返回 `None`：停用的人不能换个新号回来—— /// email 的唯一索引只管启用中的账号，挡不住这一步，得在这里查。 /// @@ -389,13 +391,20 @@ pub async fn account_for_sso( display_name: &str, ) -> AppResult> { let mut tx = pool.begin().await?; - let active: Option = sqlx::query_as( - "SELECT * FROM users WHERE lower(email) = lower($1) AND deactivated_at IS NULL", + // 本次只拒绝快照中已有的歧义；阻止并发开户产生重复邮箱需要另行迁移并处理存量重复。 + let mut active: Vec = sqlx::query_as( + "SELECT * FROM users WHERE lower(email) = lower($1) AND deactivated_at IS NULL LIMIT 2", ) .bind(email) - .fetch_optional(&mut *tx) + .fetch_all(&mut *tx) .await?; - let user = match active { + if active.len() > 1 { + return Err(AppError::CodedConflict { + code: "unlinked", + message: "Multiple active accounts match the SSO email".into(), + }); + } + let user = match active.pop() { Some(u) => u, None => { let deactivated: bool = sqlx::query_scalar( diff --git a/crates/utopia-store/tests/store/an_sso_login_finds_or_opens_its_account.rs b/crates/utopia-store/tests/store/an_sso_login_finds_or_opens_its_account.rs index 7aa956c40..a8ea51095 100644 --- a/crates/utopia-store/tests/store/an_sso_login_finds_or_opens_its_account.rs +++ b/crates/utopia-store/tests/store/an_sso_login_finds_or_opens_its_account.rs @@ -5,6 +5,55 @@ use uuid::Uuid; const ISSUER: &str = "https://id.sso.test"; +#[tokio::test] +async fn ambiguous_active_emails_leave_accounts_and_identities_unchanged() -> anyhow::Result<()> { + let Some(url) = utopia_store::test_db::url() else { + return Ok(()); + }; + let pool = PgPool::connect(&url).await?; + let org = Uuid::now_v7(); + sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, 'sso-ambiguity-test')") + .bind(org) + .execute(&pool) + .await?; + let run = async { + let email = format!("{}-ann@sso.test", Uuid::now_v7().simple()); + let first = Uuid::now_v7(); + for (id, address) in [(first, email.to_uppercase()), (Uuid::now_v7(), email.clone())] { + sqlx::query("INSERT INTO users (id, org_id, email, display_name, password_hash) VALUES ($1, $2, $3, 'Ann', 'x')") + .bind(id).bind(org).bind(address).execute(&pool).await?; + } + let snapshot = || async { + sqlx::query_scalar::<_, serde_json::Value>( + "SELECT jsonb_build_object( + 'users', (SELECT jsonb_agg(to_jsonb(u) ORDER BY id) FROM users u WHERE org_id = $1), + 'identities', (SELECT jsonb_agg(to_jsonb(i) ORDER BY subject) FROM oidc_identities i JOIN users u ON u.id = i.user_id WHERE u.org_id = $1))", + ).bind(org).fetch_one(&pool).await + }; + for already_linked in [false, true] { + if already_linked { + sqlx::query("INSERT INTO oidc_identities (issuer, subject, user_id) VALUES ($1, $2, $3)") + .bind(ISSUER).bind(format!("existing-{first}")).bind(first).execute(&pool).await?; + } + let before = snapshot().await?; + // 两种精确拼写都必须拒绝；已有绑定也不能让另一个候选变得可选。 + for address in [&email, &email.to_uppercase()] { + let result = utopia_store::accounts::account_for_sso( + &pool, ISSUER, &format!("new-{first}"), address, "Changed", + ).await; + assert!(matches!(result, Err(utopia_core::AppError::CodedConflict { code: "unlinked", ref message }) if message == "Multiple active accounts match the SSO email")); + assert_eq!(snapshot().await?, before); + } + } + anyhow::Ok(()) + }.await; + sqlx::query("DELETE FROM organizations WHERE id = $1") + .bind(org) + .execute(&pool) + .await?; + run +} + /// 一个比库里其它组织都早的组织，开户会落进它的默认工作区 /// 用「最早的组织」的测试一个一个跑，组织的时间也比库里现有的都早一天： /// 并行时谁更早说不准，中断的旧测试留下的组织也会抢走「最早」 diff --git a/docs/decisions/0069-the-identity-provider-vouches-for-listed-domains.md b/docs/decisions/0069-the-identity-provider-vouches-for-listed-domains.md index 9449a47e9..22c220ce7 100644 --- a/docs/decisions/0069-the-identity-provider-vouches-for-listed-domains.md +++ b/docs/decisions/0069-the-identity-provider-vouches-for-listed-domains.md @@ -1,6 +1,6 @@ # 0069 · The identity provider vouches for listed domains -- **Status**: Implemented · 2026-10-16 · #1237 · open: no per-domain role; an SSO-opened account has no password and no reset +- **Status**: Implemented · 2026-10-16 · #1237, #1258 · open: no per-domain role; an SSO-opened account has no password and no reset; no global case-insensitive email uniqueness - **Written**: 2026-10-16 (conventions in the [README](README.md)) - **Related**: migration `0056` (an identity is linked by its owner); [0066](0066-another-app-may-trade-a-fresh-id-token-for-a-session.md) (the exchange, whose mapping this widens); [0014](0014-identity-from-the-person-scope-from-the-token.md) (identity comes from the person); [#1237](https://github.com/deeplethe/utopia/pull/1237) @@ -14,6 +14,8 @@ Migration 0056 rejected linking by email for a reason that still holds in genera 1. **`UTOPIA_OIDC_ALLOWED_DOMAINS` lists the domains the identity provider is authoritative for.** On a login or an exchange whose subject is not linked, a token whose `email` is in a listed domain and whose `email_verified` is true is mapped by email: to the active account with that address (case-insensitive), or to a new one. The subject is then linked to it, so later sign-ins go by subject as before. The list is empty by default, and an empty list keeps 0056's rule exactly. + **Revision 2026-10-16 (#1258).** This assumed the active match was unique. Checking `users_email_active_idx` showed that it compares the stored spelling, while registration and administrator-created accounts preserve casing: two active accounts can differ only in email case. `account_for_sso` now refuses multiple active case-insensitive matches as `unlinked` before creating an account or writing an identity, rather than choosing one. The check covers the query snapshot; a unique index on `lower(email)`, normalization at registration and administrator creation, and a decision about existing duplicates remain separate work. + 2. **The trust assumption.** Listing a domain says: every token this provider signs with a verified address in that domain belongs to the person who owns that address. That holds for a provider that only knows the company's directory. It does not hold for a plain Google OAuth client: Google also signs verified tokens for consumer accounts someone registered with a company address, and that person need not be, or still be, an employee. A deployment on such a client should put a gate in front that admits only the company's Workspace accounts, or restrict the client to its organization at the provider. 3. **The account and its link are written in one transaction** (`accounts::account_for_sso`). If the link fails (the subject is already linked to someone else, or the account already has an identity at this issuer), no account is left behind that nobody could sign in to. From e9b2a2421e194cbcc65cb07a2da2ac2797897c7d Mon Sep 17 00:00:00 2001 From: Daniel <118035379+Floating-Y@users.noreply.github.com> Date: Sun, 18 Oct 2026 11:41:32 +0800 Subject: [PATCH 04/19] Keep an active system administrator when revocation and deactivation overlap (#1259) * Keep an active system administrator when account removals overlap Signed-off-by: Floating-Y <118035379+Floating-Y@users.noreply.github.com> * Serialize administrator removals without blocking foreign key checks Signed-off-by: Floating-Y <118035379+Floating-Y@users.noreply.github.com> --------- Signed-off-by: Floating-Y <118035379+Floating-Y@users.noreply.github.com> Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- crates/utopia-store/src/accounts.rs | 42 +++++++- .../tests/store/a_retired_account.rs | 100 ++++++++++++++++++ .../store/an_admin_grants_system_admin.rs | 15 ++- 3 files changed, 153 insertions(+), 4 deletions(-) diff --git a/crates/utopia-store/src/accounts.rs b/crates/utopia-store/src/accounts.rs index b55da1264..49ad7add1 100644 --- a/crates/utopia-store/src/accounts.rs +++ b/crates/utopia-store/src/accounts.rs @@ -292,6 +292,7 @@ pub async fn deactivate_user(pool: &PgPool, target: Uuid, actor: Uuid) -> AppRes return Err(AppError::Validation("不能停用自己的账号".into())); } let mut tx = pool.begin().await?; + lock_account_organization(&mut tx, target).await?; let victim: Option<(bool, Uuid, Option>)> = sqlx::query_as("SELECT is_admin, org_id, deactivated_at FROM users WHERE id = $1") .bind(target) @@ -360,18 +361,57 @@ pub async fn set_system_admin( if target == actor && !value { return Err(AppError::Validation("不能收回自己的系统管理员".into())); } + let mut tx = pool.begin().await?; + let org_id = lock_account_organization(&mut tx, target).await?; + let is_admin: bool = + sqlx::query_scalar("SELECT is_admin FROM users WHERE id = $1 AND deactivated_at IS NULL") + .bind(target) + .fetch_optional(&mut *tx) + .await? + .ok_or(AppError::NotFound)?; + if is_admin && !value { + let others: i64 = sqlx::query_scalar( + "SELECT count(*) FROM users + WHERE org_id = $1 AND is_admin AND deactivated_at IS NULL AND id <> $2", + ) + .bind(org_id) + .bind(target) + .fetch_one(&mut *tx) + .await?; + if others == 0 { + return Err(AppError::Validation( + "这是组织里最后一个管理员，收回权限之后没人能管成员".into(), + )); + } + } let res = sqlx::query("UPDATE users SET is_admin = $2 WHERE id = $1 AND deactivated_at IS NULL") .bind(target) .bind(value) - .execute(pool) + .execute(&mut *tx) .await?; if res.rows_affected() == 0 { return Err(AppError::NotFound); } + tx.commit().await?; Ok(()) } +async fn lock_account_organization( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + target: Uuid, +) -> AppResult { + // 撤权与停用共用组织锁；拿锁后再读账号、数管理员，不能信任请求鉴权时的快照。 + sqlx::query_scalar( + "SELECT id FROM organizations + WHERE id = (SELECT org_id FROM users WHERE id = $1) FOR NO KEY UPDATE", + ) + .bind(target) + .fetch_optional(&mut **tx) + .await? + .ok_or(AppError::NotFound) +} + /// 单点登录允许自动开户时，用身份提供方核实过的 email 找账号；找不到就按注册的 /// 规则开一个（默认工作区的 viewer；部署里还没人时就是首个管理员），并把这个 /// subject 绑上。开户与绑定同一个事务：绑定撞车（`taken` / `already_linked`）时 diff --git a/crates/utopia-store/tests/store/a_retired_account.rs b/crates/utopia-store/tests/store/a_retired_account.rs index fd4374c81..ce84193ac 100644 --- a/crates/utopia-store/tests/store/a_retired_account.rs +++ b/crates/utopia-store/tests/store/a_retired_account.rs @@ -130,3 +130,103 @@ async fn the_last_admin_and_oneself_are_protected() -> anyhow::Result<()> { .await?; run } + +async fn concurrent_admin_removal(deactivate: [bool; 2]) -> anyhow::Result<()> { + let Some(url) = utopia_store::test_db::url() else { + return Ok(()); + }; + let pool = PgPool::connect(&url).await?; + let (org, a, b) = org_with_two_admins(&pool).await?; + let run = async { + let mut gate = pool.begin().await?; + sqlx::query("SELECT id FROM users WHERE org_id = $1 FOR UPDATE") + .bind(org) + .execute(&mut *gate) + .await?; + let mut tasks = tokio::task::JoinSet::new(); + let mut pids = Vec::new(); + for ((actor, target), retire) in [(a, b), (b, a)].into_iter().zip(deactivate) { + let writer = sqlx::postgres::PgPoolOptions::new() + .max_connections(1) + .connect(&url) + .await?; + pids.push( + sqlx::query_scalar::<_, i32>("SELECT pg_backend_pid()") + .fetch_one(&writer) + .await?, + ); + tasks.spawn(async move { + if retire { + utopia_store::accounts::deactivate_user(&writer, target, actor).await + } else { + utopia_store::accounts::set_system_admin(&writer, target, actor, false).await + } + }); + } + // Observe real lock waits, not elapsed time: before the fix both writers + // have passed their checks; afterwards one waits on the organization lock. + let waiting = tokio::time::timeout(std::time::Duration::from_secs(10), async { + loop { + let blocked: i64 = sqlx::query_scalar( + "SELECT count(*) FROM pg_stat_activity + WHERE pid = ANY($1) AND cardinality(pg_blocking_pids(pid)) > 0", + ) + .bind(&pids) + .fetch_one(&pool) + .await?; + if blocked == 2 { + break anyhow::Ok(()); + } + tokio::task::yield_now().await; + } + }) + .await; + gate.commit().await?; + waiting??; + let mut succeeded = 0; + while let Some(result) = + tokio::time::timeout(std::time::Duration::from_secs(10), tasks.join_next()).await? + { + match result? { + Ok(()) => succeeded += 1, + Err(utopia_core::AppError::Validation(_)) => {} + Err(error) => return Err(error.into()), + } + } + let remaining: i64 = sqlx::query_scalar( + "SELECT count(*) FROM users + WHERE org_id = $1 AND is_admin AND deactivated_at IS NULL", + ) + .bind(org) + .fetch_one(&pool) + .await?; + anyhow::Ok((succeeded, remaining)) + } + .await; + sqlx::query("DELETE FROM organizations WHERE id = $1") + .bind(org) + .execute(&pool) + .await?; + let (succeeded, remaining) = run?; + assert_eq!( + remaining, 1, + "concurrent removals must retain an active administrator" + ); + assert_eq!(succeeded, 1, "exactly one removal must succeed"); + Ok(()) +} + +#[tokio::test] +async fn concurrent_revocations_keep_an_active_admin() -> anyhow::Result<()> { + concurrent_admin_removal([false, false]).await +} + +#[tokio::test] +async fn concurrent_deactivations_keep_an_active_admin() -> anyhow::Result<()> { + concurrent_admin_removal([true, true]).await +} + +#[tokio::test] +async fn concurrent_revocation_and_deactivation_keep_an_active_admin() -> anyhow::Result<()> { + concurrent_admin_removal([false, true]).await +} diff --git a/crates/utopia-store/tests/store/an_admin_grants_system_admin.rs b/crates/utopia-store/tests/store/an_admin_grants_system_admin.rs index c1b733648..553bfe1c4 100644 --- a/crates/utopia-store/tests/store/an_admin_grants_system_admin.rs +++ b/crates/utopia-store/tests/store/an_admin_grants_system_admin.rs @@ -44,6 +44,13 @@ async fn granting_a_member_makes_them_admin() -> anyhow::Result<()> { let run = async { utopia_store::accounts::set_system_admin(&pool, member, admin, true).await?; assert!(is_admin(&pool, member).await?); + utopia_store::accounts::set_system_admin(&pool, member, admin, false).await?; + assert!(!is_admin(&pool, member).await?); + // A request admitted before the actor's revocation still cannot remove + // the last administrator. Repeating a non-admin's revocation is harmless. + let refused = utopia_store::accounts::set_system_admin(&pool, admin, member, false).await; + assert!(matches!(refused, Err(utopia_core::AppError::Validation(_)))); + utopia_store::accounts::set_system_admin(&pool, member, admin, false).await?; Ok::<_, anyhow::Error>(()) } .await; @@ -90,9 +97,11 @@ async fn granting_a_deactivated_or_unknown_user_is_not_found() -> anyhow::Result let run = async { utopia_store::accounts::deactivate_user(&pool, member, admin).await?; for target in [member, Uuid::now_v7()] { - let missing = - utopia_store::accounts::set_system_admin(&pool, target, admin, true).await; - assert!(matches!(missing, Err(utopia_core::AppError::NotFound))); + for value in [true, false] { + let missing = + utopia_store::accounts::set_system_admin(&pool, target, admin, value).await; + assert!(matches!(missing, Err(utopia_core::AppError::NotFound))); + } } assert!(!is_admin(&pool, member).await?); Ok::<_, anyhow::Error>(()) From 177ad0ba48582806092b6df68a2f7ac773779e01 Mon Sep 17 00:00:00 2001 From: Szabo Bogdan Date: Sun, 18 Oct 2026 17:48:16 +0200 Subject: [PATCH 05/19] The importer reads utopia:temporal on a property (#1291) * The importer reads utopia:temporal on a property The export has written utopia:temporal on every property that is not a state since 0031, and the importer never read it back: an ontology that said a relation is an event came in as a state, and the rule of #1242, which binds a pushed phrase by the declaration only when the relation is not a state, could never apply to an imported vocabulary. The turtle projection reads urn:utopia:ns:temporal with one of state, event or eternal; any other value is reported under the preview's unprojected predicates and the property stays unset. A new relation is created with the value, a re-import updates an existing relation's temporal to what the file says, and a file that says nothing leaves it alone, so a value a person set on the page survives an unannotated re-import. Attributes stay state. Signed-off-by: Bogdan Szabo * The import reports a bad utopia:temporal under its IRI, the export writes state too, and an import that changes temporal moves updated_at Signed-off-by: Bogdan Szabo --------- Signed-off-by: Bogdan Szabo Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- crates/utopia-ingest/src/ontology_rdf.rs | 54 +++++++++++ .../src/api/token_setup_tests.rs | 94 +++++++++++++++++++ crates/utopia-server/src/owl_import.rs | 5 + crates/utopia-server/src/rdf.rs | 18 +++- crates/utopia-store/src/ontology.rs | 53 +++++++---- .../tests/store/axioms_reach_the_database.rs | 1 + ...1-an-event-holds-at-the-moment-it-names.md | 2 + docs/design/ontology.md | 4 +- 8 files changed, 210 insertions(+), 21 deletions(-) diff --git a/crates/utopia-ingest/src/ontology_rdf.rs b/crates/utopia-ingest/src/ontology_rdf.rs index 00090a0cf..824e5a0d0 100644 --- a/crates/utopia-ingest/src/ontology_rdf.rs +++ b/crates/utopia-ingest/src/ontology_rdf.rs @@ -62,6 +62,10 @@ pub struct OwlProperty { /// `author rangeIncludes Organization, Person` 就会被读成 /// "必须既是组织又是人"，然后降级成 text——一条边就这么没了 pub ranges_union: bool, + /// `utopia:temporal` 的值：`state` / `event` / `eternal`（0031）。导出端早就写这一条， + /// 导入端从前不读——一份本体自己说「这条关系是一个时刻」，导进来还是 `state`。 + /// 没写或写了别的值都是 `None`，落库时按 `state`；数据属性读了也不用 + pub temporal: Option, } /// 一次解析的结果。`unprojected` 是**报告**不是错误——见模块文档。 @@ -85,6 +89,8 @@ pub type VocabDatatypes = BTreeMap; const RDF_TYPE: &str = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"; const RDFS: &str = "http://www.w3.org/2000/01/rdf-schema#"; const OWL: &str = "http://www.w3.org/2002/07/owl#"; +/// 我们自己的注解命名空间，与导出端（`utopia-server::rdf`）同一个 +const UTOPIA_NS: &str = "urn:utopia:ns:"; /// schema.org 自己那套 domain/range。**不是标准词汇**，所以写死在这里， /// 但值得认：schema.org 及派生词汇表一个 `rdfs:domain` 都没有， @@ -342,6 +348,9 @@ pub fn project(bytes: &[u8], format: RdfFormat) -> anyhow::Result let mut ranges: BTreeMap> = BTreeMap::new(); // rangeIncludes 用过的属性，它的 range 按并集读 let mut union_ranged: BTreeSet = BTreeSet::new(); + // `utopia:temporal` 说的时间语义；只认三个值 + let mut temporal: BTreeMap = BTreeMap::new(); + let utopia_temporal = format!("{UTOPIA_NS}temporal"); let known = |p: &str| { p == RDF_TYPE || p == format!("{RDFS}label") @@ -365,6 +374,12 @@ pub fn project(bytes: &[u8], format: RdfFormat) -> anyhow::Result }; for t in &triples { let p = t.predicate.as_str(); + // 导出端写的就是这三个字面值（0031）。别的值不猜：落到下面按谓词报告，落库按 `state` + let declared_temporal = t + .object_lit + .as_ref() + .map(|(v, _)| v.as_str()) + .filter(|v| p == utopia_temporal && matches!(*v, "state" | "event" | "eternal")); if p == format!("{RDFS}label") { if let Some(l) = &t.object_lit { labels.entry(t.subject.clone()).or_default().push(l.clone()); @@ -444,6 +459,8 @@ pub fn project(bytes: &[u8], format: RdfFormat) -> anyhow::Result ranges.entry(t.subject.clone()).or_default().push(o.clone()); union_ranged.insert(t.subject.clone()); } + } else if let Some(v) = declared_temporal { + temporal.entry(t.subject.clone()).or_insert(v.to_string()); } else if !known(p) { // 报告而不是丢弃：预览页要能说清"这个文件里还有什么我们没消费" *proj.unprojected.entry(p.to_string()).or_insert(0) += 1; @@ -582,6 +599,7 @@ pub fn project(bytes: &[u8], format: RdfFormat) -> anyhow::Result domains: domains.get(iri).cloned().unwrap_or_default(), ranges: rs, ranges_union: union_ranged.contains(iri), + temporal: temporal.get(iri).cloned(), iri: iri.clone(), }); } @@ -1716,6 +1734,42 @@ schema:employees a rdf:Property ; rdfs:label "employees" ; "**认了就不该再算进「暂未投影」**" ); } + + /// `utopia:temporal` 要被读出来（0031）。导出端一直写它，导入端从前不读： + /// 一份本体自己说「这条关系是一个时刻」，导进来还是 `state`，#1242 的那条 + /// 「非 state 的关系不经模型就能绑」对它永远不成立。 + #[test] + fn a_property_annotated_with_utopia_temporal_keeps_the_value_it_declared() { + let ttl = r#" +@prefix owl: . +@prefix rdfs: . +@prefix utopia: . +@prefix ex: . + +ex:Person a owl:Class . +ex:attended a owl:ObjectProperty ; rdfs:label "attended" ; utopia:temporal "event" . +ex:born_of a owl:ObjectProperty ; rdfs:label "born of" ; utopia:temporal "eternal" . +ex:works_at a owl:ObjectProperty ; rdfs:label "works at" . +ex:met a owl:ObjectProperty ; rdfs:label "met" ; utopia:temporal "sometimes" . +"#; + let p = project(ttl.as_bytes(), RdfFormat::Turtle).expect("解析"); + let by = |k: &str| { + p.properties + .iter() + .find(|x| x.key == k) + .unwrap_or_else(|| panic!("没有 {k}")) + }; + assert_eq!(by("attended").temporal.as_deref(), Some("event")); + assert_eq!(by("born_of").temporal.as_deref(), Some("eternal")); + assert_eq!( + by("works_at").temporal, + None, + "没写就是没写，落库再按 state" + ); + // 三个值以外的不猜：留空并按谓词报告出来，只有 `met` 那一条，预览页能说清它没进来 + assert_eq!(by("met").temporal, None); + assert_eq!(p.unprojected.get("urn:utopia:ns:temporal"), Some(&1)); + } } #[cfg(test)] diff --git a/crates/utopia-server/src/api/token_setup_tests.rs b/crates/utopia-server/src/api/token_setup_tests.rs index bf794868b..05b16b4ba 100644 --- a/crates/utopia-server/src/api/token_setup_tests.rs +++ b/crates/utopia-server/src/api/token_setup_tests.rs @@ -470,3 +470,97 @@ async fn importing_an_ontology_queues_alignment_behind_the_index_refresh() -> an f.cleanup().await?; result } + +/// 导入读 `utopia:temporal`（0031）。导出端一直写这一条，导入端从前不读：一份本体 +/// 说「这条关系是一个时刻」，导进来照样 `state`，#1242 的「非 state 的关系不经模型 +/// 就能绑」对导入的本体永远不成立。新建的照本体建；重导入时本体说了的改过来， +/// 没说的不动——与 domain/range 同一个道理。真改了推 `updated_at`，与页面上改一样 +#[tokio::test] +async fn an_imported_property_keeps_the_temporal_it_declares() -> anyhow::Result<()> { + let Some(f) = Fixture::new().await? else { + return Ok(()); + }; + let result = async { + let kb = f.seeded_kb("temporal").await?; + let token = f.token(f.admin, "write", None).await; + const PREFIXES: &str = "@prefix owl: .\n\ + @prefix rdfs: .\n\ + @prefix utopia: .\n\ + @prefix ex: .\n"; + let works_at = || { + let pool = f.pool.clone(); + async move { + sqlx::query_as::<_, (String, chrono::DateTime)>( + "SELECT temporal, updated_at FROM relation_types + WHERE kb_id = $1 AND iri = 'http://example.org/worksAt'", + ) + .bind(kb) + .fetch_one(&pool) + .await + } + }; + + // 第一版没有注解：worksAt 建成 state + let first = + format!("{PREFIXES}ex:worksAt a owl:ObjectProperty ; rdfs:label \"works at\" .\n"); + let (status, body) = f.import_turtle(kb, &token, &first).await?; + anyhow::ensure!(status == StatusCode::OK, "{status} {body}"); + let (temporal, created) = works_at().await?; + anyhow::ensure!(temporal == "state"); + + // 第二版：worksAt 加了注解，attended 是带注解新建的 + let second = format!( + "{PREFIXES}ex:worksAt a owl:ObjectProperty ; rdfs:label \"works at\" ; \ + utopia:temporal \"eternal\" .\n\ + ex:attended a owl:ObjectProperty ; rdfs:label \"attended\" ; \ + utopia:temporal \"event\" .\n" + ); + let (status, body) = f.import_turtle(kb, &token, &second).await?; + anyhow::ensure!(status == StatusCode::OK, "{status} {body}"); + let (temporal, changed) = works_at().await?; + anyhow::ensure!( + temporal == "eternal", + "a re-import writes the temporal the file now declares" + ); + anyhow::ensure!( + changed > created, + "a temporal changed by import moves updated_at" + ); + let attended: String = sqlx::query_scalar( + "SELECT temporal FROM relation_types + WHERE kb_id = $1 AND iri = 'http://example.org/attended'", + ) + .bind(kb) + .fetch_one(&f.pool) + .await?; + anyhow::ensure!( + attended == "event", + "a fresh import creates the relation with the temporal it declares" + ); + + // 第三版把注解去掉：没说就不动，人在页面上改过的也不会被重导入抹掉 + let (status, body) = f.import_turtle(kb, &token, &first).await?; + anyhow::ensure!(status == StatusCode::OK, "{status} {body}"); + anyhow::ensure!( + works_at().await? == ("eternal".to_string(), changed), + "an unannotated re-import leaves temporal and updated_at alone" + ); + + // 第四版说回 state：导出端写 state，所以一份导出重导入能把关系带回 state + let back = format!( + "{PREFIXES}ex:worksAt a owl:ObjectProperty ; rdfs:label \"works at\" ; \ + utopia:temporal \"state\" .\n" + ); + let (status, body) = f.import_turtle(kb, &token, &back).await?; + anyhow::ensure!(status == StatusCode::OK, "{status} {body}"); + let (temporal, returned) = works_at().await?; + anyhow::ensure!( + temporal == "state" && returned > changed, + "a re-import declaring state returns the relation to state" + ); + anyhow::Ok(()) + } + .await; + f.cleanup().await?; + result +} diff --git a/crates/utopia-server/src/owl_import.rs b/crates/utopia-server/src/owl_import.rs index f5443ba0f..a714e7377 100644 --- a/crates/utopia-server/src/owl_import.rs +++ b/crates/utopia-server/src/owl_import.rs @@ -610,6 +610,7 @@ pub async fn apply( &p.description, &domains, &ranges, + p.temporal.as_deref(), ) .await? { @@ -631,6 +632,8 @@ pub async fn apply( symmetric: p.symmetric, asymmetric: p.asymmetric, irreflexive: p.irreflexive, + // 本体用 `utopia:temporal` 说了的照它说的（0031）；没说的是 state + temporal: p.temporal.as_deref().unwrap_or("state").to_string(), }); pending_links.push((p.key.clone(), domains, ranges)); } @@ -719,6 +722,8 @@ pub async fn apply( symmetric: false, asymmetric: false, irreflexive: false, + // 属性建成 state，页面也不给别的选（0031 的「未决」里那一条） + temporal: "state".to_string(), }); pending_attr_domains.push((p.key.clone(), domain_ids)); } diff --git a/crates/utopia-server/src/rdf.rs b/crates/utopia-server/src/rdf.rs index 740b580a1..f921d4da9 100644 --- a/crates/utopia-server/src/rdf.rs +++ b/crates/utopia-server/src/rdf.rs @@ -398,10 +398,8 @@ pub fn emit_relation( } // 时间语义也照抄（0031）：一个 event 谓词的事实两端是同一刻，一个 eternal 谓词的 // 事实没有日期——读的人不看这一条，会把前者读成一天的状态、后者读成从不知何时起。 - // 状态是默认，不写 - if r.temporal != "state" { - sink.l(&iri, &utopia("temporal"), &text(r.temporal.clone()))?; - } + // `state` 也写：导入把「没写」读成「不动」，不写它，改回 state 的关系重导入后还是 event + sink.l(&iri, &utopia("temporal"), &text(r.temporal.clone()))?; for d in &r.domains { if let Some(c) = vocab.class(*d) { let c = c.clone(); @@ -1600,6 +1598,18 @@ mod tests { )); } + /// 改回 state 的关系重导入也得是 state：导入把没写的 `utopia:temporal` 读成「不动」 + #[test] + fn a_state_relation_exports_its_temporal() { + let quads = export(Format::Turtle, |_, _, _| {}); + assert!(has( + &quads, + &format!("<{WORKS_FOR}>"), + "urn:utopia:ns:temporal", + "\"state\"" + )); + } + #[test] fn a_live_fact_is_both_a_statement_and_a_triple() { let quads = export(Format::Turtle, |sink, names, vocab| { diff --git a/crates/utopia-store/src/ontology.rs b/crates/utopia-store/src/ontology.rs index 32ff379a1..6ffe76820 100644 --- a/crates/utopia-store/src/ontology.rs +++ b/crates/utopia-store/src/ontology.rs @@ -273,6 +273,16 @@ pub fn lower_camel(label: &str) -> String { out } +/// `temporal` 只有三个值（0031）：单条建、单条改、批量建、重导入更新都走这一句 +fn validate_temporal(temporal: &str) -> AppResult<()> { + if !matches!(temporal, "state" | "event" | "eternal") { + return Err(AppError::Validation( + "temporal must be state / event / eternal".into(), + )); + } + Ok(()) +} + fn validate_key(key: &str) -> AppResult<()> { let ok = !key.is_empty() && key.len() <= 40 @@ -608,11 +618,7 @@ pub async fn create_relation_type<'a>( unit: Option<&str>, ) -> AppResult { validate_key(key)?; - if !matches!(temporal, "state" | "event" | "eternal") { - return Err(AppError::Validation( - "temporal must be state / event / eternal".into(), - )); - } + validate_temporal(temporal)?; validate_attribute_fields(kind, domains, datatype)?; let mut tx = pool.begin().await?; let label = &lower_camel(label); @@ -826,11 +832,7 @@ pub async fn update_relation_type( } // 改名也归一：不然界面上改一次就能把小驼峰改回 "access to" let label = &lower_camel(label); - if !matches!(temporal, "state" | "event" | "eternal") { - return Err(AppError::Validation( - "temporal must be state / event / eternal".into(), - )); - } + validate_temporal(temporal)?; if datatype.is_some() && !matches!(datatype, Some("text" | "number" | "date" | "bool")) { return Err(AppError::Validation( "datatype must be text / number / date / bool".into(), @@ -1133,6 +1135,11 @@ pub async fn create_relation_with_iri( /// /// **key 不动**（它可能已被事实引用），**空描述不覆盖**（人写过的比上游的准）， /// **domain/range 整体重写**——它们是上游声明的结构，不是人调过的措辞。 +/// **`temporal` 只在本体说了才写**（`Some`）：没写 `utopia:temporal` 的本体不动它， +/// 人在页面上改过的就留着；写了的照本体，与 domain/range 同一个道理。 +/// 真改了就推 `updated_at`，与页面上改一样：0053 的短语绑定依据读它， +/// state 与非 state 决定短语怎么绑，旧值下做的决定要能被重审。 +#[allow(clippy::too_many_arguments)] pub async fn update_relation_from_import( pool: &PgPool, kb_id: Uuid, @@ -1141,11 +1148,17 @@ pub async fn update_relation_from_import( description: &str, domains: &[Uuid], ranges: &[Uuid], + temporal: Option<&str>, ) -> AppResult { + if let Some(t) = temporal { + validate_temporal(t)?; + } let row: Option<(Uuid,)> = sqlx::query_as( "UPDATE relation_types SET label = $3, - description = CASE WHEN $4 = '' THEN description ELSE $4 END + description = CASE WHEN $4 = '' THEN description ELSE $4 END, + temporal = COALESCE($5::text, temporal), + updated_at = CASE WHEN $5::text <> temporal THEN now() ELSE updated_at END WHERE kb_id = $1 AND iri = $2 RETURNING id", ) @@ -1153,6 +1166,7 @@ pub async fn update_relation_from_import( .bind(iri) .bind(label) .bind(description) + .bind(temporal) .fetch_optional(pool) .await?; let Some((id,)) = row else { @@ -1683,13 +1697,16 @@ pub struct BulkRelation { pub symmetric: bool, pub asymmetric: bool, pub irreflexive: bool, + /// `state` / `event` / `eternal`（0031）。OWL 自己没有这个概念，标准词表里的对象 + /// 属性全是 `state`；只有本体用 `utopia:temporal` 说了的才不是——导出端写的那一条 + pub temporal: String, } /// 一次插完一批关系或属性，返回 key → id。语义同 [`create_entity_types_bulk`]。 /// /// `kind` 决定走关系通道还是属性通道；`datatype` 只对属性有意义，关系传 `None`。 -/// `temporal` 固定 `state`——OWL 里没有对应概念，猜 event 或 eternal 都更差 -/// （与单条版的判断一致）。 +/// `temporal` 照每一行写：没有 `utopia:temporal` 注解的本体一律 `state` +///（与单条版的默认一致），写了的按它说的。 pub async fn create_relation_types_bulk( pool: &PgPool, kb_id: Uuid, @@ -1700,6 +1717,7 @@ pub async fn create_relation_types_bulk( } for r in rows { validate_key(&r.key)?; + validate_temporal(&r.temporal)?; } let keys: Vec<&str> = rows.iter().map(|r| r.key.as_str()).collect(); // 导入来的词表自己就可能不一致（unece.org 里 `brandName` 与 @@ -1716,16 +1734,18 @@ pub async fn create_relation_types_bulk( let syms: Vec = rows.iter().map(|r| r.symmetric).collect(); let asyms: Vec = rows.iter().map(|r| r.asymmetric).collect(); let irrefs: Vec = rows.iter().map(|r| r.irreflexive).collect(); + let temporals: Vec<&str> = rows.iter().map(|r| r.temporal.as_str()).collect(); let out: Vec<(Uuid, String)> = sqlx::query_as( "INSERT INTO relation_types (id, kb_id, key, label, temporal, functional, inverse_functional, description, kind, datatype, iri, is_transitive, is_symmetric, is_asymmetric, is_irreflexive) - SELECT gen_random_uuid(), $1, k, l, 'state', fu, iv, d, kind, dt, i, + SELECT gen_random_uuid(), $1, k, l, tm, fu, iv, d, kind, dt, i, tr, sy, asym, irr FROM UNNEST($2::text[], $3::text[], $4::text[], $5::text[], $6::text[], $7::text[], - $8::bool[], $9::bool[], $10::bool[], $11::bool[], $12::bool[], $13::bool[]) - AS t(k, l, d, i, kind, dt, fu, iv, tr, sy, asym, irr) + $8::bool[], $9::bool[], $10::bool[], $11::bool[], $12::bool[], $13::bool[], + $14::text[]) + AS t(k, l, d, i, kind, dt, fu, iv, tr, sy, asym, irr, tm) ON CONFLICT (kb_id, key) DO NOTHING RETURNING id, key", ) @@ -1742,6 +1762,7 @@ pub async fn create_relation_types_bulk( .bind(&syms) .bind(&asyms) .bind(&irrefs) + .bind(&temporals) .fetch_all(pool) .await?; Ok(out.into_iter().map(|(id, k)| (k, id)).collect()) diff --git a/crates/utopia-store/tests/store/axioms_reach_the_database.rs b/crates/utopia-store/tests/store/axioms_reach_the_database.rs index 8fffd4a8f..15046544e 100644 --- a/crates/utopia-store/tests/store/axioms_reach_the_database.rs +++ b/crates/utopia-store/tests/store/axioms_reach_the_database.rs @@ -59,6 +59,7 @@ async fn every_axiom_survives_the_bulk_insert() -> anyhow::Result<()> { symmetric: s, asymmetric: a, irreflexive: r, + temporal: "state".into(), } }; // 一条全真、一条全假：全假那条守的是"没声明的不该被写成真" diff --git a/docs/decisions/0031-an-event-holds-at-the-moment-it-names.md b/docs/decisions/0031-an-event-holds-at-the-moment-it-names.md index 30e823fba..a23987066 100644 --- a/docs/decisions/0031-an-event-holds-at-the-moment-it-names.md +++ b/docs/decisions/0031-an-event-holds-at-the-moment-it-names.md @@ -27,6 +27,8 @@ The rule sits in `insert_fact_inner` and `correct_interval`, not in the writers. 2026-09-27: under a property declared as a state, the same rule now refuses a row whose two ends are equal, and a single date bound to a state is read by what its binding says it marks. See [0053](0053-a-phrase-decision-records-the-inputs-it-considered.md)'s revision of that date (#966). +2026-10-18: the export writes `utopia:temporal` on every property, `state` included, and the import reads it on an object property: a new relation takes the value the file declares, a re-import changes an existing one to it and moves its `updated_at` as an edit on the page does, and a file that says nothing leaves a relation alone, so a value a person set survives an unannotated re-import while re-importing an export is exact. A value other than the three is reported under the predicate in the import preview, and the relation stays as it was (#1291). + Both ends carry the value because that is the one shape the row can explain on its own. A reader that does not consult the predicate — an old client, an export consumer, a hand-written query — sees a fact that held on one day. It cannot see *holds from that day on*, which is what an open end would have told it. The representation fails safe. An undated observation of an event merges into an existing dated one, the mirror of the refinement path that already lets a dated observation supersede a bare row: "acquired in March" followed by "acquired" is one acquisition, not two. diff --git a/docs/design/ontology.md b/docs/design/ontology.md index 6f17dac79..340ea8be1 100644 --- a/docs/design/ontology.md +++ b/docs/design/ontology.md @@ -238,7 +238,9 @@ the prompt, a description is read by people and by the aligner. per signature costs distinct phrasings, not documents, and a changed ontology recomputes only the affected signatures instead of re-extracting [0044]. - **Standard vocabularies reify events as classes**, so every pack object property is a state; a - person changes `temporal` on the page and reads pick it up at once [0031]. + person changes `temporal` on the page and reads pick it up at once, and a vocabulary that + annotates a property with `utopia:temporal` (what the export writes) is imported with that value, + a re-import updating an existing relation's [0031]. ## Proposed and not built From fea5171ffc1d7e9608c0cace6e52cdd9443aff1e Mon Sep 17 00:00:00 2001 From: Maya Kid Date: Mon, 19 Oct 2026 05:20:58 +1100 Subject: [PATCH 06/19] An end moved to the end of its period says so on the row (#1295) * An end moved to the end of its period says so on the row Signed-off-by: Maya Kid * Name the pull request and the close-by-date issue in the 0053 revision Signed-off-by: Maya Kid * Leave the backfill to its own pull request and keep the mark's rule in one helper Signed-off-by: Maya Kid --------- Signed-off-by: Maya Kid --- crates/utopia-cli/src/main.rs | 2 +- crates/utopia-core/src/models.rs | 3 + .../src/api/chat_graph_citation_tests.rs | 2 + crates/utopia-server/src/api/graph_routes.rs | 84 +++++- .../src/api/graph_routes_time_tests.rs | 168 +++++++++++ crates/utopia-server/src/api/tools.rs | 1 + crates/utopia-server/src/api/tools_graph.rs | 1 + crates/utopia-store/src/errata.rs | 5 +- crates/utopia-store/src/graph.rs | 117 ++++++-- crates/utopia-store/src/materialize.rs | 7 +- crates/utopia-store/src/pending.rs | 1 + crates/utopia-store/src/temporal.rs | 121 ++++++-- .../a_corrected_row_says_which_end_changed.rs | 1 + .../tests/store/a_fact_awaits_a_nod.rs | 1 + .../tests/store/a_moment_marks_its_state.rs | 274 +++++++++++++++++- .../tests/store/a_path_joins_two_entities.rs | 1 + ...line_is_recomputed_from_the_rows_it_has.rs | 2 + .../store/a_wrong_time_can_be_corrected.rs | 1 + .../store/an_end_date_closes_the_open_span.rs | 3 + .../an_event_holds_at_the_moment_it_names.rs | 3 + .../an_unknown_date_is_not_an_open_one.rs | 1 + .../the_engine_redraws_only_what_it_drew.rs | 1 + ...1-an-event-holds-at-the-moment-it-names.md | 2 + ...cision-records-the-inputs-it-considered.md | 121 +++++++- docs/design/ledger.md | 2 +- docs/design/time.md | 7 +- ...114_an_end_that_names_a_period_says_so.sql | 19 ++ 27 files changed, 889 insertions(+), 62 deletions(-) create mode 100644 crates/utopia-server/src/api/graph_routes_time_tests.rs create mode 100644 migrations/0114_an_end_that_names_a_period_says_so.sql diff --git a/crates/utopia-cli/src/main.rs b/crates/utopia-cli/src/main.rs index 7a9ba5011..c88fc1b7c 100644 --- a/crates/utopia-cli/src/main.rs +++ b/crates/utopia-cli/src/main.rs @@ -82,7 +82,7 @@ struct ManifestDataDir { /// not a side effect of a code change. // 是迁移文件的**个数**，不是最大的编号（守卫 `schema_version_policy_compares_against_current` // 按个数比）：编号有空缺时两者不同——0071 由一个开放 PR 占着，0072 先落，个数是 71 -const CURRENT_SCHEMA_VERSION: u32 = 101; +const CURRENT_SCHEMA_VERSION: u32 = 102; fn main() -> anyhow::Result<()> { dotenvy::dotenv().ok(); diff --git a/crates/utopia-core/src/models.rs b/crates/utopia-core/src/models.rs index 360bef0c5..81ba60c41 100644 --- a/crates/utopia-core/src/models.rs +++ b/crates/utopia-core/src/models.rs @@ -830,6 +830,9 @@ pub struct EntityFact { /// 终点是时间线推出来的（`facts.end_derived`）：后一条事实开始时引擎把它关上，原文没说 /// 这一天（#970） pub end_derived: bool, + /// `valid_to` 仍是存的边界；true 说原文讲的是在它之前结束的那个时段（`facts.end_after_period`， + /// #994）：「2024 年离开」存 2025-01-01，原文说的是 2024 + pub end_after_period: bool, /// 区间是人改过的（#970）：`corrected_ends` 有值。与 `corrected` 不同，那一位是任何一次 /// 改写：自动接续、审阅裁决、人手改 pub time_corrected: bool, diff --git a/crates/utopia-server/src/api/chat_graph_citation_tests.rs b/crates/utopia-server/src/api/chat_graph_citation_tests.rs index f5c23b284..8b09de749 100644 --- a/crates/utopia-server/src/api/chat_graph_citation_tests.rs +++ b/crates/utopia-server/src/api/chat_graph_citation_tests.rs @@ -696,6 +696,7 @@ async fn a_fact_line_says_when_a_date_is_not_the_passages() -> anyhow::Result<() to_precision: Some("day"), attested_at: None, from_grade: None, + end_after_period: false, }, ) .await? @@ -820,6 +821,7 @@ async fn a_fact_line_says_when_a_date_is_not_the_passages() -> anyhow::Result<() to_precision: Some("day"), attested_at: None, from_grade: None, + end_after_period: false, }, ) .await? diff --git a/crates/utopia-server/src/api/graph_routes.rs b/crates/utopia-server/src/api/graph_routes.rs index eed176b99..728c51f07 100644 --- a/crates/utopia-server/src/api/graph_routes.rs +++ b/crates/utopia-server/src/api/graph_routes.rs @@ -10,6 +10,10 @@ use crate::auth::AuthUser; use crate::error::ApiResult; use crate::state::AppState; +#[cfg(test)] +#[path = "graph_routes_time_tests.rs"] +mod time_tests; + pub(super) async fn require_kb( state: &AppState, user: &utopia_core::models::User, @@ -273,7 +277,8 @@ pub async fn update_entity( /// 一条事实的新有效区间。**整体替换，不是部分更新**：区间的两端互相定义， /// 「把结束端清空」与「这次不动结束端」必须能分辨，而缺字段与 null 在 JSON 里 -/// 分不开。界面提交的是表单里的四个值，不是差异。 +/// 分不开。界面提交的是表单里的四个值，不是差异。`end_after_period` 是例外：缺省即 +/// 保留，见那一格。 #[derive(Deserialize)] pub struct FactTimePatch { #[serde(default)] @@ -289,6 +294,13 @@ pub struct FactTimePatch { /// 这里升格成字段 #[serde(default)] pub note: Option, + /// 终点是不是原文所说时段之后的那一刻（`facts.end_after_period`，#994）。**唯一一个 + /// 缺省即保留的字段**：读出来什么就发回什么的客户端（不认识这个字段的也一样）不该 + /// 改掉它。不给时，终点（截到精度之后的值与精度）没变、起点还在，就留着存的标记；清掉 + /// 起点的保留时刻、丢掉标记（没有起点的行第一刀不标）；别的终点照给的收下。路由从不 + /// 换算：存的、收的都是边界本身，按时段显示与换算是界面的事 + #[serde(default)] + pub end_after_period: Option, } // 梯子只有一张（0024）：与数据库的 CHECK、抽取端、显示端同一列 @@ -301,9 +313,21 @@ struct FactInterval { valid_from_precision: Option, valid_to: Option>, valid_to_precision: Option, + end_after_period: bool, + /// 谓词声明成状态：标记只能落在这样的行上 + declared_state: bool, +} + +/// 显式要标记，却不是一个能带它的终点 +fn bad_end_after_period() -> AppError { + AppError::invalid( + "bad_end_after_period", + "An end that names a period needs a state property, a start and an end date.", + ) } -/// 复刻 `facts` 表的两条 CHECK，外加一条数据库没有的：起点不能晚于终点。 +/// 复刻 `facts` 表的 CHECK：两端的精度，标记要两端带得了它（`period_mark_fits`，属性是不是 +/// 状态读到行之后再查）；外加一条数据库没有的：截到各自精度之后，起点不能晚于终点。 /// /// 在这里挡住是为了让错误说人话。交给数据库挡会得到一句约束名，而这三态 /// （仍在持续 / 结束了不知哪天 / 某日结束）本来就是账本里最容易记混的地方。 @@ -330,8 +354,19 @@ fn check_interval(p: &FactTimePatch) -> Result<(), AppError> { )) } } + // 显式要标记：两端得带得了它。属性是不是状态要读到行才知道，这里先当它是，只看两端 + if p.end_after_period == Some(true) + && !utopia_store::graph::period_mark_fits(true, p.valid_from, p.valid_to, false) + { + return Err(bad_end_after_period()); + } + // 比截到各自精度之后的两端：存下的就是截过的。比原值时，小时精度的 08:30 终点与分钟精度 + // 的 08:10 起点放得过去，截完成了 08:10 → 08:00，存下一段倒着的区间 + let truncate = utopia_store::graph::truncate_to; if let (Some(from), Some(to)) = (p.valid_from, p.valid_to) { - if from > to { + if truncate(from, p.valid_from_precision.as_deref()) + > truncate(to, p.valid_to_precision.as_deref()) + { return Err(AppError::invalid( "interval_inverted", "The start of the interval is after its end.", @@ -360,9 +395,12 @@ pub async fn update_fact_time( // 归属与状态校验：本 KB 的、未作废的事实才可改。派生事实不在此列—— // 它的区间来自前提，改它等于改一个算出来的值，下一轮推理就会覆盖 let before: Option = sqlx::query_as( - "SELECT valid_from, valid_from_precision, valid_to, valid_to_precision - FROM facts - WHERE id = $1 AND kb_id = $2 AND invalidated_at IS NULL AND derived_by_rule IS NULL", + "SELECT f.valid_from, f.valid_from_precision, f.valid_to, f.valid_to_precision, + f.end_after_period, COALESCE(r.temporal = 'state', false) AS declared_state + FROM facts f + LEFT JOIN relation_types r ON r.id = f.predicate_id + WHERE f.id = $1 AND f.kb_id = $2 AND f.invalidated_at IS NULL + AND f.derived_by_rule IS NULL", ) .bind(fact_id) .bind(kb_id) @@ -372,12 +410,39 @@ pub async fn update_fact_time( let Some(before) = before else { return Err(AppError::NotFound.into()); }; + // 存下的是截到精度之后的值：比较、判标记都按它 + let truncate = utopia_store::graph::truncate_to; + let from = req + .valid_from + .map(|t| truncate(t, req.valid_from_precision.as_deref())); + let to = req + .valid_to + .map(|t| truncate(t, req.valid_to_precision.as_deref())); + // 读到了行，属性是不是状态也知道了：显式要的标记按整条规矩再判一遍 + if req.end_after_period == Some(true) + && !utopia_store::graph::period_mark_fits(before.declared_state, from, to, false) + { + return Err(bad_end_after_period().into()); + } + // 不给标记：发回存的终点（时刻与精度）、这次的两端还带得了它，就是没动它，留着；清掉起点 + // 的带不了，丢掉（时刻照旧）。属性这里不再判：存的标记是按当时的属性写下的，原样发回的 + // 请求不改写这一行 + let end_after_period = match req.end_after_period { + Some(mark) => mark, + None => { + before.end_after_period + && to == before.valid_to + && req.valid_to_precision == before.valid_to_precision + && utopia_store::graph::period_mark_fits(true, from, to, false) + } + }; // 一字未改就不写一条修正进账本：History 上多一行「改过」而区间没动， // 读者会去找那个不存在的差异 - if before.valid_from == req.valid_from + if before.valid_from == from && before.valid_from_precision == req.valid_from_precision - && before.valid_to == req.valid_to + && before.valid_to == to && before.valid_to_precision == req.valid_to_precision + && before.end_after_period == end_after_period { return Ok(Json(json!({ "ok": true, "unchanged": true }))); } @@ -393,6 +458,7 @@ pub async fn update_fact_time( // 修正行从被替代的行继承锚点（correct_interval 的 SQL 里）：改区间是重述 // 同一份证据，不是更新的证据 attested_at: None, + end_after_period, }; let Some(corrected) = utopia_store::temporal::correct_interval(&state.pool, fact_id, validity).await? @@ -417,12 +483,14 @@ pub async fn update_fact_time( "valid_from_precision": before.valid_from_precision, "valid_to": before.valid_to.map(|t| t.to_rfc3339()), "valid_to_precision": before.valid_to_precision, + "end_after_period": before.end_after_period, }); d["to"] = json!({ "valid_from": req.valid_from.map(|t| t.to_rfc3339()), "valid_from_precision": req.valid_from_precision, "valid_to": req.valid_to.map(|t| t.to_rfc3339()), "valid_to_precision": req.valid_to_precision, + "end_after_period": end_after_period, }); if let Some(note) = req.note.as_deref().map(str::trim).filter(|s| !s.is_empty()) { d["note"] = json!(note); diff --git a/crates/utopia-server/src/api/graph_routes_time_tests.rs b/crates/utopia-server/src/api/graph_routes_time_tests.rs new file mode 100644 index 000000000..3ddaa6a33 --- /dev/null +++ b/crates/utopia-server/src/api/graph_routes_time_tests.rs @@ -0,0 +1,168 @@ +//! 改区间的路由收发的是存的边界（#994）：`end_after_period` 是唯一一个缺省即保留的字段。 +//! +//! 实体详情读得到标记；不带它的请求，终点（时刻与精度）没动、起点还在就留着，改了终点或清掉 +//! 起点就丢掉（时刻照旧）；显式的值照收，只改标记也算改了终点；显式 true 得是状态属性、有起点、 +//! 终点有日期；倒置按截到精度之后的两端判。每一步的结果收进一张表一起比，dev 上哪几步不同一次 +//! 看全。没有 `UTOPIA_DATABASE_URL` 时跳过。 +use axum::body::{to_bytes, Body}; +use axum::http::{Request, StatusCode}; +use axum::Router; +use chrono::{DateTime, Utc}; +use serde_json::{json, Value}; +use std::sync::Arc; +use tower::ServiceExt; +use uuid::Uuid; + +const MARCH: &str = "2024-03-01T00:00:00Z"; +const FEB: &str = "2024-02-01T00:00:00Z"; +const END: &str = "2025-01-01T00:00:00Z"; +const LATER: &str = "2026-01-01T00:00:00Z"; + +async fn send( + app: &Router, + token: &str, + method: &str, + uri: String, + body: &Value, +) -> anyhow::Result<(StatusCode, Value)> { + let response = app + .clone() + .oneshot( + Request::builder() + .method(method) + .uri(uri) + .header("content-type", "application/json") + .header("authorization", format!("Bearer {token}")) + .body(Body::from(body.to_string()))?, + ) + .await?; + let status = response.status(); + let bytes = to_bytes(response.into_body(), usize::MAX).await?; + Ok((status, serde_json::from_slice(&bytes)?)) +} + +/// 一个请求体：起点日精度、终点年精度，空串是没有；标记给了才带 +fn span(from: &str, to: &str, mark: Option) -> Value { + let from = Some(from).filter(|t| !t.is_empty()); + let to = Some(to).filter(|t| !t.is_empty()); + let mut body = json!({ + "valid_from": from, "valid_from_precision": from.map(|_| "day"), + "valid_to": to, "valid_to_precision": to.map(|_| "year"), + }); + if let Some(mark) = mark { + body["end_after_period"] = json!(mark); + } + body +} + +#[tokio::test] +async fn a_correction_keeps_the_mark_unless_the_request_changes_it() -> anyhow::Result<()> { + let Some(url) = utopia_store::test_db::url() else { + return Ok(()); + }; + let pool = sqlx::PgPool::connect(&url).await?; + utopia_store::db::migrate(&pool).await?; + let [org, ws, kb, editor, works_for, lin, meridian, marked, loose] = + std::array::from_fn(|_| Uuid::now_v7()); + // Only locally generated UUIDs and the constants above are interpolated into fixture SQL. + sqlx::raw_sql(&format!( + "INSERT INTO organizations(id,name) VALUES ('{org}','fact-time-route'); + INSERT INTO workspaces(id,org_id,name) VALUES ('{ws}','{org}','fact-time-route'); + INSERT INTO users(id,org_id,email,display_name,password_hash) + VALUES ('{editor}','{org}','{editor}@time.test','editor','unused'); + INSERT INTO knowledge_bases(id,workspace_id,name) VALUES ('{kb}','{ws}','times'); + INSERT INTO kb_members(kb_id,user_id,role) VALUES ('{kb}','{editor}','editor'); + INSERT INTO relation_types(id,kb_id,key,label,temporal) + VALUES ('{works_for}','{kb}','works_for','works for','state'); + INSERT INTO entities(id,kb_id,canonical_name) + VALUES ('{lin}','{kb}','Lin Zhao'), ('{meridian}','{kb}','Meridian Systems'); + -- 「left in 2024」关上的一行，记着说的是 2024；同样的区间、没有谓词的一行 + INSERT INTO facts(id,kb_id,subject_id,predicate_id,object_id,layer,confidence, + valid_from,valid_from_precision,valid_to,valid_to_precision, + end_after_period) VALUES + ('{marked}','{kb}','{lin}','{works_for}','{meridian}','typed',0.9, + '{MARCH}','day','{END}','year',TRUE), + ('{loose}','{kb}','{lin}',NULL,'{meridian}','typed',0.9, + '{MARCH}','day','{END}','year',FALSE);" + )) + .execute(&pool) + .await?; + let result = async { + let dir = tempfile::tempdir()?; + let config = utopia_core::config::AppConfig { + data_dir: dir.path().to_string_lossy().into_owned(), + ..Default::default() + }; + let search = Arc::new(utopia_search::SearchIndex::open(&dir.path().join("search"))?); + let state = crate::state::AppState::new(pool.clone(), &config, search, "test-only".into()); + let token = crate::auth::issue_token(&state, editor)?; + let app = crate::api::router(state, &config); + + let detail = format!("/api/v1/kbs/{kb}/entities/{lin}"); + let (_, detail) = send(&app, &token, "GET", detail, &Value::Null).await?; + let mut read = detail["facts"].as_array().into_iter().flatten(); + let read = read.find(|x| x["id"] == json!(marked)); + let mut seen = vec![("entity detail", json!(read.map(|x| &x["end_after_period"])))]; + let mut wanted = vec![("entity detail", json!(true))]; + // 每一步：改哪一行（None 是上一步写下的那行）、请求体、要的结果——写下的那一行（标记、 + // 人改过哪一端、终点），或拒绝的代码。不带标记的：只改起点留着，改了终点的时刻或精度、 + // 清掉起点都丢掉；显式的照收，只改标记把终点那一端记成人改的；显式 true 缺起点、缺终点、 + // 不是状态的都不收 + let bad = json!([422, "bad_end_after_period"]); + let steps = [ + ("start only", None, span(FEB, END, None), json!([true, "start", END])), + ("mark false", None, span(FEB, END, Some(false)), json!([false, "both", END])), + ("mark true", None, span(FEB, END, Some(true)), json!([true, "both", END])), + ("new end", None, span(FEB, LATER, None), json!([false, "both", LATER])), + ("mark true again", None, span(FEB, END, Some(true)), json!([true, "both", END])), + ( + "precision only", + None, + json!({"valid_from": FEB, "valid_from_precision": "day", + "valid_to": END, "valid_to_precision": "month"}), + json!([false, "both", END]), + ), + ("marked once more", None, span(FEB, END, Some(true)), json!([true, "both", END])), + ("start cleared", None, span("", END, None), json!([false, "both", END])), + ("true, no start", None, span("", END, Some(true)), bad.clone()), + ("true, no end", None, span(FEB, "", Some(true)), bad.clone()), + ("true, no state", Some(loose), span(MARCH, END, Some(true)), bad), + // 小时精度的 08:30 是 08:00，早于分钟精度的起点 08:10 + ( + "inverted once truncated", + Some(loose), + json!({"valid_from": "2026-06-01T08:10:00Z", "valid_from_precision": "minute", + "valid_to": "2026-06-01T08:30:00Z", "valid_to_precision": "hour"}), + json!([422, "interval_inverted"]), + ), + ]; + let mut fact = marked; + for (label, target, body, want) in steps { + let uri = format!("/api/v1/kbs/{kb}/facts/{}", target.unwrap_or(fact)); + let (status, reply) = send(&app, &token, "PATCH", uri, &body).await?; + let got = match reply["fact_id"].as_str() { + Some(id) => { + fact = id.parse()?; + let row: (bool, Option, Option>) = sqlx::query_as( + "SELECT end_after_period, corrected_ends, valid_to FROM facts WHERE id = $1", + ) + .bind(fact) + .fetch_one(&pool) + .await?; + json!(row) + } + None => json!([status.as_u16(), reply["code"]]), + }; + seen.push((label, got)); + wanted.push((label, want)); + } + assert_eq!(seen, wanted); + anyhow::Ok(()) + } + .await; + sqlx::query("DELETE FROM organizations WHERE id = $1") + .bind(org) + .execute(&pool) + .await?; + result +} diff --git a/crates/utopia-server/src/api/tools.rs b/crates/utopia-server/src/api/tools.rs index 149f68d0a..ec5c6b4e0 100644 --- a/crates/utopia-server/src/api/tools.rs +++ b/crates/utopia-server/src/api/tools.rs @@ -1532,6 +1532,7 @@ mod tests { stale: false, corrected: false, end_derived: false, + end_after_period: false, time_corrected: false, corrected_ends: None, closed_by_id: None, diff --git a/crates/utopia-server/src/api/tools_graph.rs b/crates/utopia-server/src/api/tools_graph.rs index 0a1beb834..70fa110fa 100644 --- a/crates/utopia-server/src/api/tools_graph.rs +++ b/crates/utopia-server/src/api/tools_graph.rs @@ -1500,6 +1500,7 @@ mod tests { stale: false, corrected: false, end_derived: false, + end_after_period: false, time_corrected: false, corrected_ends: None, closed_by_id: None, diff --git a/crates/utopia-store/src/errata.rs b/crates/utopia-store/src/errata.rs index c8ab95af9..e98c92d5e 100644 --- a/crates/utopia-store/src/errata.rs +++ b/crates/utopia-store/src/errata.rs @@ -617,7 +617,7 @@ async fn apply( let fact = fact_id.ok_or_else(|| AppError::Validation("no fact to retract".into()))?; validity_row = sqlx::query_as( "SELECT valid_from, valid_from_precision, valid_from_grade, valid_to, valid_to_precision, - attested_from FROM facts WHERE id = $1", + end_after_period, attested_from FROM facts WHERE id = $1", ) .bind(fact) .fetch_optional(pool) @@ -647,6 +647,8 @@ async fn apply( to: v.valid_to, to_precision: v.valid_to_precision.as_deref(), attested_at: v.attested_from.or(doc_time), + // 新行接旧行的终点，也接它说的是哪个时段（#994）；改到事件、恒常属性上的，写入口丢掉 + end_after_period: v.end_after_period, }; let object = match (target.object_id, &target.object_value) { (Some(o), _) => FactObject::Entity(o), @@ -719,6 +721,7 @@ struct ValidityRow { valid_from_grade: Option, valid_to: Option>, valid_to_precision: Option, + end_after_period: bool, attested_from: Option>, } diff --git a/crates/utopia-store/src/graph.rs b/crates/utopia-store/src/graph.rs index 1ef3c1a87..ad2c802c7 100644 --- a/crates/utopia-store/src/graph.rs +++ b/crates/utopia-store/src/graph.rs @@ -340,6 +340,10 @@ pub struct Validity<'a> { /// `C` 有时间词但锚不到。`None` = 没经过时间解析（人写的、规则算的），按 A 算。 /// 时态引擎读它决定这一行能不能关上前任 pub from_grade: Option<&'a str>, + /// 终点是原文所说时段之后的那一刻（`facts.end_after_period`，#994）：「2024 年离开」关上 + /// 2024-03-01 起的那段，存 2025-01-01，原文说的是 2024。只看值分不出它和「到 2025 年」。 + /// 写入口归一（`drop_period_mark_unless`），只留在带得了它的行上（[`period_mark_fits`]） + pub end_after_period: bool, } /// `valid_to_precision` 表示「结束了，但不知道是哪天」。 @@ -458,6 +462,7 @@ impl<'a> Validity<'a> { to_precision: None, attested_at: None, from_grade: None, + end_after_period: false, } } @@ -527,6 +532,15 @@ impl<'a> Validity<'a> { self } + /// 「终点说的是它之前那个时段」带不了的行（[`period_mark_fits`]），标记在这里丢掉：两个 + /// 写入口（`insert_fact_on`、`correct_interval`）在 [`Validity::under`] 之后各过一遍。 + /// 事件归一成一刻、恒常抹掉日期、没有谓词的行不过 `under`，都落在这里。进写入口的终点是 + /// 写明的，不是时间线画的 + pub(crate) fn drop_period_mark_unless(mut self, declared_state: bool) -> Self { + self.end_after_period &= period_mark_fits(declared_state, self.from, self.to, false); + self + } + /// 这条断言是否已经不再成立——**两种结束都算**。 /// /// 判据写在这里而不是散在各处的 `valid_to.is_some()`：那种写法会把 @@ -536,6 +550,23 @@ impl<'a> Validity<'a> { } } +/// 一个终点带不带得了「说的是它之前那个时段」的标记（`end_after_period`，#994）：属性声明成 +/// 状态、有起点、终点有日期、不是时间线画的。 +/// +/// 与约束 `facts_end_after_period_needs_a_stated_end` 同一句话（属性是不是状态约束看不到， +/// 由这里查）。Rust 这边判这件事的都调它：写入口的归一（`Validity::drop_period_mark_unless`）、 +/// `temporal::rewrite_end_tx` 落标记前、`temporal::mark_end_after_period` 补标记前、改区间的 +/// 路由。没有起点的行第一刀不标：终点只在起点落在那个时段里时才被挪到时段的尽头。第二刀让 +/// 时间解析自己写这种终点时，起点那一条在这里和约束里一起去掉 +pub fn period_mark_fits( + declared_state: bool, + from: Option>, + to: Option>, + end_derived: bool, +) -> bool { + declared_state && from.is_some() && to.is_some() && !end_derived +} + /// 一段起止同值的状态（#966）：写入、改区间、关上都拒它，说的是同一句话 pub fn empty_state_span() -> AppError { AppError::invalid( @@ -592,7 +623,8 @@ pub(crate) async fn insert_fact_on( validity.truncated().under(temporal)? } else { validity.truncated() - }; + } + .drop_period_mark_unless(declared_state); let same_sql = match object { FactObject::Entity(_) => { "SELECT id, valid_from, valid_to, valid_to_precision FROM facts @@ -650,9 +682,14 @@ pub(crate) async fn insert_fact_on( && vtp.as_deref() == Some(ENDED_UNKNOWN) && validity.from.is_none_or(|f| Some(f) == *vf) }) { - if let Some(stated) = - crate::temporal::state_derived_end(&mut *conn, *ended, None, validity.attested_at) - .await? + if let Some(stated) = crate::temporal::state_derived_end( + &mut *conn, + *ended, + None, + false, + validity.attested_at, + ) + .await? { return Ok((stated, true)); } @@ -674,27 +711,43 @@ pub(crate) async fn insert_fact_on( // 照旧关在时段的开头（0053 修订 2026-09-27）。同一天开始又结束的也是这样：关在 // 那一天的尽头，不关在它自己的起点——那是一段不成立的状态（#966） let period_end = declared_state.then(|| bucket_end(to, Some(precision))); + // 关在哪一刻，和终点是不是因此挪到了时段的尽头：挪了的，行上记下终点说的是它之前 + // 那个时段（`end_after_period`，#994），读者才印得出原文说的 2024 let closes_at = |from: Option>| match (from, period_end) { - (Some(f), Some(end)) if to <= f && f < end => end, - _ => to, + (Some(f), Some(end)) if to <= f && f < end => (end, true), + _ => (to, false), }; // 已经关在那一刻的：同一件事，复用那一行（引擎推的终点改成写明的，同上） if let Some((ended, from, _, _)) = same .iter() - .find(|(_, vf, vt, _)| *vt == Some(closes_at(*vf))) + .find(|(_, vf, vt, _)| *vt == Some(closes_at(*vf).0)) { + let (at, moved) = closes_at(*from); if let Some(stated) = crate::temporal::state_derived_end( &mut *conn, *ended, - Some((closes_at(*from), precision)), + Some((at, precision)), + moved, validity.attested_at, ) .await? { return Ok((stated, true)); } - attest_earlier(&mut *conn, *ended, validity.attested_at, true).await?; - return Ok((*ended, false)); + // 写明的终点被另一种说法再说一遍：那一行是「到 2025 年」关上的，这次是「2024 年 + // 离开」，同一个终点。标上它，两句谁先到都一样。上面只比了时刻：人改过的终点、 + // 别的精度写明的终点不改，由 `mark_end_after_period` 锁住那一行再查。 + // 标上了也是并进那一行，不报新行：报成新行，物化会给它记上这句的 + // `from_statement_id`、规则会记上 `implied`，人关上的行就成了算出来的，这句的 + // 陈述一没，它跟着被作废 + let marked = if moved { + crate::temporal::mark_end_after_period(&mut *conn, *ended, precision).await? + } else { + None + }; + let row = marked.unwrap_or(*ended); + attest_earlier(&mut *conn, row, validity.attested_at, true).await?; + return Ok((row, false)); } let open = same .iter() @@ -708,13 +761,9 @@ pub(crate) async fn insert_fact_on( }) .max_by_key(|(_, vf, _, _)| *vf); if let Some((open, from, _, _)) = open { - if let Some(closed) = crate::temporal::close_superseded( - &mut *conn, - *open, - closes_at(*from), - precision, - ) - .await? + let (at, moved) = closes_at(*from); + if let Some(closed) = + crate::temporal::close_stated(&mut *conn, *open, at, precision, moved).await? { return Ok((closed, true)); } @@ -724,13 +773,16 @@ pub(crate) async fn insert_fact_on( // 精确重复：同 valid_from → 复用。同起点、**这次带了终点、那行还开着** → 关上它 // （「自 2020-01-10 起任董事」之后读到「2020-01-10 至 2024-04-30 任董事」） if let Some((existing, _, vt, vtp)) = same.iter().find(|(_, vf, _, _)| *vf == validity.from) { + // 这次带来的终点若说的是它之前那个时段（勘误改出来的行接旧行的标记），关上时一起带上； + // 复用的那行原样留着它自己的终点和标记 if temporal == Temporal::State && vt.is_none() && vtp.is_none() { if let Some(to) = validity.to { - if let Some(closed) = crate::temporal::close_superseded( + if let Some(closed) = crate::temporal::close_stated( &mut *conn, *existing, to, validity.to_precision.unwrap_or("day"), + validity.end_after_period, ) .await? { @@ -747,6 +799,7 @@ pub(crate) async fn insert_fact_on( &mut *conn, *existing, stated_to, + validity.end_after_period, validity.attested_at, ) .await? @@ -800,22 +853,22 @@ pub(crate) async fn insert_fact_on( if vf.is_some() { return None; } - let end = match (validity.to, *vt) { + let (end, moved) = match (validity.to, *vt) { (None, Some(t)) if declared_state && t <= from => { let end = bucket_end(t, Some(vtp.as_deref().unwrap_or("day"))); if from >= end { return None; } - Some(end) + (Some(end), true) } (Some(to), Some(t)) if to != t => return None, - (_, vt) => vt, + (_, vt) => (vt, false), }; - Some((*id, end, vtp.clone())) + Some((*id, end, vtp.clone(), moved)) }), None => None, }; - if let Some((_, Some(vt), vtp)) = &refine_target { + if let Some((_, Some(vt), vtp, moved)) = &refine_target { if validity.to.is_none() { validity.to = Some(*vt); validity.to_precision = vtp.as_deref().map(|p| match p { @@ -827,9 +880,11 @@ pub(crate) async fn insert_fact_on( "second" => "second", _ => ENDED_UNKNOWN, }); + // 沿用来的终点挪到了时段的尽头：与收尾那条路同一个标记，两种陈述顺序写下同一行 + validity.end_after_period = *moved; } } - let refine_target = refine_target.map(|(id, _, _)| id); + let refine_target = refine_target.map(|(id, _, _, _)| id); let id = Uuid::now_v7(); let insert_sql = match object { @@ -838,17 +893,17 @@ pub(crate) async fn insert_fact_on( "INSERT INTO facts (id, kb_id, subject_id, predicate_id, object_id, valid_from, valid_from_precision, valid_to, valid_to_precision, confidence, - attested_from, attested_to, valid_from_grade) + attested_from, attested_to, valid_from_grade, end_after_period) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, COALESCE($11, now()), - CASE WHEN $9::text = 'unknown' THEN COALESCE($11, now()) END, $12)" + CASE WHEN $9::text = 'unknown' THEN COALESCE($11, now()) END, $12, $13)" } FactObject::Value(_) => { "INSERT INTO facts (id, kb_id, subject_id, predicate_id, object_value, valid_from, valid_from_precision, valid_to, valid_to_precision, confidence, - attested_from, attested_to, valid_from_grade) + attested_from, attested_to, valid_from_grade, end_after_period) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, COALESCE($11, now()), - CASE WHEN $9::text = 'unknown' THEN COALESCE($11, now()) END, $12)" + CASE WHEN $9::text = 'unknown' THEN COALESCE($11, now()) END, $12, $13)" } }; let mut ins = sqlx::query(insert_sql) @@ -867,6 +922,7 @@ pub(crate) async fn insert_fact_on( .bind(confidence) .bind(validity.attested_at) .bind(validity.from_grade) + .bind(validity.end_after_period) .execute(&mut *conn) .await?; @@ -1750,7 +1806,7 @@ pub async fn entity_detail( WHERE fe.fact_id = f.id AND {chunk_live}) ) AS stale, (f.supersedes IS NOT NULL) AS corrected, - f.end_derived, {corrected_ends} AS corrected_ends, + f.end_derived, f.end_after_period, {corrected_ends} AS corrected_ends, {corrected_ends} IS NOT NULL AS time_corrected, closer.closed_by_id, closer.closed_by, closer.closed_by_value, CASE WHEN {corrected_ends} IS NOT NULL THEN {correction_note} END @@ -2543,6 +2599,7 @@ mod temporal_shape_tests { to_precision: Some("day"), attested_at: None, from_grade: None, + end_after_period: false, } .under(Temporal::Event) .unwrap(); @@ -2560,6 +2617,7 @@ mod temporal_shape_tests { to_precision: Some("month"), attested_at: None, from_grade: None, + end_after_period: false, } .under(Temporal::Event) .unwrap(); @@ -2604,6 +2662,7 @@ mod temporal_shape_tests { to_precision: Some("day"), attested_at: None, from_grade: None, + end_after_period: false, }; let refused = moment .under(Temporal::State) diff --git a/crates/utopia-store/src/materialize.rs b/crates/utopia-store/src/materialize.rs index cd81523b5..2fc72d21a 100644 --- a/crates/utopia-store/src/materialize.rs +++ b/crates/utopia-store/src/materialize.rs @@ -289,7 +289,8 @@ async fn materialize_in_tx( let reverse = d.direction == "reverse"; // 一刻的陈述在状态属性下按绑定说的读（#966）：开始写成从那一刻起，结束写成到那一刻 // 为止（走 `insert_fact_on`「某天结束了」那条路，关上开放的那段）。什么都不算的 - // 在查询里已经排掉了（[`MOMENT_UNREAD`]） + // 在查询里已经排掉了（[`MOMENT_UNREAD`]）。陈述不带 `end_after_period`：终点挪没挪到 + // 时段的尽头由写入路径定（#994 第一刀） let moment = d.temporal.as_deref() == Some("state") && d.valid_from.is_some() && d.valid_from == d.valid_to; @@ -301,6 +302,7 @@ async fn materialize_in_tx( to: d.valid_to, to_precision: d.valid_to_precision.as_deref(), attested_at: d.attested_from, + end_after_period: false, }, (true, Some("start")) => Validity { from: d.valid_from, @@ -309,6 +311,7 @@ async fn materialize_in_tx( to: None, to_precision: None, attested_at: d.attested_from, + end_after_period: false, }, (true, Some("end")) => Validity { from: None, @@ -317,6 +320,7 @@ async fn materialize_in_tx( to: d.valid_from, to_precision: d.valid_from_precision.as_deref(), attested_at: d.attested_from, + end_after_period: false, }, (true, _) => continue, }; @@ -592,6 +596,7 @@ async fn imply_in_tx( to: d.valid_to, to_precision: d.valid_to_precision.as_deref(), attested_at: d.attested_from, + end_after_period: false, }; let value; let object = match (d.object_id, &d.object_value) { diff --git a/crates/utopia-store/src/pending.rs b/crates/utopia-store/src/pending.rs index 822803515..8d581902d 100644 --- a/crates/utopia-store/src/pending.rs +++ b/crates/utopia-store/src/pending.rs @@ -315,6 +315,7 @@ pub async fn confirm(pool: &PgPool, kb_id: Uuid, id: Uuid) -> AppResult { diff --git a/crates/utopia-store/src/temporal.rs b/crates/utopia-store/src/temporal.rs index 7b21cef75..9b68dda3b 100644 --- a/crates/utopia-store/src/temporal.rs +++ b/crates/utopia-store/src/temporal.rs @@ -590,7 +590,7 @@ async fn tidy( for (id, end) in diff(&rows, ends) { // 重新打开的行只是开着，终点不是谁推出来的 let derived = end != End::Open; - if let Some(corrected) = rewrite_end_tx(tx, id, &end, derived, false).await? { + if let Some(corrected) = rewrite_end_tx(tx, id, &end, derived, false, false).await? { rewritten.insert(id, corrected); report.corrected.push(corrected); } @@ -880,6 +880,19 @@ pub async fn close_superseded<'a>( fact_id: Uuid, valid_to: DateTime, valid_to_precision: &str, +) -> AppResult> { + // 按日期关上的（审核里关、冲突的 close_at、违规的 fact_closed）给的就是终点那一刻，不标 + close_stated(pool, fact_id, valid_to, valid_to_precision, false).await +} + +/// [`close_superseded`]，外加这个终点是不是原文所说时段之后的那一刻（`end_after_period`， +/// #994）：写入路径把「2024 年离开」关在 2025-01-01 时标上，读者才印得出 2024 +pub(crate) async fn close_stated<'a>( + pool: impl sqlx::Acquire<'a, Database = Postgres>, + fact_id: Uuid, + valid_to: DateTime, + valid_to_precision: &str, + after_period: bool, ) -> AppResult> { // 闭合点截到它的精度（0024）：月精度的闭合就是那个月的 1 日 0 点 let end = End::At( @@ -887,7 +900,7 @@ pub async fn close_superseded<'a>( valid_to_precision.to_string(), ); let mut tx = pool.begin().await?; - let corrected = rewrite_end_tx(&mut tx, fact_id, &end, false, false).await?; + let corrected = rewrite_end_tx(&mut tx, fact_id, &end, false, false, after_period).await?; tx.commit().await?; Ok(corrected) } @@ -903,19 +916,28 @@ pub async fn close_with_unknown_end<'a>( attested_at: Option>, ) -> AppResult> { let mut tx = pool.begin().await?; - let corrected = - rewrite_end_tx(&mut tx, fact_id, &End::Unknown(attested_at), false, true).await?; + let corrected = rewrite_end_tx( + &mut tx, + fact_id, + &End::Unknown(attested_at), + false, + true, + false, + ) + .await?; tx.commit().await?; Ok(corrected) } /// 原文说出了一条**引擎关上**的行的终点：改写成原文说的，此后不再重算（#679 第三轮评审）。 -/// `valid_to` 为 `None` 是「结束了，不知哪天」，锚在 `attested_at`。返回修正行 id； -/// 行已作废，或它的终点本来就是写明的，返回 `None` +/// `valid_to` 为 `None` 是「结束了，不知哪天」，锚在 `attested_at`；`after_period`：原文说的 +/// 是那一刻之前的那个时段（#994）。返回修正行 id；行已作废，或它的终点本来就是写明的， +/// 返回 `None` pub async fn state_derived_end<'a>( pool: impl sqlx::Acquire<'a, Database = Postgres>, fact_id: Uuid, valid_to: Option<(DateTime, &str)>, + after_period: bool, attested_at: Option>, ) -> AppResult> { let end = match valid_to { @@ -933,15 +955,60 @@ pub async fn state_derived_end<'a>( .fetch_optional(&mut *tx) .await?; let corrected = match derived { - Some(true) => rewrite_end_tx(&mut tx, fact_id, &end, false, false).await?, + Some(true) => rewrite_end_tx(&mut tx, fact_id, &end, false, false, after_period).await?, _ => None, }; tx.commit().await?; Ok(corrected) } +/// 一条写明的终点又被另一种说法说了一遍：「until 2025」关上的行又读到「left in 2024」。 +/// 时刻不变，改写成带标记的一行（#994）；反过来的顺序在收尾那条路上已经标了，两种顺序 +/// 写下同一行。 +/// +/// 调用方（`insert_fact_on`）只比了时刻，这里锁住那一行再查两件事：精度与这次说的相同—— +/// 日精度写明的 2025-01-01 是那一天，不是 2024 年的尽头；终点不是人改过的 +/// （`corrected_ends` 为空或只有起点）——人把显示的 2024 改成 2025，就是要到 2025 年。 +/// 已经标过的不再标。那一行带不带得了标记（引擎画的终点、没有起点的行带不了），读出来交给 +/// `graph::period_mark_fits` 判。返回修正行 id；不必标的返回 `None` +pub(crate) async fn mark_end_after_period<'a>( + pool: impl sqlx::Acquire<'a, Database = Postgres>, + fact_id: Uuid, + precision: &str, +) -> AppResult> { + let mut tx = pool.begin().await?; + #[allow(clippy::type_complexity)] + let locked_row: Option<(Option>, Option>, bool, bool)> = + sqlx::query_as( + "SELECT f.valid_from, f.valid_to, f.end_derived, + COALESCE(r.temporal = 'state', false) AS declared_state + FROM facts f LEFT JOIN relation_types r ON r.id = f.predicate_id + WHERE f.id = $1 AND f.invalidated_at IS NULL + AND f.valid_to_precision = $2 AND NOT f.end_after_period + AND (f.corrected_ends IS NULL OR f.corrected_ends = 'start') + FOR UPDATE OF f", + ) + .bind(fact_id) + .bind(precision) + .fetch_optional(&mut *tx) + .await?; + let marked = match locked_row { + // 起点那一半人改过的照旧是人的：改写的 CASE 把 'start' 原样带过去 + Some((from, Some(at), end_derived, declared_state)) + if crate::graph::period_mark_fits(declared_state, from, Some(at), end_derived) => + { + let end = End::At(at, precision.to_string()); + rewrite_end_tx(&mut tx, fact_id, &end, false, false, true).await? + } + _ => None, + }; + tx.commit().await?; + Ok(marked) +} + /// 作废 + 改写一行的终点。`derived`：这个终点是引擎按时间线推出来的（0057），之后随 -/// 时间线重算；`require_open`：只改还开着的行。 +/// 时间线重算；`require_open`：只改还开着的行；`after_period`：终点是原文所说时段之后的 +/// 那一刻（#994）。 /// /// 先 `FOR UPDATE` 锁住旧行：并行的两次改写只有一次看得见它还活着，另一次拿到 `None`， /// 不会各插一条修正行；往这一行上添证据的也要等它（`graph::add_evidence`），证据不会落在 @@ -952,6 +1019,7 @@ async fn rewrite_end_tx( end: &End, derived: bool, require_open: bool, + after_period: bool, ) -> AppResult> { let alive: Option<(Option>, Option)> = sqlx::query_as( "SELECT valid_from, predicate_id FROM facts @@ -983,13 +1051,22 @@ async fn rewrite_end_tx( End::At(at, precision) => (Some(*at), Some(precision.as_str()), None), End::Unknown(anchor) => (None, Some(crate::graph::ENDED_UNKNOWN), *anchor), }; + // 标记只落在带得了它的终点上（`graph::period_mark_fits`）：时间线画的、不知哪天的、重新 + // 打开的终点，没有起点或属性不是状态的行，一律不带——起点和谓词是上面锁住时读的。要标时 + // 才去查属性：不标的改写（时间线重画、关成不知哪天）不多一次查询 + let after_period = after_period && { + let declared_state = predicate.is_some() + && crate::graph::predicate_temporal(&mut **tx, predicate).await? + == crate::graph::Temporal::State; + crate::graph::period_mark_fits(declared_state, from, valid_to, derived) + }; let corrected = Uuid::now_v7(); sqlx::query( "INSERT INTO facts (id, kb_id, subject_id, predicate_id, object_id, object_value, valid_from, valid_from_precision, valid_from_grade, valid_to, valid_to_precision, confidence, supersedes, attested_from, attested_by, attested_to, end_derived, - from_statement_id, implied, corrected_ends) + from_statement_id, implied, corrected_ends, end_after_period) SELECT $1, kb_id, subject_id, predicate_id, object_id, object_value, valid_from, valid_from_precision, valid_from_grade, $3, $4, confidence, id, attested_from, attested_by, @@ -998,7 +1075,8 @@ async fn rewrite_end_tx( -- 终点在这里被重新写下（文档说了终点，或时间线推了一个），人改过的终点就不在了； -- 起点那一半照旧 CASE corrected_ends WHEN 'both' THEN 'start' WHEN 'end' THEN NULL - ELSE corrected_ends END + ELSE corrected_ends END, + $7 FROM facts WHERE id = $2", ) .bind(corrected) @@ -1007,6 +1085,7 @@ async fn rewrite_end_tx( .bind(precision) .bind(anchor) .bind(derived) + .bind(after_period) .execute(&mut **tx) .await?; carry_open_conflicts(tx, fact_id, corrected).await?; @@ -1073,11 +1152,11 @@ pub async fn rehome_tx( valid_from, valid_from_precision, valid_to, valid_to_precision, confidence, derived_by_rule, supersedes, attested_from, attested_to, end_derived, - from_statement_id, implied, corrected_ends) + from_statement_id, implied, corrected_ends, end_after_period) SELECT $1, kb_id, COALESCE($3, subject_id), predicate_id, COALESCE($4, object_id), object_value, valid_from, valid_from_precision, valid_to, valid_to_precision, confidence, derived_by_rule, id, attested_from, attested_to, end_derived, - from_statement_id, implied, corrected_ends + from_statement_id, implied, corrected_ends, end_after_period FROM facts WHERE id = $2", ) .bind(moved) @@ -1184,12 +1263,15 @@ async fn copy_evidence( /// 改写同一条规矩（[`copy_materialization_links`]）。不带过去的话，陈述不再被活着的 /// 类型化行代表，画面上它按原话、原来的区间回来，下一轮物化还照它再算一行。 /// -/// 改了哪一端记在修正行上（`corrected_ends`，#970）：值或精度变了的那一端，与旧行已有的 -/// 取并集，与修正同一个事务——事实行据此说「这个日期是人改的」，不再靠事后那条审计。 +/// 改了哪一端记在修正行上（`corrected_ends`，#970）：值或精度（终点还有标记）变了的那一端， +/// 与旧行已有的取并集，与修正同一个事务——事实行据此说「这个日期是人改的」，不再靠事后那条审计。 /// 人交来的是整段区间，改完两端都是人的（`end_derived` 清掉，时间线不再重画它）：原来 /// 是时间线推出来的终点，人没改它的值也是把它钉住了，一样算人的；原来就是原文说的那一端 /// 没改，仍是原文的 /// +/// 终点是不是原文所说时段之后的那一刻（`end_after_period`，#994）由调用方给，这里只归一： +/// 修正路由按请求定它，从不替人换算 +/// /// 返回修正行 id；`None` 表示这条已被并发改写或作废，本次没有动手。 pub async fn correct_interval( pool: &PgPool, @@ -1209,15 +1291,17 @@ pub async fn correct_interval( let validity = match predicate { Some(_) => validity.truncated().under(temporal)?, None => validity.truncated(), - }; + } + .drop_period_mark_unless(predicate.is_some() && temporal == crate::graph::Temporal::State); let mut tx = pool.begin().await?; let corrected = Uuid::now_v7(); + // 标记变了也算终点变了（#994）：人把显示的 2024 改成 2025，存的时刻没动，说的却是另一年 let inserted: Option<(Uuid,)> = sqlx::query_as( "INSERT INTO facts (id, kb_id, subject_id, predicate_id, object_id, object_value, valid_from, valid_from_precision, valid_to, valid_to_precision, confidence, supersedes, attested_from, attested_to, - from_statement_id, implied, corrected_ends) + from_statement_id, implied, corrected_ends, end_after_period) SELECT $1, kb_id, subject_id, predicate_id, object_id, object_value, $3, $4, $5, $6, confidence, id, attested_from, @@ -1225,7 +1309,8 @@ pub async fn correct_interval( from_statement_id, implied, CASE WHEN start_marked AND end_marked THEN 'both' WHEN start_marked THEN 'start' - WHEN end_marked THEN 'end' END + WHEN end_marked THEN 'end' END, + $7 FROM (SELECT f.*, c.start_changed OR COALESCE(f.corrected_ends IN ('start', 'both'), false) AS start_marked, @@ -1238,6 +1323,7 @@ pub async fn correct_interval( AS start_changed, f.valid_to IS DISTINCT FROM $5::timestamptz OR f.valid_to_precision IS DISTINCT FROM $6::text + OR f.end_after_period IS DISTINCT FROM $7::boolean AS end_changed) c WHERE f.id = $2 AND f.invalidated_at IS NULL) old RETURNING id", @@ -1248,6 +1334,7 @@ pub async fn correct_interval( .bind(validity.from_precision) .bind(validity.to) .bind(validity.to_precision) + .bind(validity.end_after_period) .fetch_optional(&mut *tx) .await?; // 已被并发修正过：不重复动手（与 close_superseded 同一防线） diff --git a/crates/utopia-store/tests/a_corrected_row_says_which_end_changed.rs b/crates/utopia-store/tests/a_corrected_row_says_which_end_changed.rs index 581d46b22..a421e6b72 100644 --- a/crates/utopia-store/tests/a_corrected_row_says_which_end_changed.rs +++ b/crates/utopia-store/tests/a_corrected_row_says_which_end_changed.rs @@ -36,6 +36,7 @@ fn between(from: &str, to: Option<&str>) -> Validity<'static> { to_precision: to.map(|_| "day"), attested_at: None, from_grade: None, + ..Default::default() } } diff --git a/crates/utopia-store/tests/store/a_fact_awaits_a_nod.rs b/crates/utopia-store/tests/store/a_fact_awaits_a_nod.rs index 0249cc06e..c2a0a0825 100644 --- a/crates/utopia-store/tests/store/a_fact_awaits_a_nod.rs +++ b/crates/utopia-store/tests/store/a_fact_awaits_a_nod.rs @@ -120,6 +120,7 @@ async fn a_remembered_fact_waits_for_a_nod() -> anyhow::Result<()> { to_precision: None, attested_at: None, from_grade: None, + ..Default::default() }, confidence: 0.9, chunk_id: f.chunk, diff --git a/crates/utopia-store/tests/store/a_moment_marks_its_state.rs b/crates/utopia-store/tests/store/a_moment_marks_its_state.rs index 7f2634eb8..d454a9979 100644 --- a/crates/utopia-store/tests/store/a_moment_marks_its_state.rs +++ b/crates/utopia-store/tests/store/a_moment_marks_its_state.rs @@ -14,6 +14,10 @@ //! 照旧；已有的空段不并进后来的观察，也不被它关上；勘误改不了一段不成立的时间，也不能把 //! 一刻的事件改成状态；规则没有 marks，只说了一刻的陈述不蕴含状态，只有起点的照旧。 //! +//! 终点因此挪到时段尽头的行记着原文说的是那个时段（`end_after_period`，#994）：收尾、精化 +//! 两条路都记；关在「until 2025」的行再读到「left in 2024」也记（人只改过起点的也记），时间线画 +//! 的终点被它说出来也记；搬主宾、勘误改出来的行带着它，改成事件的不带。 +//! //! 没有 `UTOPIA_DATABASE_URL` 时跳过而不是失败。自建自拆，绝不碰已有的库。 use chrono::{DateTime, Utc}; @@ -45,6 +49,7 @@ fn moment() -> Validity<'static> { to_precision: Some("day"), attested_at: None, from_grade: None, + ..Default::default() } } @@ -57,6 +62,7 @@ fn ending(at: DateTime) -> Validity<'static> { to_precision: Some("day"), attested_at: None, from_grade: None, + ..Default::default() } } @@ -274,6 +280,21 @@ impl Base { ) } + /// Lin Zhao —works_for→ object，走写入口（抽取、点头、人写的事实同一个门） + async fn write_to(&self, object: Uuid, v: Validity<'_>) -> anyhow::Result { + let (fact, _) = graph::insert_fact( + &self.pool, + self.kb, + self.lin, + Some(self.works_for), + object, + v, + 0.9, + ) + .await?; + Ok(fact) + } + async fn remove(self) -> anyhow::Result<()> { sqlx::query("DELETE FROM organizations WHERE id = $1") .bind(self.org) @@ -997,7 +1018,8 @@ async fn a_row_a_person_set_to_a_single_date_is_not_retired() -> anyhow::Result< /// 终点说的是一个时段，比的却是时刻（#975 评审）：「left … in 2024」存成 2024-01-01，年精度。 /// 「works for … from 2024-03-01」从这个时段里开始，关在时段的尽头 2025-01-01，而不是另立一行 -/// `- → 2024`、让它一直开着到 2025 年以后。先读到哪一句都一样 +/// `- → 2024`、让它一直开着到 2025 年以后。先读到哪一句都一样；行上记着终点说的是 2024 +/// （#994：收尾那条路和精化那条路都记） #[tokio::test] async fn a_coarse_end_closes_a_row_that_starts_inside_its_period() -> anyhow::Result<()> { let Some(pool) = pool().await? else { @@ -1008,6 +1030,7 @@ async fn a_coarse_end_closes_a_row_that_starts_inside_its_period() -> anyhow::Re t("2024-01-01T00:00:00Z"), t("2025-01-01T00:00:00Z"), ); + let mut marks: Vec> = Vec::new(); for order in [["works for", "left"], ["left", "works for"]] { let base = Base::new(&pool).await?; let run = async { @@ -1037,12 +1060,22 @@ async fn a_coarse_end_closes_a_row_that_starts_inside_its_period() -> anyhow::Re "{order:?}: not still working there in 2025" ); assert_eq!(materialize(&pool, base.kb).await?, Outcome::default()); + marks.push( + sqlx::query_scalar( + "SELECT end_after_period FROM facts + WHERE kb_id = $1 AND layer = 'typed' AND invalidated_at IS NULL", + ) + .bind(base.kb) + .fetch_all(&pool) + .await?, + ); anyhow::Ok(()) } .await; base.remove().await?; run?; } + assert_eq!(marks, [[true], [true]], "the end names 2024, not 2025"); Ok(()) } @@ -1108,6 +1141,245 @@ async fn a_coarse_end_leaves_rows_outside_its_period_as_before() -> anyhow::Resu Ok(()) } +/// 一个终点，两种说法（#994）：人写的 2024-03-01 起的一行关在「until 2025」，文档又说「left in +/// 2024」，同一个终点：标上，并进那一行，它仍是人的（反过来的顺序是收尾那条路）；另一份文档再说 +/// 一次，标过的原样留着。人只把起点挪到 4 月的一行（Dune）也标。日精度写明的 2025-01-01（Aster） +/// 是那一天，人改过的终点（Cobalt）是人的，都不改写 +#[tokio::test] +async fn one_end_named_two_ways_is_marked() -> anyhow::Result<()> { + let Some(pool) = pool().await? else { + return Ok(()); + }; + let base = Base::new(&pool).await?; + let run = async { + let (year, next_year) = (t("2024-01-01T00:00:00Z"), t("2025-01-01T00:00:00Z")); + let since = Validity::starting(Some(t("2024-03-01T00:00:00Z")), Some("day")); + let until_2025 = Validity { + to_precision: Some("year"), + ..ending(next_year) + }; + let (cobalt, dune) = (Uuid::now_v7(), Uuid::now_v7()); + sqlx::query( + "INSERT INTO entities (id, kb_id, type_id, canonical_name) + VALUES ($1, $2, $3, 'Cobalt'), ($4, $2, $3, 'Dune')", + ) + .bind(cobalt) + .bind(base.kb) + .bind(base.company) + .bind(dune) + .execute(&pool) + .await?; + for (object, end) in [(base.meridian, until_2025), (base.aster, ending(next_year))] { + base.write_to(object, since).await?; + base.write_to(object, end).await?; + } + let by_hand = Validity { + to: Some(next_year), + to_precision: Some("year"), + ..since + }; + let open = base.write_to(cobalt, since).await?; + utopia_store::temporal::correct_interval(&pool, open, by_hand).await?; + let stated = base.write_to(dune, by_hand).await?; + let april = Validity { + from: Some(t("2024-04-01T00:00:00Z")), + ..by_hand + }; + utopia_store::temporal::correct_interval(&pool, stated, april).await?; + for object in [base.meridian, base.meridian, base.aster, cobalt, dune] { + base.statement_to("left", object, Some(year), Some(year), "year") + .await?; + } + base.bind("left", Some("end")).await?; + materialize(&pool, base.kb).await?; + // 每个宾语活着的那一行：标记，仍是人的（不是这句算出来的），那边被取代过几行 + let rows: Vec<(String, bool, bool, i64)> = sqlx::query_as( + "SELECT o.canonical_name, f.end_after_period, f.from_statement_id IS NULL, + (SELECT count(*) FROM facts g WHERE g.layer = 'typed' + AND g.object_id = f.object_id AND g.invalidated_at IS NOT NULL) + FROM facts f JOIN entities o ON o.id = f.object_id + WHERE f.kb_id = $1 AND f.layer = 'typed' AND f.invalidated_at IS NULL + ORDER BY 1", + ) + .bind(base.kb) + .fetch_all(&pool) + .await?; + // Meridian 开着的那行先被「until 2025」关上、再被标上，各取代一次；Dune 改过一次起点 + let expected = [ + ("Aster Labs", false, 1), + ("Cobalt", false, 1), + ("Dune", true, 2), + ("Meridian Systems", true, 2), + ] + .map(|(o, mark, gone)| (o.to_string(), mark, true, gone)); + assert_eq!(rows, expected); + anyhow::Ok(()) + } + .await; + base.remove().await?; + run +} + +/// 时间线画的终点被原文说出来（#994）：works_for 声明成函数性、继任者 2025-01-01 起，时间线在那天 +/// 画上 Meridian 的终点（日精度）；「left in 2024」说的是同一刻，改成写明的年精度终点，记着说的是 +/// 2024 +#[tokio::test] +async fn a_drawn_end_the_text_names_as_a_period_is_marked() -> anyhow::Result<()> { + let Some(pool) = pool().await? else { + return Ok(()); + }; + let base = Base::new(&pool).await?; + let run = async { + let next_year = t("2025-01-01T00:00:00Z"); + sqlx::query("UPDATE relation_types SET functional = true WHERE id = $1") + .bind(base.works_for) + .execute(&pool) + .await?; + let since = Validity::starting(Some(t("2024-03-01T00:00:00Z")), Some("day")); + base.write_to(base.meridian, since).await?; + let next = Validity::starting(Some(next_year), Some("day")); + let next = base.write_to(base.aster, next).await?; + utopia_store::temporal::reconcile_moved_facts(&pool, base.kb, &[next]).await?; + // Meridian 那一行的终点：精度、是不是时间线画的、标记 + let end = || { + sqlx::query_as::<_, (String, bool, bool)>( + "SELECT valid_to_precision, end_derived, end_after_period FROM facts + WHERE object_id = $1 AND invalidated_at IS NULL", + ) + .bind(base.meridian) + .fetch_one(&pool) + }; + let drawn = end().await?; + let left_in_2024 = Validity { + to_precision: Some("year"), + ..ending(t("2024-01-01T00:00:00Z")) + }; + base.write_to(base.meridian, left_in_2024).await?; + assert_eq!( + (drawn, end().await?), + (("day".into(), true, false), ("year".into(), false, true)), + "drawn, then stated as the end of 2024" + ); + anyhow::Ok(()) + } + .await; + base.remove().await?; + run +} + +/// 标记随行走（#994）：撤回合并搬主宾用的 `rehome_tx` 带着它；勘误改出来的行接旧行的标记——改到 +/// 同起点、还开着的一行的宾语上，那一行经同起点那条路关上；改到同起点、终点是时间线画的一行的 +/// 宾语上，那个终点改成写明的——都带着它；属性改成事件的，写入口丢掉它 +#[tokio::test] +async fn a_marked_end_rides_through_a_rehome_and_a_revision() -> anyhow::Result<()> { + let Some(pool) = pool().await? else { + return Ok(()); + }; + let base = Base::new(&pool).await?; + let run = async { + let kb = base.kb; + let since = Validity::starting(Some(t("2024-03-01T00:00:00Z")), Some("day")); + let closed = Validity { + to: Some(t("2025-01-01T00:00:00Z")), + to_precision: Some("year"), + ..since + }; + let row = base.write_to(base.meridian, closed).await?; + let (cobalt, drawn, doc) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7()); + let (company, lin, works_for) = (base.company, base.lin, base.works_for); + // Only locally generated UUIDs are interpolated into fixture SQL. + sqlx::raw_sql(&format!( + "UPDATE facts SET end_after_period = TRUE WHERE id = '{row}'; + INSERT INTO entities (id, kb_id, type_id, canonical_name) + VALUES ('{cobalt}', '{kb}', '{company}', 'Cobalt'); + INSERT INTO facts (id, kb_id, subject_id, predicate_id, object_id, layer, confidence, + valid_from, valid_from_precision, valid_to, valid_to_precision, + end_derived) + VALUES ('{drawn}', '{kb}', '{lin}', '{works_for}', '{cobalt}', 'typed', 0.9, + '2024-03-01T00:00:00Z', 'day', '2025-01-01T00:00:00Z', 'day', TRUE); + INSERT INTO documents (id, kb_id, filename, sha256) + VALUES ('{doc}', '{kb}', 'cv.txt', 'x');" + )) + .execute(&pool) + .await?; + let mut tx = pool.begin().await?; + let moved = utopia_store::temporal::rehome_tx(&mut tx, row, None, Some(base.aster)) + .await? + .expect("the row was live"); + tx.commit().await?; + + // 勘误改三次：宾语改回 Meridian，那边已有一行 2024-03-01 起开着；改到 Cobalt，那边同起点的 + // 一行终点是时间线画的（日精度，上面写下的）；属性改成 acquired（事件） + let open = base.write_to(base.meridian, since).await?; + let mut candidate = Candidate { + fact_id: moved, + statement_id: None, + predicate_id: base.works_for, + subject_id: base.lin, + subject: "Lin Zhao".into(), + subject_class: Some("person".into()), + property: "works_for".into(), + property_label: "works for".into(), + object_id: Some(base.aster), + object: "Aster Labs".into(), + object_class: Some("organization".into()), + flag: Some("name_absent".into()), + quote: None, + }; + // 一行的标记，和它取代的是哪一行 + let read = |id: Uuid| { + sqlx::query_as::<_, (bool, Option)>( + "SELECT end_after_period, supersedes FROM facts WHERE id = $1", + ) + .bind(id) + .fetch_one(&pool) + }; + let mut rows = vec![read(moved).await?]; + let run_id = errata::start_run(&pool, kb, doc, 3, 0).await?; + for (property, object) in [ + (None, Some("Meridian Systems")), + (None, Some("Cobalt")), + (Some("acquired"), None), + ] { + let recorded = errata::record( + &pool, + kb, + ActionInput { + run_id, + document_id: doc, + candidate: Some(&candidate), + proposed: Proposed::Revise { + property: property.map(Into::into), + object: object.map(Into::into), + }, + reason: "the document says otherwise", + quote: Some("Lin Zhao left Aster Labs for Meridian Systems"), + document_text: "Lin Zhao left Aster Labs for Meridian Systems in 2024.", + }, + ) + .await?; + candidate.fact_id = recorded + .new_fact_id + .unwrap_or_else(|| panic!("{recorded:?}")); + rows.push(read(candidate.fact_id).await?); + } + assert_eq!( + rows, + [ + (true, Some(row)), + (true, Some(open)), + (true, Some(drawn)), + (false, None) + ], + "rehomed; revised onto an open row and a drawn end; onto an event" + ); + anyhow::Ok(()) + } + .await; + base.remove().await?; + run +} + /// 两个值同一刻开始的冲突（works_for 声明成函数性）：人选「关上旧的」却不给日期，旧的就关在 /// 新的起点——也是它自己的起点，一段不成立的状态，拒绝（`empty_state_span`）。冲突留着等人 /// 给一个日期 diff --git a/crates/utopia-store/tests/store/a_path_joins_two_entities.rs b/crates/utopia-store/tests/store/a_path_joins_two_entities.rs index 36d516ae6..4ba81a867 100644 --- a/crates/utopia-store/tests/store/a_path_joins_two_entities.rs +++ b/crates/utopia-store/tests/store/a_path_joins_two_entities.rs @@ -109,6 +109,7 @@ async fn seed(pool: &PgPool) -> anyhow::Result { to_precision: to.map(|_| "month"), attested_at: None, from_grade: None, + ..Default::default() }; let mut direct = Uuid::nil(); for (s, p, o, v) in [ diff --git a/crates/utopia-store/tests/store/a_timeline_is_recomputed_from_the_rows_it_has.rs b/crates/utopia-store/tests/store/a_timeline_is_recomputed_from_the_rows_it_has.rs index f6216db57..079f7c750 100644 --- a/crates/utopia-store/tests/store/a_timeline_is_recomputed_from_the_rows_it_has.rs +++ b/crates/utopia-store/tests/store/a_timeline_is_recomputed_from_the_rows_it_has.rs @@ -120,6 +120,7 @@ async fn observe(pool: &PgPool, f: &Fixture, x: Seen<'_>) -> anyhow::Result anyhow::Result<()> to_precision: Some("day"), attested_at: None, from_grade: None, + ..Default::default() }, ) .await? diff --git a/crates/utopia-store/tests/store/a_wrong_time_can_be_corrected.rs b/crates/utopia-store/tests/store/a_wrong_time_can_be_corrected.rs index d06286d1d..c0e85d1f8 100644 --- a/crates/utopia-store/tests/store/a_wrong_time_can_be_corrected.rs +++ b/crates/utopia-store/tests/store/a_wrong_time_can_be_corrected.rs @@ -257,6 +257,7 @@ async fn an_end_that_was_never_there_can_be_taken_back() -> anyhow::Result<()> { to_precision: Some("year"), attested_at: None, from_grade: None, + ..Default::default() }, ) .await? diff --git a/crates/utopia-store/tests/store/an_end_date_closes_the_open_span.rs b/crates/utopia-store/tests/store/an_end_date_closes_the_open_span.rs index 0d2a2cd2a..da6c351ea 100644 --- a/crates/utopia-store/tests/store/an_end_date_closes_the_open_span.rs +++ b/crates/utopia-store/tests/store/an_end_date_closes_the_open_span.rs @@ -26,6 +26,7 @@ fn span( to_precision: to.map(|_| "day"), attested_at: Some(day(attested)), from_grade: None, + ..Default::default() } } @@ -54,6 +55,7 @@ fn ends(to: &str, attested: &str) -> utopia_store::graph::Validity<'static> { to_precision: Some("day"), attested_at: Some(day(attested)), from_grade: None, + ..Default::default() } } @@ -147,6 +149,7 @@ async fn an_end_date_closes_the_open_span() -> anyhow::Result<()> { to_precision: None, attested_at: Some(day("2025-08-01")), from_grade: None, + ..Default::default() }, 0.9, ) diff --git a/crates/utopia-store/tests/store/an_event_holds_at_the_moment_it_names.rs b/crates/utopia-store/tests/store/an_event_holds_at_the_moment_it_names.rs index 53ac19c73..784d50a3a 100644 --- a/crates/utopia-store/tests/store/an_event_holds_at_the_moment_it_names.rs +++ b/crates/utopia-store/tests/store/an_event_holds_at_the_moment_it_names.rs @@ -312,6 +312,7 @@ async fn a_span_an_ending_or_an_unknown_end_given_to_an_event_collapses() -> any to_precision: Some("day"), attested_at: None, from_grade: None, + ..Default::default() }, ) .await?; @@ -338,6 +339,7 @@ async fn a_span_an_ending_or_an_unknown_end_given_to_an_event_collapses() -> any to_precision: Some("month"), attested_at: None, from_grade: None, + ..Default::default() }, ) .await?; @@ -581,6 +583,7 @@ async fn a_person_correcting_an_event_leaves_it_one_moment() -> anyhow::Result<( to_precision: Some("day"), attested_at: None, from_grade: None, + ..Default::default() }, ) .await? diff --git a/crates/utopia-store/tests/store/an_unknown_date_is_not_an_open_one.rs b/crates/utopia-store/tests/store/an_unknown_date_is_not_an_open_one.rs index 302a93db1..010a99b3d 100644 --- a/crates/utopia-store/tests/store/an_unknown_date_is_not_an_open_one.rs +++ b/crates/utopia-store/tests/store/an_unknown_date_is_not_an_open_one.rs @@ -392,6 +392,7 @@ async fn the_anchor_moves_earlier_and_is_inherited() -> anyhow::Result<()> { to_precision: Some("day"), attested_at: None, from_grade: None, + ..Default::default() }, ) .await? diff --git a/crates/utopia-store/tests/the_engine_redraws_only_what_it_drew.rs b/crates/utopia-store/tests/the_engine_redraws_only_what_it_drew.rs index 8108ce5f3..ad013843c 100644 --- a/crates/utopia-store/tests/the_engine_redraws_only_what_it_drew.rs +++ b/crates/utopia-store/tests/the_engine_redraws_only_what_it_drew.rs @@ -152,6 +152,7 @@ async fn observe(pool: &PgPool, f: &Fixture, x: Seen<'_>) -> anyhow::Result Date: Mon, 19 Oct 2026 04:50:59 +0200 Subject: [PATCH 07/19] The typed recompute plans its whole-base statements for the base it is given, and the source cleanup reads only that base's statements. (#1307) Signed-off-by: Bogdan Szabo --- crates/utopia-store/src/materialize.rs | 18 +++++++++++ .../src/materialize_plan_tests.rs | 31 +++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 crates/utopia-store/src/materialize_plan_tests.rs diff --git a/crates/utopia-store/src/materialize.rs b/crates/utopia-store/src/materialize.rs index 2fc72d21a..2383072be 100644 --- a/crates/utopia-store/src/materialize.rs +++ b/crates/utopia-store/src/materialize.rs @@ -156,6 +156,11 @@ async fn materialize_in_tx( tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, kb_id: Uuid, ) -> AppResult<(Outcome, Vec)> { + // The statements that scan the whole base (steps 0 to 3, the implication scans and the + // attestation sync) are not cached: a cached statement switches to a generic plan after a few + // runs, and a generic plan cannot see how many rows `kb_id = $1` selects. The source cleanup's + // generic plan ran for hours on a production base where the plan for its own id takes two + // seconds (#1298). The per-row statements in the loops stay cached. // 这一轮写下的行（新建的和并入的），提交后对账 let mut written: Vec = Vec::new(); // 0. 起止同值的物化状态行（#966 之前照抄一刻写下的）任何时刻都不成立：作废，它的陈述 @@ -181,6 +186,7 @@ async fn materialize_in_tx( OR s.id IN (SELECT i.statement_id FROM implied_fact_sources i WHERE i.fact_id = t.id)))", ) + .persistent(false) .bind(kb_id) .execute(&mut **tx) .await? @@ -198,6 +204,7 @@ async fn materialize_in_tx( LEFT JOIN entities oe ON oe.id = s.object_id JOIN phrase_bindings b ON {MATCH} WHERE s.id = src.statement_id + AND s.kb_id = $1 AND s.invalidated_at IS NULL AND t.invalidated_at IS NULL AND b.status = 'bound' @@ -208,6 +215,7 @@ async fn materialize_in_tx( WHERE q.fact_id = s.id AND q.role = 'mood') AND NOT {MOMENT_UNREAD})" )) + .persistent(false) .bind(kb_id) .execute(&mut **tx) .await?; @@ -232,6 +240,7 @@ async fn materialize_in_tx( RULE_MATCH = crate::implication_rules::RULE_MATCH, KIND = kind_word_sql("e.specific_type"), )) + .persistent(false) .bind(kb_id) .execute(&mut **tx) .await?; @@ -245,6 +254,7 @@ async fn materialize_in_tx( AND NOT EXISTS (SELECT 1 FROM typed_fact_sources src WHERE src.fact_id = t.id) AND NOT EXISTS (SELECT 1 FROM implied_fact_sources i WHERE i.fact_id = t.id)", ) + .persistent(false) .bind(kb_id) .execute(&mut **tx) .await? @@ -280,6 +290,7 @@ async fn materialize_in_tx( AND t.predicate_id = b.relation_type_id) ORDER BY s.id" )) + .persistent(false) .bind(kb_id) .fetch_all(&mut **tx) .await?; @@ -496,6 +507,7 @@ pub async fn sync_typed_attestation<'e>( OR t.attested_by IS DISTINCT FROM src.by) RETURNING t.id", ) + .persistent(false) .bind(kb_id) .bind(only) .fetch_all(ex) @@ -561,6 +573,7 @@ async fn imply_in_tx( TEXT = crate::implication_rules::object_text_sql(), RULE_MATCH = crate::implication_rules::RULE_MATCH, )) + .persistent(false) .bind(kb_id) .fetch_all(&mut **tx) .await?; @@ -583,6 +596,7 @@ async fn imply_in_tx( ORDER BY e.id", KIND = kind_word_sql("e.specific_type"), )) + .persistent(false) .bind(kb_id) .fetch_all(&mut **tx) .await?; @@ -703,3 +717,7 @@ pub async fn count(pool: &PgPool, kb_id: Uuid) -> AppResult { #[cfg(test)] #[path = "materialize_delivery_tests.rs"] mod delivery_tests; + +#[cfg(test)] +#[path = "materialize_plan_tests.rs"] +mod plan_tests; diff --git a/crates/utopia-store/src/materialize_plan_tests.rs b/crates/utopia-store/src/materialize_plan_tests.rs new file mode 100644 index 000000000..507b93667 --- /dev/null +++ b/crates/utopia-store/src/materialize_plan_tests.rs @@ -0,0 +1,31 @@ +//! The recompute's whole-base statements are planned for the base they are given (#1298): a cached +//! statement switches to a generic plan after a few runs, and the generic plan of the source +//! cleanup ran for hours on a production base where the plan for the base's own id takes seconds. +use super::materialize; +use sqlx::postgres::PgPoolOptions; +use uuid::Uuid; + +#[tokio::test] +async fn a_recompute_leaves_none_of_its_whole_base_statements_cached_on_its_connection( +) -> anyhow::Result<()> { + let Some(url) = crate::test_db::url() else { + return Ok(()); + }; + let pool = PgPoolOptions::new() + .max_connections(1) + .connect(&url) + .await?; + let base = Uuid::now_v7(); + + materialize(&pool, base).await?; + materialize(&pool, base).await?; + let cached: Vec = sqlx::query_scalar( + "SELECT statement FROM pg_prepared_statements + WHERE statement LIKE '%kb_id = $1%' AND statement NOT LIKE '%pg_prepared_statements%'", + ) + .fetch_all(&pool) + .await?; + + assert_eq!(cached, Vec::::new()); + Ok(()) +} From 02af3452c480509200c7f8a057a8c0138fb420ce Mon Sep 17 00:00:00 2001 From: Szabo Bogdan Date: Mon, 19 Oct 2026 04:55:58 +0200 Subject: [PATCH 08/19] Exact-name resolution counts a candidate's facts only when it has to choose between candidates (#1309) Signed-off-by: Bogdan Szabo Co-authored-by: Claude Opus 5.5 Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- crates/utopia-store/src/resolution.rs | 68 +++++++++++---- .../a_single_name_candidate_is_not_counted.rs | 84 +++++++++++++++++++ crates/utopia-store/tests/store/main.rs | 1 + 3 files changed, 137 insertions(+), 16 deletions(-) create mode 100644 crates/utopia-store/tests/store/a_single_name_candidate_is_not_counted.rs diff --git a/crates/utopia-store/src/resolution.rs b/crates/utopia-store/src/resolution.rs index e9f11d59e..08d6badf3 100644 --- a/crates/utopia-store/src/resolution.rs +++ b/crates/utopia-store/src/resolution.rs @@ -193,7 +193,28 @@ struct Candidate { canonical_name: String, profile_embedding: Option, profile_n: i32, - degree: i64, +} + +/// 这些实体各自的度数（不数名字事实），与 `ids` 同序；没有事实的是 0。 +async fn degrees(pool: &PgPool, kb_id: Uuid, ids: &[Uuid]) -> AppResult> { + let rows: HashMap = sqlx::query_as(&format!( + "SELECT n.id, count(*) + FROM unnest($2::uuid[]) AS n(id) + JOIN facts f ON f.kb_id = $1 AND (f.subject_id = n.id OR f.object_id = n.id) + AND f.invalidated_at IS NULL AND {not_name} + GROUP BY n.id", + not_name = crate::names::not_a_name("f"), + )) + .bind(kb_id) + .bind(ids) + .fetch_all(pool) + .await? + .into_iter() + .collect(); + Ok(ids + .iter() + .map(|id| rows.get(id).copied().unwrap_or(0)) + .collect()) } /// 消解结果：mention 落到了哪个实体；附带需要入队的疑似重复审核对 @@ -378,11 +399,9 @@ async fn resolve_by_name( let keys = recall_keys(&name); // 名字召回读名字事实（0041 决定 1）：本名、简称、曾用名都是 `known_as` 上的一条。 // 度数不数名字事实——名字不是一条「关于它的事」，数进去每个实体都凭空多一 + // 度数要等真有几个候选得比时才数：每个候选一条计数子查询，大库上很贵（#1296） let candidates: Vec = sqlx::query_as(&format!( - "SELECT e.id, e.canonical_name, e.profile_embedding, e.profile_n, - (SELECT count(*) FROM facts f - WHERE f.kb_id = e.kb_id AND (f.subject_id = e.id OR f.object_id = e.id) - AND f.invalidated_at IS NULL AND {not_name}) AS degree + "SELECT e.id, e.canonical_name, e.profile_embedding, e.profile_n FROM entities e -- IS NOT DISTINCT FROM 而不是 =（0009 的那个陷阱）：开放图谱里的实体都没有类 -- （类由对齐来定），`type_id = NULL` 永远不成立，同名的它就永远撞不上—— @@ -392,7 +411,6 @@ async fn resolve_by_name( -- 不是召回的桥——两篇文档里描述得一样的两个东西不能因此接到一起 AND e.description IS NULL AND (lower(e.canonical_name) = ANY($3) OR {named})", - not_name = crate::names::not_a_name("f"), named = crate::names::has_name_in("e", 1, 3), )) .bind(kb_id) @@ -412,10 +430,19 @@ async fn resolve_by_name( let Some(ctx) = context else { // 无向量可比：v1 兼容 —— 归并到事实最多的同名候选 - let best = candidates - .iter() - .max_by_key(|c| c.degree) - .expect("non-empty"); + let best = match candidates.as_slice() { + [only] => only, + _ => { + let ids: Vec = candidates.iter().map(|c| c.id).collect(); + let degree = degrees(pool, kb_id, &ids).await?; + candidates + .iter() + .zip(degree) + .max_by_key(|(_, d)| *d) + .map(|(c, _)| c) + .expect("non-empty") + } + }; touch_entity(pool, best.id).await?; return Ok(Resolution { entity_id: best.id, @@ -435,7 +462,7 @@ async fn resolve_by_name( // 有画像的候选算相似度；无画像（历史数据/无 embedding 期创建）单独归类 let mut scored: Vec<(&Candidate, f32)> = Vec::new(); - let mut unprofiled: Option<&Candidate> = None; + let mut unprofiled: Vec<&Candidate> = Vec::new(); for c in &candidates { match c .profile_embedding @@ -443,11 +470,7 @@ async fn resolve_by_name( .and_then(|p| cosine(p.as_slice(), ctx)) { Some(sim) => scored.push((c, sim)), - None => { - if unprofiled.map(|u| c.degree > u.degree).unwrap_or(true) { - unprofiled = Some(c); - } - } + None => unprofiled.push(c), } } // 最高分候选：并列时保留先遇到的那个（与旧的 `sim > s` 严格大于一致） @@ -522,6 +545,19 @@ async fn resolve_by_name( }); } } + let unprofiled = match unprofiled.as_slice() { + [] => None, + [only] => Some(*only), + many => { + let ids: Vec = many.iter().map(|c| c.id).collect(); + let degree = degrees(pool, kb_id, &ids).await?; + many.iter() + .copied() + .zip(degree) + .reduce(|acc, cur| if cur.1 > acc.1 { cur } else { acc }) + .map(|(c, _)| c) + } + }; if let Some(c) = unprofiled { // 无画像候选无从判别：v1 兼容归并，并用本次上下文初始化画像 update_profile(pool, c.id, c.profile_n, ctx).await?; diff --git a/crates/utopia-store/tests/store/a_single_name_candidate_is_not_counted.rs b/crates/utopia-store/tests/store/a_single_name_candidate_is_not_counted.rs new file mode 100644 index 000000000..63febee75 --- /dev/null +++ b/crates/utopia-store/tests/store/a_single_name_candidate_is_not_counted.rs @@ -0,0 +1,84 @@ +//! 同名候选只有一个时没有可比的，不必数它的事实（#1296）。 +//! 一条连接的池子：sqlx 把它跑过的语句都缓存在 `pg_prepared_statements` 里，没跑过的就不在。 + +use sqlx::postgres::PgPoolOptions; +use utopia_store::names; +use uuid::Uuid; + +#[tokio::test] +async fn a_single_name_candidate_is_attached_without_counting_its_facts() -> anyhow::Result<()> { + let Some(url) = utopia_store::test_db::url() else { + return Ok(()); + }; + let pool = PgPoolOptions::new() + .max_connections(1) + .connect(&url) + .await?; + let (org, ws, kb) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7()); + let (person, entity) = (Uuid::now_v7(), Uuid::now_v7()); + let tag = "single-name-candidate"; + sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, $2)") + .bind(org) + .bind(tag) + .execute(&pool) + .await?; + sqlx::query("INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, $3)") + .bind(ws) + .bind(org) + .bind(tag) + .execute(&pool) + .await?; + sqlx::query("INSERT INTO knowledge_bases (id, workspace_id, name) VALUES ($1, $2, $3)") + .bind(kb) + .bind(ws) + .bind(tag) + .execute(&pool) + .await?; + sqlx::query("INSERT INTO entity_types (id, kb_id, key, label) VALUES ($1, $2, $3, $4)") + .bind(person) + .bind(kb) + .bind("person") + .bind("Person") + .execute(&pool) + .await?; + sqlx::query( + "INSERT INTO entities (id, kb_id, type_id, canonical_name) VALUES ($1, $2, $3, $4)", + ) + .bind(entity) + .bind(kb) + .bind(person) + .bind("林小满") + .execute(&pool) + .await?; + names::record(&pool, kb, entity, "林小满", None, None).await?; + + let run = async { + let r = utopia_store::resolution::resolve_mention( + &pool, + kb, + Some(person), + "林小满", + None, + None, + None, + &[], + ) + .await?; + assert!(!r.created && r.entity_id == entity, "唯一的同名候选就是它"); + let counted: Vec = sqlx::query_scalar( + "SELECT statement FROM pg_prepared_statements WHERE statement LIKE '%count(*)%facts f%'", + ) + .persistent(false) + .fetch_all(&pool) + .await?; + assert_eq!(counted, Vec::::new(), "只有一个候选，不该数事实"); + anyhow::Ok(()) + } + .await; + + sqlx::query("DELETE FROM organizations WHERE id = $1") + .bind(org) + .execute(&pool) + .await?; + run +} diff --git a/crates/utopia-store/tests/store/main.rs b/crates/utopia-store/tests/store/main.rs index 1f61b94b2..2915505a2 100644 --- a/crates/utopia-store/tests/store/main.rs +++ b/crates/utopia-store/tests/store/main.rs @@ -88,6 +88,7 @@ mod a_secret_is_sealed_at_rest; mod a_signature_holds_on_every_path; mod a_signature_vector_lasts_while_its_text_does; mod a_similar_name_is_proposed_not_merged; +mod a_single_name_candidate_is_not_counted; mod a_source_kind_is_listed_once; mod a_source_reaches_only_where_it_was_granted; mod a_statement_is_attested_by_what_its_document_says; From 5d915e2abf2a70d55bf495aeafcb9928f14d29ba Mon Sep 17 00:00:00 2001 From: Szabo Bogdan Date: Mon, 19 Oct 2026 05:01:23 +0200 Subject: [PATCH 09/19] Binding a pushed document's declared phrases counts the base's signatures only for keys not already settled (#1310) Signed-off-by: Bogdan Szabo Co-authored-by: Claude Opus 5.5 Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- crates/utopia-server/src/phrase_alignment.rs | 37 ++++++++++++++++--- .../src/phrase_alignment_declared_tests.rs | 35 ++++++++++++++++-- 2 files changed, 63 insertions(+), 9 deletions(-) diff --git a/crates/utopia-server/src/phrase_alignment.rs b/crates/utopia-server/src/phrase_alignment.rs index 8082f7bee..bdce28481 100644 --- a/crates/utopia-server/src/phrase_alignment.rs +++ b/crates/utopia-server/src/phrase_alignment.rs @@ -237,6 +237,18 @@ pub(crate) async fn bind_declared( .into_iter() .map(|b| (b.key(), b)) .collect(); + // 判过的先剔掉：库级的签名是整库分组，每次推送都数一遍太贵（#1296） + let keys: HashSet = keys + .into_iter() + .filter(|k| { + !existing + .get(k) + .is_some_and(|b| settled(b, &keyed(k), &props, &closure)) + }) + .collect(); + if keys.is_empty() { + return Ok(0); + } let mut written = 0; for s in phrase_bindings::signatures(pool, kb_id) .await? @@ -246,12 +258,6 @@ pub(crate) async fn bind_declared( let Some(p) = declared(s, &props, &closure) else { continue; }; - if existing - .get(&s.key()) - .is_some_and(|b| settled(b, s, &props, &closure)) - { - continue; - } if phrase_bindings::decide( pool, kb_id, @@ -277,6 +283,25 @@ pub(crate) async fn bind_declared( /// 签名的键：短语 + 两端的类 + 宾语是不是字面值（与 `PhraseSignature::key` 同形） type SignatureKey = (String, Option, Option, bool); + +/// 只有键的签名：`settled` 只看键上的这几格，不必为它把库里的签名数一遍 +fn keyed( + (phrase, subject_type_id, object_type_id, object_is_value): &SignatureKey, +) -> PhraseSignature { + PhraseSignature { + phrase: phrase.clone(), + subject_type_id: *subject_type_id, + subject_type_key: None, + object_type_id: *object_type_id, + object_type_key: None, + object_is_value: *object_is_value, + count: 0, + examples: Vec::new(), + quotes: Vec::new(), + example_statement_ids: Vec::new(), + } +} + /// 每条签名此刻的候选与指纹 type Considered<'a> = HashMap, String)>; diff --git a/crates/utopia-server/src/phrase_alignment_declared_tests.rs b/crates/utopia-server/src/phrase_alignment_declared_tests.rs index ac2a8cce0..ca07dd702 100644 --- a/crates/utopia-server/src/phrase_alignment_declared_tests.rs +++ b/crates/utopia-server/src/phrase_alignment_declared_tests.rs @@ -86,8 +86,8 @@ impl Rig { })) } - /// 推一篇文档并抽取：一条陈述，写成 [主语, 它的类别词, 短语, 宾语, 它的类别词] - async fn push(&self, external_id: &str, said: [&str; 5]) -> anyhow::Result<()> { + /// 推一篇文档并抽取：一条陈述，写成 [主语, 它的类别词, 短语, 宾语, 它的类别词]。返回文档 + async fn push(&self, external_id: &str, said: [&str; 5]) -> anyhow::Result { let [subject, subject_kind, phrase, object, object_kind] = said; let payload: Value = json!({ "external_id": external_id, @@ -118,7 +118,8 @@ impl Rig { doc, utopia_core::models::Proposer::default(), ) - .await + .await?; + Ok(doc) } async fn bindings(&self) -> anyhow::Result> { @@ -274,6 +275,34 @@ async fn only_a_phrase_naming_one_relation_that_fits_both_ends_binds() -> anyhow rig.cleanup().await } +/// 签名都已判过的推送不数库里的签名（#1296）。一条连接的池上 sqlx 缓存它跑过的每条语句， +/// 没跑的不在 `pg_prepared_statements` 里 +#[tokio::test] +async fn a_push_whose_signatures_are_settled_skips_the_base_signatures() -> anyhow::Result<()> { + let Some(rig) = Rig::new().await? else { + return Ok(()); + }; + rig.push("obs-1", ["S-1", "statement", "validated_by", "T-1", "test"]) + .await?; + let second = rig + .push("obs-2", ["S-2", "statement", "validated_by", "T-2", "test"]) + .await?; + let one = sqlx::postgres::PgPoolOptions::new() + .max_connections(1) + .connect_with((*rig.pool.connect_options()).clone()) + .await?; + assert_eq!(bind_declared(&one, rig.kb, second).await?, 0); + let grouped: Vec = sqlx::query_scalar( + "SELECT statement FROM pg_prepared_statements + WHERE statement LIKE '%WITH live AS%' AND statement NOT LIKE '%pg_prepared_statements%'", + ) + .fetch_all(&one) + .await?; + assert_eq!(grouped, Vec::::new()); + one.close().await; + rig.cleanup().await +} + #[tokio::test] async fn a_person_decision_is_not_overwritten_by_the_ontology() -> anyhow::Result<()> { let Some(rig) = Rig::new().await? else { From fa77a956797e86fc0465971d890c59b85ee971ec Mon Sep 17 00:00:00 2001 From: Szabo Bogdan Date: Mon, 19 Oct 2026 05:06:47 +0200 Subject: [PATCH 10/19] While a source pushes, the graph overview recounts the base every few seconds instead of three times a second (#1311) Signed-off-by: Bogdan Szabo Co-authored-by: Claude Opus 5.5 Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- web/src/pages/Graph.tsx | 4 +++- web/src/useEventReconnect.test.ts | 29 +++++++++++++++++++++++++--- web/src/useKbEvents.ts | 32 ++++++++++++++++++++++++++----- 3 files changed, 56 insertions(+), 9 deletions(-) diff --git a/web/src/pages/Graph.tsx b/web/src/pages/Graph.tsx index a878a7853..fc4e4cb9d 100644 --- a/web/src/pages/Graph.tsx +++ b/web/src/pages/Graph.tsx @@ -397,7 +397,9 @@ export function Graph() { canUpload: kb?.my_role !== "viewer", }); const data = useQuery({ - queryKey: ["graph", kb?.id, focusEntity, nodeBudget], + queryKey: focusEntity + ? ["graph", kb?.id, "neighborhood", focusEntity] + : ["graph", kb?.id, "overview", nodeBudget], queryFn: () => focusEntity ? api.graphNeighborhood(kb!.id, focusEntity) diff --git a/web/src/useEventReconnect.test.ts b/web/src/useEventReconnect.test.ts index 2e37fe5c4..c54d22f15 100644 --- a/web/src/useEventReconnect.test.ts +++ b/web/src/useEventReconnect.test.ts @@ -1,7 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { QueryClient, QueryObserver, type QueryKey } from "@tanstack/react-query"; import { DEFAULT_STALE_MS, STREAM_KEYS, STREAM_STALE_MS, applyQueryDefaults } from "./queryDefaults"; -import { SETTLE_MS, useKbEvents } from "./useKbEvents"; +import { OVERVIEW_SETTLE_MS, SETTLE_MS, useKbEvents } from "./useKbEvents"; import { useAlertEvents } from "./useAlertEvents"; import { HIDDEN_RELEASE_MS } from "./eventStream"; @@ -178,6 +178,7 @@ describe("KB stream reconnect", () => { it("flushes a pending burst together with reconnect instead of scheduling another refetch", async () => { const client = createClient(); const graph = await watch(client, ["graph", "kb-a"]); + const overview = await watch(client, ["graph", "kb-a", "overview", 200]); const documents = await watch(client, ["documents", "kb-a"]); const badge = await watch(client, ["alerts", "unread"]); const list = await watch(client, ["alerts", "list", "", 0]); @@ -190,11 +191,13 @@ describe("KB stream reconnect", () => { source.emit("error"); source.emit("open"); expect(graph.fetch).toHaveBeenCalledTimes(2); + expect(overview.fetch).toHaveBeenCalledTimes(2); expect(documents.fetch).toHaveBeenCalledTimes(2); expect(badge.fetch).toHaveBeenCalledTimes(2); expect(list.fetch).toHaveBeenCalledTimes(2); - await vi.advanceTimersByTimeAsync(SETTLE_MS); + await vi.advanceTimersByTimeAsync(OVERVIEW_SETTLE_MS); expect(graph.fetch).toHaveBeenCalledTimes(2); + expect(overview.fetch).toHaveBeenCalledTimes(2); expect(documents.fetch).toHaveBeenCalledTimes(2); expect(badge.fetch).toHaveBeenCalledTimes(2); expect(list.fetch).toHaveBeenCalledTimes(2); @@ -203,7 +206,7 @@ describe("KB stream reconnect", () => { it("keeps normal business events coalesced", async () => { const client = createClient(); - const graph = await watch(client, ["graph", "kb-a"]); + const graph = await watch(client, ["graph", "kb-a", "neighborhood", "e1"]); const documents = await watch(client, ["documents", "kb-a"]); const { source } = mount(() => useKbEvents("kb-a")); source.emit("open"); @@ -219,6 +222,26 @@ describe("KB stream reconnect", () => { expect(documents.fetch).toHaveBeenCalledTimes(2); }); + it("a burst of graph events refetches the overview once after OVERVIEW_SETTLE_MS and the neighbourhood after SETTLE_MS", async () => { + const client = createClient(); + const overview = await watch(client, ["graph", "kb-a", "overview", 200]); + const neighborhood = await watch(client, ["graph", "kb-a", "neighborhood", "e1"]); + const { source } = mount(() => useKbEvents("kb-a")); + source.emit("open"); + source.emit("graph"); + await vi.advanceTimersByTimeAsync(SETTLE_MS); + expect(neighborhood.fetch).toHaveBeenCalledTimes(2); + expect(overview.fetch).toHaveBeenCalledTimes(1); + for (let elapsed = SETTLE_MS; elapsed < 1_000; elapsed += 100) { + source.emit("graph"); + await vi.advanceTimersByTimeAsync(100); + } + await vi.advanceTimersByTimeAsync(OVERVIEW_SETTLE_MS - 1_000 - 1); + expect(overview.fetch).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(1); + expect(overview.fetch).toHaveBeenCalledTimes(2); + }); + it("ignores late callbacks from the old connection after switching KB", async () => { const client = createClient(); const oldKb = await watch(client, ["graph", "kb-a"]); diff --git a/web/src/useKbEvents.ts b/web/src/useKbEvents.ts index 8ca6a47f3..ea7c39fe9 100644 --- a/web/src/useKbEvents.ts +++ b/web/src/useKbEvents.ts @@ -18,6 +18,9 @@ import { STREAM_KEYS } from "./queryDefaults"; /** 一阵事件之间的静默期。抽取落事实的间隔远小于它，人眼看不出这点延迟 */ export const SETTLE_MS = 300; +/** 全图 overview 要数遍整个库：推送期间几秒重数一次就够了（#1296） */ +export const OVERVIEW_SETTLE_MS = 3_000; + /** * 把一阵失效合并成一次：`push` 只记键，静默 `settleMs` 后一起交给 `invalidate`， * 同一个键及其前缀只失效一次；`flush` 给卸载与重连用，攒着的刷掉而不是丢掉。 @@ -37,7 +40,7 @@ export function createInvalidationCoalescer( const keys = [...pending.values()]; pending.clear(); for (const key of keys) { - // 正常 graph 事件可能已排了 ["graph"]，重连又补上 ["graph", kbId]： + // 正常 graph 事件可能已排了 ["graph", kbId, "neighborhood"]，重连又补上 ["graph", kbId]： // 前缀已经覆盖后者，不能取消刚开始的请求再重取一遍。 if (keys.some((prefix) => prefix.length < key.length && partialMatchKey(key, prefix))) continue; invalidate(key); @@ -59,12 +62,31 @@ export function useKbEvents(kbId: string | undefined) { (key) => queryClient.invalidateQueries({ queryKey: key }), SETTLE_MS, ); + // overview 到点也经快的那层刷：重连补上的 ["graph", kbId] 已覆盖它，不再多取一遍 + const overview = createInvalidationCoalescer((key) => { + push(key); + flush(); + }, OVERVIEW_SETTLE_MS); + const pushGraph = () => { + push(["graph", kbId, "neighborhood"]); + overview.push(["graph", kbId, "overview"]); + }; + const flushAll = () => { + overview.flush(); + flush(); + }; const unsubscribe = subscribeEvents( `/api/v1/kbs/${kbId}/events`, { - document: () => push(["documents", kbId], ["graph"]), - graph: () => push(["graph"], ["ontologyRegressions", kbId]), + document: () => { + push(["documents", kbId]); + pushGraph(); + }, + graph: () => { + push(["ontologyRegressions", kbId]); + pushGraph(); + }, // 映射探索跑完发的也是 review：Pending 那一栏得跟着刷新 review: () => push(["review", kbId], ["mappings", kbId], ["ontologyRegressions", kbId]), // 一句记忆抽出了等人点头的事实（0015）：对话里那张确认卡跟着长出来 @@ -77,13 +99,13 @@ export function useKbEvents(kbId: string | undefined) { // 事件没有回放，staleTime 到期也不会自己发请求。断线后恢复必须补刷； // 首次正常连接交给页面已有的读取，首次连接失败再恢复则同样可能漏事件。 push(...STREAM_KEYS.map((head) => [head, kbId]), ["alerts"]); - flush(); + flushAll(); }, ); return () => { unsubscribe(); // 卸载时把攒着的刷掉而不是丢掉：换页回来看到的必须是新数据 - flush(); + flushAll(); }; }, [kbId, queryClient]); } From 68cb3ff707145c8e2f00874aaa45adeb01629b37 Mon Sep 17 00:00:00 2001 From: Szabo Bogdan Date: Mon, 19 Oct 2026 05:12:33 +0200 Subject: [PATCH 11/19] A token reads the entities, history and fact evidence of the bases it reaches, so a service mirroring into Utopia can read its own base back without a person's session (#1263) Signed-off-by: Bogdan Szabo Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- .../utopia-server/src/api/documents_routes.rs | 2 +- crates/utopia-server/src/api/graph_routes.rs | 19 +-- crates/utopia-server/src/api/mod.rs | 3 + .../utopia-server/src/api/token_read_tests.rs | 146 ++++++++++++++++++ crates/utopia-server/src/auth.rs | 2 +- ...ty-from-the-person-scope-from-the-token.md | 15 +- docs/design/access-and-audit.md | 5 +- 7 files changed, 178 insertions(+), 14 deletions(-) create mode 100644 crates/utopia-server/src/api/token_read_tests.rs diff --git a/crates/utopia-server/src/api/documents_routes.rs b/crates/utopia-server/src/api/documents_routes.rs index da196cc5e..0d184dea7 100644 --- a/crates/utopia-server/src/api/documents_routes.rs +++ b/crates/utopia-server/src/api/documents_routes.rs @@ -81,7 +81,7 @@ fn content_headers( Ok(headers) } -async fn require_reader_kb( +pub(crate) async fn require_reader_kb( state: &AppState, reader: &SessionOrToken, kb_id: Uuid, diff --git a/crates/utopia-server/src/api/graph_routes.rs b/crates/utopia-server/src/api/graph_routes.rs index 728c51f07..3116474a2 100644 --- a/crates/utopia-server/src/api/graph_routes.rs +++ b/crates/utopia-server/src/api/graph_routes.rs @@ -6,7 +6,8 @@ use utopia_core::models::Role; use utopia_core::AppError; use uuid::Uuid; -use crate::auth::AuthUser; +use super::documents_routes::require_reader_kb; +use crate::auth::{AuthUser, SessionOrToken}; use crate::error::ApiResult; use crate::state::AppState; @@ -138,11 +139,11 @@ pub struct EntitySearchQuery { pub async fn search_entities( State(state): State, - AuthUser(user): AuthUser, + reader: SessionOrToken, Path(kb_id): Path, Query(query): Query, ) -> ApiResult> { - require_kb(&state, &user, kb_id, Role::Viewer).await?; + require_reader_kb(&state, &reader, kb_id).await?; // 先校验时刻：一个写错的 `as_of` 配空查询也该回 400，而不是看起来成功 let as_of = parse_instant("as_of", query.as_of.as_deref())?; if query.q.trim().is_empty() { @@ -171,11 +172,11 @@ pub struct EntityDetailQuery { pub async fn entity_detail( State(state): State, - AuthUser(user): AuthUser, + reader: SessionOrToken, Path((kb_id, entity_id)): Path<(Uuid, Uuid)>, Query(q): Query, ) -> ApiResult> { - require_kb(&state, &user, kb_id, Role::Viewer).await?; + require_reader_kb(&state, &reader, kb_id).await?; let as_of = parse_instant("as_of", q.as_of.as_deref())?; // 世界轴不过滤（at = None）：面板要的是整条时间线，「此刻成立」那一档由前端 // 按 holds_from / holds_to 分出来（0022） @@ -518,10 +519,10 @@ pub async fn update_fact_time( pub async fn fact_evidence( State(state): State, - AuthUser(user): AuthUser, + reader: SessionOrToken, Path((kb_id, fact_id)): Path<(Uuid, Uuid)>, ) -> ApiResult> { - require_kb(&state, &user, kb_id, Role::Viewer).await?; + require_reader_kb(&state, &reader, kb_id).await?; let evidence = utopia_store::graph::fact_evidence(&state.pool, kb_id, fact_id).await?; Ok(Json(json!({ "evidence": evidence }))) } @@ -626,11 +627,11 @@ fn default_history_per() -> i64 { /// 与 entity_detail（有效时间轴，只看现行）互补。 pub async fn entity_history( State(state): State, - AuthUser(user): AuthUser, + reader: SessionOrToken, Path((kb_id, entity_id)): Path<(Uuid, Uuid)>, Query(q): Query, ) -> ApiResult> { - require_kb(&state, &user, kb_id, Role::Viewer).await?; + require_reader_kb(&state, &reader, kb_id).await?; let per = q.per.clamp(1, 200); let page = q.page.max(0); let (events, total) = diff --git a/crates/utopia-server/src/api/mod.rs b/crates/utopia-server/src/api/mod.rs index f0773a6c0..ec25e1fe6 100644 --- a/crates/utopia-server/src/api/mod.rs +++ b/crates/utopia-server/src/api/mod.rs @@ -661,5 +661,8 @@ mod oidc_exchange_tests; #[cfg(test)] mod token_setup_tests; +#[cfg(test)] +mod token_read_tests; + #[cfg(test)] mod rule_expression_tests; diff --git a/crates/utopia-server/src/api/token_read_tests.rs b/crates/utopia-server/src/api/token_read_tests.rs new file mode 100644 index 000000000..1855c84f1 --- /dev/null +++ b/crates/utopia-server/src/api/token_read_tests.rs @@ -0,0 +1,146 @@ +//! 个人令牌读实体（0014 决定 7）：搜实体、实体详情、认知历史、事实证据四条读路由 +//! 收任何一枚活着的令牌。角色照旧是上限，`kb_ids` 排除了的库答 404；改实体仍只认会话。 +use crate::state::AppState; +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use std::sync::Arc; +use tower::ServiceExt; +use uuid::Uuid; + +struct Fixture { + pool: sqlx::PgPool, + app: axum::Router, + org: Uuid, + kb: Uuid, + /// 令牌够不着的另一个库 + other_kb: Uuid, + reader: Uuid, + entity: Uuid, + fact: Uuid, + _dir: tempfile::TempDir, +} + +impl Fixture { + async fn new() -> anyhow::Result> { + let Some(url) = utopia_store::test_db::url() else { + return Ok(None); + }; + let pool = sqlx::PgPool::connect(&url).await?; + utopia_store::db::migrate(&pool).await?; + let id = Uuid::now_v7; + let (org, ws, reader, kb, other_kb) = (id(), id(), id(), id(), id()); + let (person, knows, entity, peer, fact) = (id(), id(), id(), id(), id()); + // 插进去的只有本地生成的 UUID 和写死的日期 + sqlx::raw_sql(&format!( + "INSERT INTO organizations(id,name) VALUES ('{org}','token-read-test'); + INSERT INTO workspaces(id,org_id,name) VALUES ('{ws}','{org}','token-read-test'); + INSERT INTO users(id,org_id,email,display_name,password_hash) VALUES + ('{reader}','{org}','{reader}@token-read.test','Reader','unused'); + INSERT INTO knowledge_bases(id,workspace_id,name) VALUES + ('{kb}','{ws}','mirror'), ('{other_kb}','{ws}','other'); + INSERT INTO kb_members(kb_id,user_id,role) VALUES + ('{kb}','{reader}','viewer'), ('{other_kb}','{reader}','viewer'); + INSERT INTO entity_types(id,kb_id,key,label) VALUES ('{person}','{kb}','person','Person'); + INSERT INTO relation_types(id,kb_id,key,label,temporal) VALUES + ('{knows}','{kb}','knows','knows','state'); + INSERT INTO entities(id,kb_id,type_id,canonical_name,created_at) VALUES + ('{entity}','{kb}','{person}','Zhang San','2026-01-01'), + ('{peer}','{kb}','{person}','Li Si','2026-01-01'); + INSERT INTO facts(id,kb_id,subject_id,predicate_id,object_id,recorded_at,confidence) + VALUES ('{fact}','{kb}','{entity}','{knows}','{peer}','2026-01-01',0.9);" + )) + .execute(&pool) + .await?; + let dir = tempfile::tempdir()?; + let cfg = utopia_core::config::AppConfig { + data_dir: dir.path().to_string_lossy().into_owned(), + ..Default::default() + }; + let search = Arc::new(utopia_search::SearchIndex::open( + &dir.path().join("search"), + )?); + let state = AppState::new(pool.clone(), &cfg, search, "test-only".into()); + let app = super::router(state, &cfg); + Ok(Some(Self { + pool, + app, + org, + kb, + other_kb, + reader, + entity, + fact, + _dir: dir, + })) + } + + async fn read_token(&self, kb: Uuid) -> String { + utopia_store::tokens::issue(&self.pool, self.reader, "ci", "read", Some(&[kb]), None) + .await + .expect("issuing a token") + .1 + } + + fn read_paths(&self) -> [String; 4] { + let (kb, entity, fact) = (self.kb, self.entity, self.fact); + [ + format!("/kbs/{kb}/entities?q=Zhang"), + format!("/kbs/{kb}/entities/{entity}"), + format!("/kbs/{kb}/entities/{entity}/history"), + format!("/kbs/{kb}/facts/{fact}/evidence"), + ] + } + + async fn get(&self, path: &str, token: &str) -> anyhow::Result { + let request = Request::builder() + .uri(format!("/api/v1{path}")) + .header("authorization", format!("Bearer {token}")) + .body(Body::empty())?; + Ok(self.app.clone().oneshot(request).await?.status()) + } + + async fn cleanup(self) -> anyhow::Result<()> { + sqlx::query("DELETE FROM organizations WHERE id = $1") + .bind(self.org) + .execute(&self.pool) + .await?; + Ok(()) + } +} + +#[tokio::test] +async fn a_read_token_of_a_base_viewer_reads_its_entities_facts_and_history() -> anyhow::Result<()> +{ + let Some(f) = Fixture::new().await? else { + return Ok(()); + }; + let result = async { + let token = f.read_token(f.kb).await; + for path in f.read_paths() { + let status = f.get(&path, &token).await?; + anyhow::ensure!(status == StatusCode::OK, "{path}: {status}"); + } + Ok(()) + } + .await; + f.cleanup().await?; + result +} + +#[tokio::test] +async fn a_token_limited_to_another_base_gets_404_on_every_entity_read() -> anyhow::Result<()> { + let Some(f) = Fixture::new().await? else { + return Ok(()); + }; + let result = async { + let token = f.read_token(f.other_kb).await; + for path in f.read_paths() { + let status = f.get(&path, &token).await?; + anyhow::ensure!(status == StatusCode::NOT_FOUND, "{path}: {status}"); + } + Ok(()) + } + .await; + f.cleanup().await?; + result +} diff --git a/crates/utopia-server/src/auth.rs b/crates/utopia-server/src/auth.rs index 49ebf01c6..5a7889866 100644 --- a/crates/utopia-server/src/auth.rs +++ b/crates/utopia-server/src/auth.rs @@ -176,7 +176,7 @@ impl FromRequestParts for AuthUser { /// /// **不替换 `AuthUser`。** `AuthUser` 把每个 Bearer 都当 JWT 解，令牌在那几百条 /// 路由上一律是 401——那是有意的：哪条路由准令牌进来，要在 0014 里说出理由。 -/// 用这个提取器的只有文档原件（0052）和「把一个库搭起来」那几条。 +/// 用这个提取器的只有文档原件（0052）、「把一个库搭起来」那几条和读实体那几条（决定 7）。 /// /// 令牌只收窄：`pat` 在的时候，路由先问 [`Self::require_write`] 或 /// [`Self::reaches`]，角色照旧由 `require_kb` 与工作区角色判。 diff --git a/docs/decisions/0014-identity-from-the-person-scope-from-the-token.md b/docs/decisions/0014-identity-from-the-person-scope-from-the-token.md index ed3082aec..58cff2554 100644 --- a/docs/decisions/0014-identity-from-the-person-scope-from-the-token.md +++ b/docs/decisions/0014-identity-from-the-person-scope-from-the-token.md @@ -1,6 +1,6 @@ # 0014 · Identity from the person, scope from the token -- **Status**: Implemented · 2026-10-15 · #180, #318, #1243 · open: tokens across workspaces +- **Status**: Implemented · 2026-10-16 · #180, #318, #1243 · open: tokens across workspaces - **Written**: 2026-09-01 · condensed into English 2026-09-03 - **Related**: migration `0014_data_source_grants` gave data sources a grant layer; this is the same question where a machine knocks. [0004](0004-language-and-localization.md) has the server speak English only, MCP error codes included. [0015](0015-recording-a-sentence-is-not-asserting-a-fact.md) removes the main objection to opening `remember` over MCP. @@ -85,6 +85,17 @@ push documents in. Not done: rotating or deleting a source, changing a base's settings or members, and every other write still take a session. A token does not create workspaces or tokens. +7. **A token reads the graph** (added 2026-10-16, #1262). A service that mirrors into a base reads the + base back to check what landed, and a session JWT is the wrong credential for a backend + (decision 6). Four reads now also take any live token, acting as its owner: searching a base's + entities (`GET /kbs/{id}/entities`), an entity's detail (`GET /kbs/{id}/entities/{eid}`), its + record-axis history (`…/history`), and a fact's evidence (`GET /kbs/{id}/facts/{fid}/evidence`). + Search finds the entity by name; the detail gives its facts, names and derivations; the + history says when we changed our mind; the evidence ties a fact to its sentence. Read is the + lowest scope, so any token qualifies; the owner's role is still the ceiling and Viewer + suffices; a token whose `kb_ids` exclude the base gets 404, as for document content (0052). + Changing an entity on the same path (`PATCH`) still takes a session. + ## Dead ends - **Base-level machine tokens**, one per base and unrelated to a person. It would be a third @@ -109,6 +120,8 @@ push documents in. - 2026-10-16: decision 6 let a token create only `statements` sources, though an `api` source is push-only as well; a backend mirroring several kinds of documents pushes them through one. A token now creates `api` sources too. +- 2026-10-16: decision 7 opens four graph reads to any token; decision 6's "only these routes + and document content" now also counts them. ## Open questions diff --git a/docs/design/access-and-audit.md b/docs/design/access-and-audit.md index cc331f69d..70770528c 100644 --- a/docs/design/access-and-audit.md +++ b/docs/design/access-and-audit.md @@ -19,8 +19,9 @@ writes an audit row `mcp.tool_called`; the tokens page shows the plaintext once snippet [0014]. A `write` token also sets up a base as its owner: creating a base and a `statements` or `api` source, reading the source's push key, importing an ontology and deciding a kind word, with the owner's role unchanged as the ceiling; any token lists a workspace's bases; a token limited by `kb_ids` -creates no base. These routes and document content take `SessionOrToken`; apart from MCP, every -other route is session-only [0014 d6]. `ingest_token` on a source is a different kind: plaintext, push-only [0014]. +creates no base. These routes, document content and four graph reads (entity search, detail, +history, fact evidence; Viewer suffices, `kb_ids` answers 404) take `SessionOrToken`; apart from +MCP, every other route is session-only [0014 d6, d7]. `ingest_token` on a source is a different kind: plaintext, push-only [0014]. **Another app can trade its user's fresh ID token for a session.** `POST /auth/oidc/exchange` accepts an ID token from the configured issuer whose audience is in From 2bcc763140deba7c663770e7154ac2e36a375f8a Mon Sep 17 00:00:00 2001 From: Daniel <118035379+Floating-Y@users.noreply.github.com> Date: Mon, 19 Oct 2026 11:17:54 +0800 Subject: [PATCH 12/19] Recognize East Asian Excel dates with calamine 0.37.0 (#1312) Signed-off-by: Floating-Y <118035379+Floating-Y@users.noreply.github.com> Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- Cargo.lock | 14 +-- Cargo.toml | 2 +- .../tests/fixtures/calamine/LICENSE-MIT.md | 23 +++++ .../fixtures/calamine/date_east_asian.xls | Bin 0 -> 9728 bytes .../utopia-ingest/tests/spreadsheet_dates.rs | 97 +++++++++++++++++- 5 files changed, 126 insertions(+), 10 deletions(-) create mode 100644 crates/utopia-ingest/tests/fixtures/calamine/LICENSE-MIT.md create mode 100644 crates/utopia-ingest/tests/fixtures/calamine/date_east_asian.xls diff --git a/Cargo.lock b/Cargo.lock index f7a8637eb..123cae836 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -524,9 +524,9 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "calamine" -version = "0.36.1" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fa68281b1a76b54a62156474adb06bb380a67e07dd60656e3217152b42183f3" +checksum = "c9ca341b94bc4b29e08970b1a03914fad2dfe177e2ecb67fcd6de9046004aafb" dependencies = [ "atoi_simd", "byteorder", @@ -534,9 +534,9 @@ dependencies = [ "encoding_rs", "fast-float2", "log", - "quick-xml 0.41.0", + "quick-xml 0.42.0", "serde", - "zip 8.6.0", + "zip 9.0.5", ] [[package]] @@ -3423,6 +3423,7 @@ version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b" dependencies = [ + "encoding_rs", "memchr", ] @@ -6293,14 +6294,15 @@ dependencies = [ [[package]] name = "zip" -version = "8.6.0" +version = "9.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" +checksum = "6e04976da629d3b04204ba8105b396a193af8ad2583b2c79bfb3ab8af580a11e" dependencies = [ "crc32fast", "flate2", "indexmap", "memchr", + "nix", "typed-path", "zopfli", ] diff --git a/Cargo.toml b/Cargo.toml index 5b65de568..fc8457022 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -66,7 +66,7 @@ text-splitter = { version = "0.32", features = ["tiktoken-rs"] } tiktoken-rs = "0.12" # 句子边界按 Unicode 标准（UAX #29）切，不自己写标点规则 pdf-extract = "0.12" -calamine = "0.36" +calamine = "0.37" pgvector = { version = "0.4", features = ["sqlx"] } # `gzip` 不是可选项：一次嵌入回 64×1024 个浮点数，写成 JSON 文本每个数字占 # 21 字节而信息量只有 4 字节，一次响应 1.33 MB。实测同一条链路上不压缩 62 秒、 diff --git a/crates/utopia-ingest/tests/fixtures/calamine/LICENSE-MIT.md b/crates/utopia-ingest/tests/fixtures/calamine/LICENSE-MIT.md new file mode 100644 index 000000000..16a30bcb7 --- /dev/null +++ b/crates/utopia-ingest/tests/fixtures/calamine/LICENSE-MIT.md @@ -0,0 +1,23 @@ +The MIT License (MIT) + +Copyright (c) 2016 Johann Tuffe + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/crates/utopia-ingest/tests/fixtures/calamine/date_east_asian.xls b/crates/utopia-ingest/tests/fixtures/calamine/date_east_asian.xls new file mode 100644 index 0000000000000000000000000000000000000000..6c6decfab18c1f6b7c6f35d68211f8a2893e9b85 GIT binary patch literal 9728 zcmeI2TWB0r7{|ZaY?9sF_kPXpW^=z#a&6NDM_6Pl!w#s__|TJtCf zJ}JHwYPDX{)=Mw8g5rx9I5lh2oqgXJ&tMesj)z zGjrz5$$ow*w07iFdYz24ld9;ia*%2i?(ll3^1X$4d%29Sl_eu$J>EwVG-= z-}qTA&Lz(GI{E0N|0B+czU99M=`q@UV6jxVf$3Ih!?c4o?BLv?mzgwCN?H|`{1M3` zlAn;07bKq&@|@&5C`@b4{d8$^ci%Ygi+E4>`HHfwiF1d}i1W3OA&Su_T&Dcg^SIb~ zYc<`nG&eo(ZSX?kT@3%?HpuxK=qPd5va7UMTAH5E`NQ-IH4+ULXC9xMU!GgsJG`w3 zdd#5XLSHrLcA>B7bYxKIaf1#CechnLLf|Q?p%NKXf?3vsv)I3BSh?e{lzCRRc5yu;lD?emR@QrXedXu7&gNFWo9yNF)m`gPMpwU@ zJi=FgkX!G_=P(U81I|k?BAuJ8EFB(6qyBFscgNxC2t44!`ycUpqr4RIrP53vu!9t$ zpFZ|+ev#K5s*+>H_aBcv@at{)AEw|UCZ-D(7B{T84!0nzxoDT^f$W*>_#;gXV7%PV&cXW*UxTDbE_>bSX|t=;^IulH5YGbX1!nuaTAJL!)`)z z@tSA4U`cV4id)NWQgd-Hm@ZgK+?3)5*iC6}(Bgun#Z4=2klnQA;?6Vc1u6O;${`Mp53hG@+jyk7pzs>R>j5L->SJh#=Xh~%ZZy)T=X9~%|%~ewizrh zZeDTm4x889W{V3}5VxSX=&K5vi#yn?7pzU(HpOjWw@q^~Gi178MRC#hR_=ZDjz!Hy zFJZc1?c%m8F8ZK$&5c-GunuuM6c;lE9hw`pxL}>)b}DX^-A>JoSzNF#ak~^Z#%`D9 z#w{*bx47Mk8)vs$a}yR9tVi4)#Z9o=qq#|o3)U-cui_@z?bY0r#RcmVw@-2LZ0OTm zj2z7K3)U}gzvANQ)~~r4iwia&?ttRr`8aUJ-RTR{X*>jI3fV?Nui+&x0$v2X2zU|j zBH%^9i+~paF9Kc!ya@asBY+t;clL}3Ggp{_!b}oogfO#%85+DZVa5bAADF?wOax{W z@Vk#c`43Ak-JB7)n-1`2ndgCSr$t^a@&DmAlj2nERE`|i()Fau?}bqIUaozK_S1~C oWB>K&S{a@Xx>7&dp>4d@m%_C+Zfe`6>sOC|Q#IKO|EK!@2I#H@qW}N^ literal 0 HcmV?d00001 diff --git a/crates/utopia-ingest/tests/spreadsheet_dates.rs b/crates/utopia-ingest/tests/spreadsheet_dates.rs index ec7cd2ae2..ef23c5766 100644 --- a/crates/utopia-ingest/tests/spreadsheet_dates.rs +++ b/crates/utopia-ingest/tests/spreadsheet_dates.rs @@ -11,7 +11,7 @@ use std::io::{Cursor, Write}; // 3 date and time (22), 4 time of day (20), 5 elapsed time (46), 6 "#,##0.00" (4). const STYLES: &str = r#""#; -fn workbook(rows: &[Vec], date1904: bool) -> Vec { +fn workbook(rows: &[Vec], date1904: bool, styles: &str) -> Vec { let sheet_data: String = rows .iter() .enumerate() @@ -28,7 +28,7 @@ fn workbook(rows: &[Vec], date1904: bool) -> Vec { ("_rels/.rels", r#""#.to_string()), ("xl/workbook.xml", format!(r#"{workbook_pr}"#)), ("xl/_rels/workbook.xml.rels", r#""#.to_string()), - ("xl/styles.xml", STYLES.to_string()), + ("xl/styles.xml", styles.to_string()), ("xl/worksheets/sheet1.xml", format!(r#"{sheet_data}"#)), ] { zip.start_file(part, zip::write::SimpleFileOptions::default()).unwrap(); @@ -38,7 +38,7 @@ fn workbook(rows: &[Vec], date1904: bool) -> Vec { } fn read(rows: &[Vec], date1904: bool) -> String { - utopia_ingest::parse("ledger.xlsx", &workbook(rows, date1904)) + utopia_ingest::parse("ledger.xlsx", &workbook(rows, date1904, STYLES)) .unwrap() .text } @@ -141,3 +141,94 @@ fn a_workbook_counting_from_1904_reads_the_same_date() { "{text}" ); } + +fn read_format(format_id: u16, value: &str, date1904: bool, format_code: Option<&str>) -> String { + let custom = format_code.map_or_else(String::new, |code| { + let code = quick_xml::escape::escape(code); + format!( + r#""# + ) + }); + let styles = format!( + r#"{custom}"# + ); + let rows = [ + header(1), + vec![ + words("A2", "Value"), + number("B2", 0, value), + words("C2", "ok"), + ], + ]; + utopia_ingest::parse("ledger.xlsx", &workbook(&rows, date1904, &styles)) + .unwrap() + .text +} + +#[test] +fn east_asian_builtin_dates_need_no_custom_format_definition() { + for format_id in [27, 31, 50, 58] { + for (date1904, serial) in [(false, "45306"), (true, "43844")] { + let text = read_format(format_id, serial, date1904, None); + assert!( + text.contains("| Value | 2024-01-15 | ok |"), + "format {format_id}: {text}" + ); + } + } +} + +#[test] +fn east_asian_fractional_clock_times_and_explicit_formats_are_preserved() { + // calamine does not retain clock-only metadata; a serial >= 1 still gains a date. + for format_id in [32, 33] { + let text = read_format(format_id, "0.5", false, None); + assert!( + text.contains("| Value | 12:00 | ok |"), + "format {format_id}: {text}" + ); + } + for (format_id, code, serial, expected) in [ + (27, "0.00", "45306", "45306"), + (32, "yyyy-mm-dd", "45306", "2024-01-15"), + (27, "[h]:mm:ss", "1.5104166666666667", "36:15:00"), + ] { + let text = read_format(format_id, serial, false, Some(code)); + assert!( + text.contains(&format!("| Value | {expected} | ok |")), + "{text}" + ); + } +} + +#[test] +fn east_asian_builtin_formats_are_normalized_in_xls() { + // Fixture from calamine PR #734, commit d93c5a463399a5554d287ebda0ed1d2a43377ec2, + // tests/date_east_asian.xls (MIT; license alongside). Rows 0..19 are IDs 27..36/50..58; + // each row stores 45306, 0.5, 45306.123456789 with no explicit format definition. + let text = utopia_ingest::parse( + "ledger.xls", + include_bytes!("fixtures/calamine/date_east_asian.xls"), + ) + .unwrap() + .text; + let rows: Vec<_> = text + .lines() + .filter(|line| line.starts_with("| ") && !line.starts_with("| ---")) + .collect(); + assert_eq!(rows.len(), 28, "{text}"); + for row in [0, 4, 10, 18] { + // formats 27, 31, 50, 58 + assert!(rows[row].starts_with("| 2024-01-15 |"), "row {row}: {text}"); + } + for row in [5, 6] { + // formats 32, 33 + assert_eq!( + rows[row].split('|').nth(2).unwrap().trim(), + "12:00", + "{text}" + ); + } + assert!(rows[19].starts_with("| 45306 | 0.5 |"), "{text}"); // General + assert!(rows[24].starts_with("| 2024-01-15 |"), "{text}"); // Chinese custom date +} From 70063f1bcfd7404559423d992f46875da1344895 Mon Sep 17 00:00:00 2001 From: Szabo Bogdan Date: Mon, 19 Oct 2026 07:19:14 +0200 Subject: [PATCH 13/19] ADR 0070: a role word speaks while its chunk is live (#1278) * Propose that a statement's role word follows its newest live evidence instead of the first value written, as of reads included Signed-off-by: Bogdan Szabo * Narrow 0070 to supersession, back-fill role words onto every evidence chunk, give a role word its chunk's record time, and point 0037 at it Signed-off-by: Bogdan Szabo * Rename 0070 to say that a role word speaks while its chunk is live, now that no ordering decides between live rows Signed-off-by: Bogdan Szabo --------- Signed-off-by: Bogdan Szabo Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- ...7-a-relation-carries-its-own-attributes.md | 5 ++ ...ay-push-statements-in-the-open-contract.md | 1 + ...ole-word-speaks-while-its-chunk-is-live.md | 65 +++++++++++++++++++ docs/decisions/README.md | 1 + 4 files changed, 72 insertions(+) create mode 100644 docs/decisions/0070-a-role-word-speaks-while-its-chunk-is-live.md diff --git a/docs/decisions/0037-a-relation-carries-its-own-attributes.md b/docs/decisions/0037-a-relation-carries-its-own-attributes.md index 528176c21..3ccc5af5b 100644 --- a/docs/decisions/0037-a-relation-carries-its-own-attributes.md +++ b/docs/decisions/0037-a-relation-carries-its-own-attributes.md @@ -83,6 +83,11 @@ instead of a class. answer to two observations that disagree is two rows and a `fact_conflicts` entry; wiring that requires a write path that can bypass dedup on purpose, and is the next cut. + 2026-10-18: role words on open statements (`statement_qualifiers`) no longer keep the first + value. Each is recorded against its evidence chunk and speaks while that chunk is live; see + [0070](0070-a-role-word-speaks-while-its-chunk-is-live.md). The typed `fact_qualifiers` + above keep this rule. + 4. **An entity-valued qualifier is reserved, not built.** "A invested in B *through* C" needs a qualifier that points at a node. The table has the column and the `CHECK` that says one of value or entity, so the second migration is never needed; nothing writes it yet, and the diff --git a/docs/decisions/0054-a-source-may-push-statements-in-the-open-contract.md b/docs/decisions/0054-a-source-may-push-statements-in-the-open-contract.md index 3965eb342..d1c15b3ba 100644 --- a/docs/decisions/0054-a-source-may-push-statements-in-the-open-contract.md +++ b/docs/decisions/0054-a-source-may-push-statements-in-the-open-contract.md @@ -93,3 +93,4 @@ Content-Type: application/json - 2026-09-25 (#899, #900): typed materialization now reconciles the rows it writes along their uniqueness timelines, as the write path always did, so a later observation of a functional attribute closes the earlier one without a manual reconcile. A document version records the `doc_time` it was pushed with, and a fact's evidence date is taken from its own version, so a same-identity update no longer makes the earlier statement look simultaneous with the later one. The stored document carries the observation's identity and date ahead of the three arrays, so the same payload under a new identity is a second document, never a rename of the first. - 2026-10-15 ([#1242](https://github.com/deeplethe/utopia/issues/1242)): a pushed signature whose phrase equals exactly one declared relation, not a state, that admits both of its ends is bound by the ontology when the document's extraction finishes, and that extraction queues `materialize`, so a base with no chat model types what it is pushed. It binds the signature as [0044](0044-the-ontology-is-a-view-over-what-documents-say.md)'s revision of decision 3 describes, reviewable like any binding; it is not a typed write, decisions 3 and 5 stand, and no other pushed phrase binds before alignment reads it. +- 2026-10-16 ([#1267](https://github.com/deeplethe/utopia/issues/1267)): decision 7 keeps a re-pushed statement's identity, but its role words stayed first-write-wins, so a re-push with a new qualifier value kept the old one; [0070](0070-a-role-word-speaks-while-its-chunk-is-live.md) proposes that a role word is recorded against its evidence chunk and speaks while that chunk is live. diff --git a/docs/decisions/0070-a-role-word-speaks-while-its-chunk-is-live.md b/docs/decisions/0070-a-role-word-speaks-while-its-chunk-is-live.md new file mode 100644 index 000000000..7cd32a6ac --- /dev/null +++ b/docs/decisions/0070-a-role-word-speaks-while-its-chunk-is-live.md @@ -0,0 +1,65 @@ +# 0070 · A role word speaks while its chunk is live + +- **Status**: Proposed · 2026-10-16 · [#1267](https://github.com/deeplethe/utopia/issues/1267) · open: all five decisions +- **Written**: 2026-10-16 (conventions in the [README](README.md)) +- **Related**: [0037](0037-a-relation-carries-its-own-attributes.md) decided that a qualifier is not part of a fact's identity and that the first value stays; [0044](0044-the-ontology-is-a-view-over-what-documents-say.md) owns the open statement and the role words on it; [0054](0054-a-source-may-push-statements-in-the-open-contract.md) decision 7 is the re-push that supersedes a chunk; migration `0061` created `statement_qualifiers`; [0048](0048-provenance-references-stay-inside-the-knowledge-base.md) gets one more edge to keep inside the base; [0019](0019-the-second-clock-can-be-rewound.md) is the record axis `as_of` reads on; [#1267](https://github.com/deeplethe/utopia/issues/1267), [#1272](https://github.com/deeplethe/utopia/issues/1272) + +> A CI job pushes a test as a statement: `test-parses-a-date covers parse.rs`, qualifier `lines: "40-52"`. A week later lines are added above the test and the job pushes the same `external_id` with `lines: "58-70"`. The push is accepted, a new version is recorded, the chunk is replaced. The statement still says `40-52`, and will say it forever. The same goes for a `status` that went from `failing` to `passing`, or a `mood` that a later observation drops. + +## The problem + +`add_statement_qualifier` writes `ON CONFLICT (fact_id, role) DO NOTHING`. That was 0037's rule for typed qualifiers, carried over to the open layer by migration 0061: two observations that disagree should become two rows and a conflict, and until that cut exists, the first value stays. For a document that is mentioned once, the rule is invisible. For a source that pushes the same observation again with a corrected value (0054), it is wrong every time. + +Three facts of the code make it so: + +- **The statement survives the re-push.** `insert_open_statement` dedups open statements on `(kb, subject, phrase, object | value)` among live rows. A re-push with the same subject, phrase and object finds the existing statement and only adds evidence on the new chunk. The new role word arrives on a fact that already has one, and is dropped. +- **The old chunk is superseded, not deleted.** 0054 decision 7: the earlier chunk is marked `superseded_at`, and a statement whose evidence is only on superseded chunks is stale, handed to review, not invalidated. The statement's identity is right to survive; its role words are not right to stay frozen. +- **Typed rows copy once.** `materialize` copies a statement's evidence and qualifiers onto a typed row only when it creates the row. A typed row made from the first push keeps the first push's role words, and its only evidence is the superseded chunk, so every read marks it stale although the statement it came from was just confirmed. The three `q.role = 'mood'` checks look at `statement_qualifiers` as a whole, so a `mood` written once can never be taken back. + +## Decisions + +**1. One qualifier row per evidence chunk.** + +`statement_qualifiers` gains `chunk_id`, a reference to `chunks`, `NOT NULL`, and its key becomes `(fact_id, role, chunk_id)`. Each passage that states a role word records it against itself, the way `fact_evidence` already records one row per chunk. Writes stay append-only and idempotent: the same push twice writes nothing new, a new push writes rows for its new chunk and touches no old row. Every open statement has evidence, and a typed or implied row gets its statements' evidence with its role words (decision 4), so there is no row without a chunk to allow for. + +The backfill copies each existing `(fact, role)` value onto every evidence chunk of its fact, superseded ones included. Attaching it only to the earliest chunk would hide role words that show today: for a statement already re-pushed under 0054 decision 7, the earliest chunk is superseded, the live chunk carries no row because `DO NOTHING` dropped it, and under decision 2 the role would disappear. With the value on every chunk, every read after the migration returns exactly what it returned before; only the pushes that follow can change a value. + +`chunk_id` is a new provenance edge, and 0048's invariant covers it: the owner-derived trigger `statement_qualifiers_same_kb` checks that the chunk's document is in the fact's base, and the export preflight scans the edge. + +**2. A superseded chunk stops speaking for the statement.** + +The rows that speak for `(fact, role)` are the rows whose chunk is live at the read's `as_of` (`chunk_live_at`, and `document_live_at` for the chunk's document). For a stale fact, one with no live evidence at all, they are the rows on its evidence chunks that were superseded last, so a fact waiting in review still shows what it last said. This is one SQL fragment beside the `record_axis` helpers, and every reader of role words uses it: the entity panel, the tools, the export and `materialize`. + +There is no ordering among live rows. Two live documents that give different values for one role both speak, and a read returns each value with the chunk it came from, the way it returns two evidence passages. This is 0037 decision 3's answer, that two observations that disagree stay two and neither hides the other, applied to role words on open statements. Flagging such a pair as contested is left open below. + +What follows: + +- A re-push that changes a value supersedes the old chunk, so only the new value speaks. One that drops a role leaves no live row for it, and the role disappears from the read. The old rows stay in the table, as everything in the ledger does. +- Deleting a document supersedes its chunks; restoring it brings them back. The rule reads chunk and document liveness, so neither needs a recomputation. + +**3. A read in the past shows what was said then, on the chunk's clock.** + +A role word has no record time of its own. It arrives as a row on its evidence chunk, and its record time is that chunk's: it starts speaking when the chunk becomes live and stops when the chunk is superseded. An `as_of` read applies decision 2 at that instant, so a role word that changed shows the value its live evidence carried at `as_of`. In the example above, a read at a moment between the two pushes says `lines: "40-52"`; a read today says `58-70`. This is what `chunk_live_at` already does for evidence, where a passage replaced today is still live evidence on a graph read as of March, and a role word is part of that evidence. + +The copies `materialize` makes onto typed and implied rows (decision 4) keep the `chunk_id` of the row they copy, so they take that chunk's liveness too. A copy is written on the first run after its chunk arrives, and an `as_of` read between the chunk's arrival and that run shows it on the typed row a little before it was made. The read answers what the evidence said at that moment, not when the copy was written. + +**4. `materialize` keeps typed and implied rows in step.** + +On each run, scoped to the base it runs for, `materialize` copies onto every live typed row the evidence and qualifier rows of its source statements (`typed_fact_sources`) that the row does not have yet, and does the same for every live implied row from the statement that triggered it (`implied_fact_sources`). The copy is the same insert it does today at creation, `ON CONFLICT DO NOTHING`, now run for existing rows too. A typed or implied row then reads its role words through the same rule as its statements, and is stale exactly when they are. + +The three `mood` checks use the rule instead of any row: a statement has a mood when a row that speaks for `(fact, 'mood')` exists. A re-push that adds `mood` retires the typed row on the next run; one that removes it lets the row be created. + +**5. Typed qualifiers keep 0037's rule.** + +This record covers role words, the `statement_qualifiers` rows of the open layer and the copies `materialize` makes of them. The typed `fact_qualifiers` of 0037, one value per declared attribute per edge, keep that record's rule: a second mention with a different value does not overwrite the first, and is reported as `qualifier_conflict`. + +## Dead ends + +- **Last write wins on `(fact, role)`.** An `ON CONFLICT ... DO UPDATE` is one line. It overwrites an observation, which the ledger never does. It makes the value depend on the order of writes, so deleting the newer document leaves the newer value in place unless something recomputes it from what is left, and restoring needs the same. And an `as_of` read could only show today's value for any moment. +- **Qualifiers in the dedup key.** Each value would make its own statement: `covers` with `40-52` and `covers` with `58-70` as two facts. That splits one observation's identity on every change, and it contradicts 0037 decision 3, which keeps a qualifier out of identity so that a second mention adds to the same row. +- **Role words only on the chunk text.** The payload is stored verbatim as the chunk (0054 decision 2), so the value can be parsed back from it. Every reader would then parse JSON or prose to answer a question the table answers, and a document's chunk has no structure to parse. + +## Open questions + +- **Contested role words.** Two live documents that disagree on one role both speak (decision 2). Flagging such a pair the way the export flags a contested statement (0062) would make the disagreement easier to find; this record does not decide it. +- **The cost of the re-sync.** Copying evidence and role words onto every live typed and implied row on each `materialize` run is a few set-based inserts per base. It has not been measured on a large base. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 3d6c51ebc..f4ef89cb4 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -73,6 +73,7 @@ By domain; the domains are the files of [../design/](../design/README.md). **Sta | 0010 | [An unnamed relation stays empty](0010-no-relation-is-no-relation.md) | Implemented | partly (by 0044) | | 0037 | [A relation carries its own attributes](0037-a-relation-carries-its-own-attributes.md) | In progress | partly (by 0044) | | 0048 | [Provenance references stay inside the knowledge base](0048-provenance-references-stay-inside-the-knowledge-base.md) | Implemented | | +| 0070 | [A role word speaks while its chunk is live](0070-a-role-word-speaks-while-its-chunk-is-live.md) | Proposed | | ### [governance](../design/governance.md) From e3e5cdedcc3e7d69c293c943f55b575a988957e0 Mon Sep 17 00:00:00 2001 From: Szabo Bogdan Date: Mon, 19 Oct 2026 07:24:43 +0200 Subject: [PATCH 14/19] Knowledge bases named owner/name are grouped under their owner in the base switcher, the chat scope chip, the token picker and My knowledge bases (#1250) Signed-off-by: Bogdan Szabo Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- web/src/pages/Chat.tsx | 35 ++-- web/src/pages/KbMenuList.tsx | 263 +++++++++++++++++++++++++++ web/src/pages/KbSwitcher.tsx | 28 ++- web/src/pages/MyKbs.tsx | 32 +++- web/src/pages/Tokens.tsx | 3 +- web/src/pages/kbGroups.test.ts | 66 +++++++ web/src/pages/kbGroups.ts | 51 ++++++ web/src/pages/kbMenuKeys.test.ts | 20 ++ web/src/pages/kbMenuKeys.ts | 17 ++ web/src/ui/index.tsx | 30 ++- web/src/ui/withGroupHeadings.test.ts | 15 ++ web/src/ui/withGroupHeadings.ts | 15 ++ 12 files changed, 527 insertions(+), 48 deletions(-) create mode 100644 web/src/pages/KbMenuList.tsx create mode 100644 web/src/pages/kbGroups.test.ts create mode 100644 web/src/pages/kbGroups.ts create mode 100644 web/src/pages/kbMenuKeys.test.ts create mode 100644 web/src/pages/kbMenuKeys.ts create mode 100644 web/src/ui/withGroupHeadings.test.ts create mode 100644 web/src/ui/withGroupHeadings.ts diff --git a/web/src/pages/Chat.tsx b/web/src/pages/Chat.tsx index 9e74be295..5abe02f1e 100644 --- a/web/src/pages/Chat.tsx +++ b/web/src/pages/Chat.tsx @@ -11,7 +11,6 @@ import remend from "remend"; import { ArrowUp, BookOpen, - Check, ChevronDown, ChevronRight, Database, @@ -75,6 +74,7 @@ import { import { NodCard } from "./PendingFacts"; import { stepDetail, stepLabel } from "../steps"; import { NextStep, nextStep, useReadiness } from "./NextStep"; +import { KbMenuList } from "./KbMenuList"; import { DRAFT_KEY, fetchMe, lastKey } from "../authSession"; /* `Turn` 定义在 liveAnswer 里：进行中的那一次也是一串 Turn， @@ -220,7 +220,10 @@ export function Chat() { useEffect(() => { if (!scopeOpen) return; const onDoc = (e: MouseEvent) => { - if (!scopeRef.current?.contains(e.target as Node)) setScopeOpen(false); + const target = e.target as Element; + // 组的子菜单挂在 body 上，不在 scopeRef 里——点它不算点外面（KbMenuList） + if (!scopeRef.current?.contains(target) && !target.closest?.("[data-kb-submenu]")) + setScopeOpen(false); }; const onKey = (e: KeyboardEvent) => { if (e.key === "Escape") setScopeOpen(false); @@ -606,26 +609,14 @@ export function Chat() { {S.ask.scopeLabel}
- {kbs.map((k) => ( - : undefined - } - onClick={() => { - setScopeOpen(false); - if (k.id !== kb?.id) { claimView(null); setKb(k.id); } - }} - > - {k.name} - - ))} + { + setScopeOpen(false); + if (id !== kb?.id) { claimView(null); setKb(id); } + }} + />
)} diff --git a/web/src/pages/KbMenuList.tsx b/web/src/pages/KbMenuList.tsx new file mode 100644 index 000000000..12e017d6d --- /dev/null +++ b/web/src/pages/KbMenuList.tsx @@ -0,0 +1,263 @@ +/* 库菜单的那一列：顶栏切换器与提问框的作用域芯片共用。名字带 `/` 的库收进 + 一个组（`re-cinq ›`），指针停上去或点一下，右边弹出组里的库（#1249）。 + + 子菜单走 Popover 的 Portal，挂在 body 上——所以它**不在**宿主菜单的 DOM 里。 + 宿主若是自己判「点在外面就关」，要放过带 `data-kb-submenu` 的元素，否则 + mousedown 先把整个菜单关掉，点击落空。 + + 键盘：组那一行上 Enter / 空格 / → 打开并把焦点送进第一项，↑↓ 在子菜单里走， + Esc / ← 关上并把焦点还给那一行（kbMenuKeys）。没用 DropdownMenuSub：它只能 + 长在 DropdownMenu 里，而两个宿主一个是带查找框的 Popover（菜单的 typeahead + 与焦点管理会跟查找框抢键），一个是手写的 div。 */ +import { useEffect, useRef, useState, useSyncExternalStore, type KeyboardEvent } from "react"; +import { Check, ChevronRight } from "lucide-react"; +import type { Kb } from "../api"; +import { Row, cn } from "../ui"; +import { Popover, PopoverAnchor, PopoverContent } from "@/components/ui/popover"; +import { kbGroups, type KbGroupItem } from "./kbGroups"; +import { groupRowKey, submenuKey } from "./kbMenuKeys"; + +const ROW = "gap-3 px-4 py-2 text-body"; +/** 指针从组那一行移到子菜单要经过一道缝，关得太快就够不着 */ +const CLOSE_DELAY_MS = 150; + +const NARROW = "(max-width: 639px)"; + +/** 屏幕窄到放不下侧边子菜单（手机竖屏） */ +function useNarrow() { + return useSyncExternalStore( + (changed) => { + const query = window.matchMedia(NARROW); + query.addEventListener("change", changed); + return () => query.removeEventListener("change", changed); + }, + () => window.matchMedia(NARROW).matches, + ); +} + +export function KbMenuList({ + kbs, + currentId, + onPick, + grouped = true, +}: { + kbs: Kb[]; + currentId: string | undefined; + onPick: (id: string) => void; + /** 搜索时传 false：按全名平铺，`re-cinq/lo` 照样搜得到 */ + grouped?: boolean; +}) { + if (!grouped) + return kbs.map((k) => ( + onPick(k.id)} /> + )); + return kbGroups(kbs).map((entry) => + entry.kind === "kb" ? ( + onPick(entry.kb.id)} + /> + ) : ( + + ), + ); +} + +function KbRow({ + label, + current, + onClick, + indent, +}: { + label: string; + current: boolean; + onClick: () => void; + indent?: number; +}) { + return ( + /* 对勾走 `trailing` 槽。**塞进 children 的 svg 会掉到第二行**： + Row 把 children 整个包进一个 `flex-1 truncate` 的 span， + 那个 span 不是 flex 容器，preflight 又把 svg 设成块级， + 于是勾自己占一行、整行跟着变高 */ + : undefined} + onClick={onClick} + > + {label} + + ); +} + +function KbGroupRow({ + name, + items, + currentId, + onPick, +}: { + name: string; + items: KbGroupItem[]; + currentId: string | undefined; + onPick: (id: string) => void; +}) { + const [open, setOpen] = useState(false); + const narrow = useNarrow(); + const closing = useRef | undefined>(undefined); + const anchor = useRef(null); + const submenu = useRef(null); + /** 键盘打开的才把焦点送进子菜单；指针停上去的不抢（顶栏的查找框还在打字） */ + const focusInside = useRef(false); + const show = (withFocus: boolean) => { + clearTimeout(closing.current); + focusInside.current = withFocus; + setOpen(true); + }; + const leave = () => { + clearTimeout(closing.current); + closing.current = setTimeout(() => setOpen(false), CLOSE_DELAY_MS); + }; + /* 只有鼠标按悬停开合。触屏的一点会先模拟一次 enter 再来 click—— + 若 click 是开关，就刚开就关 */ + const byMouse = (run: () => void) => (e: { pointerType: string }) => { + if (e.pointerType === "mouse") run(); + }; + const onRowKey = (e: KeyboardEvent) => { + if (groupRowKey(e.key) !== "open") return; + e.preventDefault(); + // 指针已经把它打开了：onOpenAutoFocus 不会再来，焦点自己送进去 + if (open) submenu.current?.querySelector("button")?.focus(); + else show(true); + }; + useEffect(() => { + if (!narrow || !open || !focusInside.current) return; + submenu.current?.querySelector("button")?.focus(); + focusInside.current = false; + }, [narrow, open]); + const onSubmenuKey = (e: KeyboardEvent) => { + const action = submenuKey(e.key); + if (action === undefined) return; + e.preventDefault(); + // 别让提问框那个宿主的 Esc（挂在 document 上）把整个菜单也关了 + e.stopPropagation(); + if (action === "close") { + setOpen(false); + anchor.current?.querySelector("button")?.focus(); + return; + } + const buttons = [...e.currentTarget.querySelectorAll("button")]; + const at = buttons.indexOf(document.activeElement as HTMLButtonElement); + buttons[(at + action + buttons.length) % buttons.length]?.focus(); + }; + const holdsCurrent = items.some((i) => i.kb.id === currentId); + const trailing = ( + + {holdsCurrent && } + + + ); + /* 窄屏（手机）上菜单几乎占满宽度，右边左边都放不下子菜单——Radix 只好把它 + 推到屏幕外。这里改成就地展开：组下面缩进一档列出它的库 */ + if (narrow) + return ( + <> +
+ setOpen((v) => !v)} + onKeyDown={onRowKey} + > + {name} + +
+ {open && +
+ {items.map((item) => ( + { + setOpen(false); + onPick(item.kb.id); + }} + /> + ))} +
+ } + + ); + return ( + + +
show(false))} onPointerLeave={byMouse(leave)}> + show(false)} + onKeyDown={onRowKey} + > + {name} + +
+ + { + if (!focusInside.current) e.preventDefault(); + }} + // 焦点在子菜单里时 Esc 交给 onSubmenuKey（它还要把焦点还给那一行）； + // 焦点还在搜索框里时照常关掉 + onEscapeKeyDown={(e) => { + if (submenu.current?.contains(document.activeElement)) e.preventDefault(); + }} + // 按在组那一行上不算外面：否则按下就关、松开的 click 又开，闪一下 + onInteractOutside={(e) => { + if (anchor.current?.contains(e.target as Node)) e.preventDefault(); + }} + onKeyDown={onSubmenuKey} + onPointerEnter={byMouse(() => show(false))} + onPointerLeave={byMouse(leave)} + className="w-max min-w-48 max-w-80 overflow-hidden p-0" + > +
+ {items.map((item) => ( + { + setOpen(false); + onPick(item.kb.id); + }} + /> + ))} +
+ + + ); +} diff --git a/web/src/pages/KbSwitcher.tsx b/web/src/pages/KbSwitcher.tsx index 6c0ce40d8..9ce534f62 100644 --- a/web/src/pages/KbSwitcher.tsx +++ b/web/src/pages/KbSwitcher.tsx @@ -9,7 +9,7 @@ import { useState } from "react"; import { useQuery } from "@tanstack/react-query"; import { useNavigate } from "@tanstack/react-router"; -import { Check, ChevronDown, Layers, Plus } from "lucide-react"; +import { ChevronDown, Layers, Plus } from "lucide-react"; import { api, type Kb } from "../api"; import { S } from "../i18n"; import { Button, Input, Row } from "../ui"; @@ -18,6 +18,7 @@ import { PopoverContent, PopoverTrigger, } from "@/components/ui/popover"; +import { KbMenuList } from "./KbMenuList"; import { fetchMe } from "../authSession"; export function KbSwitcher({ @@ -92,22 +93,15 @@ export function KbSwitcher({ />
- {shown.map((k) => ( - : undefined - } - onClick={() => { - dismiss(); - if (k.id !== kb?.id) onChange(k.id); - }} - > - {k.name} - - ))} + { + dismiss(); + if (id !== kb?.id) onChange(id); + }} + /> {shown.length === 0 && (

{S.nav.noKbMatch}

)} diff --git a/web/src/pages/MyKbs.tsx b/web/src/pages/MyKbs.tsx index 7e57bd7c8..0eb321967 100644 --- a/web/src/pages/MyKbs.tsx +++ b/web/src/pages/MyKbs.tsx @@ -9,6 +9,7 @@ import { Lock, Plus, Search } from "lucide-react"; import { api, type MyKb } from "../api"; import { S } from "../i18n"; import { useKb } from "../kb"; +import { kbGroups } from "./kbGroups"; import { Button, Checkbox, @@ -75,6 +76,18 @@ export function MyKbs() { r.kb.name.toLowerCase().includes(q) || (r.kb.description ?? "").toLowerCase().includes(q), ); + // 不筛选时按 `owner/` 归组：组名占一行，下面是短名（#1249）；筛选时按全名平铺 + const rowById = new Map(rows.map((r) => [r.kb.id, r])); + const lines: ({ heading: string } | { row: (typeof rows)[number]; label: string })[] = q + ? rows.map((row) => ({ row, label: row.kb.name })) + : kbGroups(rows.map((r) => r.kb)).flatMap((entry) => + entry.kind === "kb" + ? [{ row: rowById.get(entry.kb.id)!, label: entry.kb.name }] + : [ + { heading: entry.name }, + ...entry.items.map((item) => ({ row: rowById.get(item.kb.id)!, label: item.label })), + ], + ); const openKb = (id: string) => { setKb(id); navigate({ to: "/kb/$kbId/graph", params: { kbId: id } }); @@ -128,15 +141,28 @@ export function MyKbs() { - {rows.map((row) => { + {lines.map((line) => { + if ("heading" in line) + return ( + + + {line.heading} + + + ); + const { row, label } = line; const canManage = row.my_role === "admin" || row.my_role === "owner"; return (
- - {row.kb.name} + + {label} {/* 这两个是**库自己的性质**，不是一列数据：缺省库只有一个， 受限的也少，单开一列会空掉大半，所以贴在名字旁边 */} diff --git a/web/src/pages/Tokens.tsx b/web/src/pages/Tokens.tsx index 72eec8528..e0d05e161 100644 --- a/web/src/pages/Tokens.tsx +++ b/web/src/pages/Tokens.tsx @@ -32,6 +32,7 @@ import { } from "../ui"; import { toast } from "../toast"; import { copyAndSay } from "../clipboard"; +import { kbOptions } from "./kbGroups"; const EXPIRY_CHOICES = [30, 90, 365, 0] as const; @@ -373,7 +374,7 @@ export function Tokens() { ({ value: k.id, label: k.name }))} + options={kbOptions(kbs)} placeholder={S.account.pickBases} emptyHint={S.account.allBases} onToggle={(id) => { diff --git a/web/src/pages/kbGroups.test.ts b/web/src/pages/kbGroups.test.ts new file mode 100644 index 000000000..aaf322e8a --- /dev/null +++ b/web/src/pages/kbGroups.test.ts @@ -0,0 +1,66 @@ +import { expect, it } from "vitest"; +import type { Kb } from "../api"; +import { kbGroups, kbOptions } from "./kbGroups"; + +const kb = (name: string) => ({ id: name, name }) as Kb; + +it("keeps bases without a slash as plain entries", () => { + const mercedes = kb("mercedes"); + expect(kbGroups([mercedes])).toEqual([{ kind: "kb", kb: mercedes }]); +}); + +it("groups bases sharing the text before the slash under that name", () => { + const [lore, floor] = [kb("re-cinq/lore"), kb("re-cinq/floor")]; + expect(kbGroups([lore, floor])).toEqual([ + { + kind: "group", + name: "re-cinq", + items: [ + { kb: lore, label: "lore" }, + { kb: floor, label: "floor" }, + ], + }, + ]); +}); + +it("places a group where its first member was and splits only on the first slash", () => { + const [a, xy, b, xyz] = [kb("alpha"), kb("x/y"), kb("beta"), kb("x/y/z")]; + expect(kbGroups([a, xy, b, xyz])).toEqual([ + { kind: "kb", kb: a }, + { kind: "group", name: "x", items: [{ kb: xy, label: "y" }, { kb: xyz, label: "y/z" }] }, + { kind: "kb", kb: b }, + ]); +}); + +it("keeps CI/CD a plain row when no other base shares the CI prefix", () => { + const [cicd, mercedes] = [kb("CI/CD"), kb("mercedes")]; + expect(kbGroups([cicd, mercedes])).toEqual([ + { kind: "kb", kb: cicd }, + { kind: "kb", kb: mercedes }, + ]); +}); + +it("keeps /foo, foo/ and / plain rows because one side of the slash is empty", () => { + const names = ["/foo", "/bar", "foo/", "foo/ ", "/", " /x", " /y"].map(kb); + expect(kbGroups(names)).toEqual(names.map((k) => ({ kind: "kb", kb: k }))); +}); + +it("puts re-cinq /lore and re-cinq/floor in one re-cinq group with trimmed labels", () => { + const [lore, floor] = [kb("re-cinq /lore"), kb("re-cinq/ floor")]; + expect(kbGroups([lore, floor])).toEqual([ + { kind: "group", name: "re-cinq", items: [{ kb: lore, label: "lore" }, { kb: floor, label: "floor" }] }, + ]); +}); + +it("gives a lone A/B tests base a plain option with its full name", () => { + expect(kbOptions([kb("A/B tests")])).toEqual([{ value: "A/B tests", label: "A/B tests" }]); +}); + +it("turns bases into select options with the group's members side by side", () => { + const [lore, mercedes, floor] = [kb("re-cinq/lore"), kb("mercedes"), kb("re-cinq/floor")]; + expect(kbOptions([lore, mercedes, floor])).toEqual([ + { value: "re-cinq/lore", label: "re-cinq/lore", group: "re-cinq", short: "lore" }, + { value: "re-cinq/floor", label: "re-cinq/floor", group: "re-cinq", short: "floor" }, + { value: "mercedes", label: "mercedes" }, + ]); +}); diff --git a/web/src/pages/kbGroups.ts b/web/src/pages/kbGroups.ts new file mode 100644 index 000000000..d33790c1e --- /dev/null +++ b/web/src/pages/kbGroups.ts @@ -0,0 +1,51 @@ +import type { Kb } from "../api"; + +export type KbGroupItem = { kb: Kb; label: string }; +export type KbEntry = + | { kind: "kb"; kb: Kb } + | { kind: "group"; name: string; items: KbGroupItem[] }; + +/** 名字带 `/` 的库按第一个 `/` 之前的部分归组（`re-cinq/lore` → 组 `re-cinq`、 + * 标签 `lore`）。两边去掉空白后都得有字，且同组至少两个库才成组——`/foo`、 + * `foo/`、只此一家的 `CI/CD` 照旧是一行全名。组落在它第一个成员原来的位置， + * 其余库的顺序不动 */ +export function kbGroups(kbs: Kb[]): KbEntry[] { + const groups = new Map(); + const owners = kbs.map((kb) => { + const split = ownerAndLabel(kb.name); + if (!split) return undefined; + groups.set(split.owner, [...(groups.get(split.owner) ?? []), { kb, label: split.label }]); + return split.owner; + }); + const placed = new Set(); + return kbs.flatMap((kb, index): KbEntry[] => { + const owner = owners[index]; + const items = owner === undefined ? undefined : groups.get(owner)!; + if (!items || items.length < 2) return [{ kind: "kb", kb }]; + if (placed.has(owner!)) return []; + placed.add(owner!); + return [{ kind: "group", name: owner!, items }]; + }); +} + +function ownerAndLabel(name: string) { + const slash = name.indexOf("/"); + const owner = name.slice(0, slash).trim(); + const label = name.slice(slash + 1).trim(); + return slash >= 0 && owner && label ? { owner, label } : undefined; +} + +/** 给多选下拉用的选项：同组的挨在一起，`label` 留全名（搜索与已选的芯片都用它）， + * `short` 是组名下面那一行显示的名字 */ +export function kbOptions(kbs: Kb[]) { + return kbGroups(kbs).flatMap((entry) => + entry.kind === "kb" + ? [{ value: entry.kb.id, label: entry.kb.name }] + : entry.items.map((item) => ({ + value: item.kb.id, + label: item.kb.name, + group: entry.name, + short: item.label, + })), + ); +} diff --git a/web/src/pages/kbMenuKeys.test.ts b/web/src/pages/kbMenuKeys.test.ts new file mode 100644 index 000000000..39138296d --- /dev/null +++ b/web/src/pages/kbMenuKeys.test.ts @@ -0,0 +1,20 @@ +import { expect, it } from "vitest"; +import { groupRowKey, submenuKey } from "./kbMenuKeys"; + +it.each(["Enter", " ", "ArrowRight"])("opens the submenu on the group row's %j key", (key) => { + expect(groupRowKey(key)).toBe("open"); +}); + +it.each(["ArrowLeft", "ArrowDown", "Tab", "a"])("leaves the group row's %j key alone", (key) => { + expect(groupRowKey(key)).toBeUndefined(); +}); + +it.each([ + ["Escape", "close"], + ["ArrowLeft", "close"], + ["ArrowDown", 1], + ["ArrowUp", -1], + ["Enter", undefined], +] as const)("maps %j inside the submenu to %j", (key, action) => { + expect(submenuKey(key)).toBe(action); +}); diff --git a/web/src/pages/kbMenuKeys.ts b/web/src/pages/kbMenuKeys.ts new file mode 100644 index 000000000..e0ed3e9b5 --- /dev/null +++ b/web/src/pages/kbMenuKeys.ts @@ -0,0 +1,17 @@ +/* 组那一行与它的子菜单认哪些键（KbMenuList）。与菜单的惯例一致： + Enter / 空格 / → 打开并进到第一项；Esc / ← 退回那一行；↑↓ 在子菜单里走 */ +const OPENS = new Set(["Enter", " ", "ArrowRight"]); +const IN_SUBMENU: Record = { + Escape: "close", + ArrowLeft: "close", + ArrowDown: 1, + ArrowUp: -1, +}; + +export function groupRowKey(key: string): "open" | undefined { + return OPENS.has(key) ? "open" : undefined; +} + +export function submenuKey(key: string): "close" | 1 | -1 | undefined { + return IN_SUBMENU[key]; +} diff --git a/web/src/ui/index.tsx b/web/src/ui/index.tsx index 7055d24b8..f7f3a0117 100644 --- a/web/src/ui/index.tsx +++ b/web/src/ui/index.tsx @@ -389,6 +389,9 @@ export interface SearchSelectOption { hint?: string; /** 树形缩进的层级（父类选择器用它） */ indent?: number; + /** 同组的选项不在搜索时排在一个组名下面，显示 `short`；搜索时照旧显示 `label` */ + group?: string; + short?: string; } export function SearchSelect({ @@ -584,7 +587,22 @@ export function MultiSearchSelect({
{open && (
- {visible.map((o, i) => ( + {(q + ? visible.map((option, index) => ({ kind: "option" as const, option, index })) + : withGroupHeadings(visible) + ).map((row) => { + if (row.kind === "heading") + return ( +
+ {row.name} +
+ ); + const { option: o, index: i } = row; + const indent = q ? 0 : (o.indent ?? 0) + (o.group ? 1 : 0); + return ( - ))} + ); + })} {visible.length === 0 && (

{S.ui.noMatches} @@ -629,6 +648,7 @@ export function MultiSearchSelect({ } import { ENTITY_PALETTE } from "../palette"; +import { withGroupHeadings } from "./withGroupHeadings"; export { ENTITY_PALETTE, colorForKey } from "../palette"; export function ColorPicker({ diff --git a/web/src/ui/withGroupHeadings.test.ts b/web/src/ui/withGroupHeadings.test.ts new file mode 100644 index 000000000..05ab7df02 --- /dev/null +++ b/web/src/ui/withGroupHeadings.test.ts @@ -0,0 +1,15 @@ +import { expect, it } from "vitest"; +import { withGroupHeadings } from "./withGroupHeadings"; + +const lore = { value: "1", label: "re-cinq/lore", group: "re-cinq", short: "lore" }; +const floor = { value: "2", label: "re-cinq/floor", group: "re-cinq", short: "floor" }; +const mercedes = { value: "3", label: "mercedes" }; + +it("puts one heading before each run of a group and none before ungrouped options", () => { + expect(withGroupHeadings([mercedes, lore, floor])).toEqual([ + { kind: "option", option: mercedes, index: 0 }, + { kind: "heading", name: "re-cinq" }, + { kind: "option", option: lore, index: 1 }, + { kind: "option", option: floor, index: 2 }, + ]); +}); diff --git a/web/src/ui/withGroupHeadings.ts b/web/src/ui/withGroupHeadings.ts new file mode 100644 index 000000000..c93f5c94e --- /dev/null +++ b/web/src/ui/withGroupHeadings.ts @@ -0,0 +1,15 @@ +type Grouped = { value: string; group?: string }; + +export type HeadedRow = + | { kind: "heading"; name: string } + | { kind: "option"; option: T; index: number }; + +/** 选项列表里，组变了的地方插一行组名。`index` 是选项在原列表里的位置—— + * 键盘的上下键按它走，组名行不占位置，选不中 */ +export function withGroupHeadings(options: T[]): HeadedRow[] { + return options.flatMap((option, index) => { + const opensGroup = !!option.group && option.group !== options[index - 1]?.group; + const row: HeadedRow = { kind: "option", option, index }; + return opensGroup ? [{ kind: "heading", name: option.group! }, row] : [row]; + }); +} From c20d494c561cea5dae921d8a0ee40c4492d73291 Mon Sep 17 00:00:00 2001 From: Nash Aschenbath Date: Mon, 19 Oct 2026 13:30:30 +0800 Subject: [PATCH 15/19] fix: preserve distinct statements and typed sources during entity merges (#1314) Signed-off-by: Aschenbath Co-authored-by: Aschenbath Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- crates/utopia-store/src/resolution.rs | 21 +++- .../a_merge_keeps_distinct_statements.rs | 108 ++++++++++++++++++ crates/utopia-store/tests/store/main.rs | 1 + 3 files changed, 127 insertions(+), 3 deletions(-) create mode 100644 crates/utopia-store/tests/store/a_merge_keeps_distinct_statements.rs diff --git a/crates/utopia-store/src/resolution.rs b/crates/utopia-store/src/resolution.rs index 08d6badf3..104712415 100644 --- a/crates/utopia-store/src/resolution.rs +++ b/crates/utopia-store/src/resolution.rs @@ -2039,14 +2039,15 @@ pub async fn merge_entities( .collect(); // 合并后 SPO+valid_from 重复的 live 事实：留最早 recorded_at 的一条，其余作废。 + // 没有谓词的行不参与：开放陈述的不同短语都带 NULL，不能因此当成同一句话。 // // **宾语两侧都要分组。** 字面值事实的 object_id 全是 NULL，只按它分组就等于 // 把同主同谓下的**所有值**当成同一条断言：一次合并之后， // (公司, 成立年份, 2015) 与 (公司, 注册资本, …) 之外，同谓词的多个值只活得下来 // 最早记的那一个，其余无声消失，且没有 supersedes 可查。 let dups: Vec<(Vec,)> = sqlx::query_as( - "SELECT (array_agg(id ORDER BY recorded_at))[2:] FROM facts - WHERE kb_id = $1 AND invalidated_at IS NULL + "SELECT array_agg(id ORDER BY recorded_at, id) FROM facts + WHERE kb_id = $1 AND invalidated_at IS NULL AND predicate_id IS NOT NULL AND (subject_id = $2 OR object_id = $2) GROUP BY subject_id, predicate_id, object_id, object_value, valid_from HAVING count(*) > 1", @@ -2055,7 +2056,21 @@ pub async fn merge_entities( .bind(target_id) .fetch_all(&mut *tx) .await?; - let dup_ids: Vec = dups.into_iter().flat_map(|(ids,)| ids).collect(); + let mut dup_ids = Vec::new(); + for (ids,) in dups { + // 先给存活行接上来源，否则下一次物化可能作废它，再按原文重建，丢掉人的修正。 + // 旧行的链接暂留：撤回合并还用得上；物化按它原有的规则清理失效链接。 + sqlx::query( + "INSERT INTO typed_fact_sources (fact_id, statement_id) + SELECT $1, statement_id FROM typed_fact_sources WHERE fact_id = ANY($2) + ON CONFLICT DO NOTHING", + ) + .bind(ids[0]) + .bind(&ids[1..]) + .execute(&mut *tx) + .await?; + dup_ids.extend_from_slice(&ids[1..]); + } if !dup_ids.is_empty() { sqlx::query("UPDATE facts SET invalidated_at = now() WHERE id = ANY($1)") .bind(&dup_ids) diff --git a/crates/utopia-store/tests/store/a_merge_keeps_distinct_statements.rs b/crates/utopia-store/tests/store/a_merge_keeps_distinct_statements.rs new file mode 100644 index 000000000..1f0774a8d --- /dev/null +++ b/crates/utopia-store/tests/store/a_merge_keeps_distinct_statements.rs @@ -0,0 +1,108 @@ +//! A merge keeps distinct open phrases and the sources of deduplicated typed facts. + +use sqlx::PgPool; +use utopia_store::resolution; +use uuid::Uuid; + +#[tokio::test] +async fn merging_preserves_open_phrases_and_the_surviving_typed_rows_sources() -> anyhow::Result<()> +{ + let Some(url) = utopia_store::test_db::url() else { + return Ok(()); + }; + let pool = PgPool::connect(&url).await?; + let [org, ws, kb, class, target, source, place, property, binding, withdrawn, ships, receives, based, kept, dropped] = + std::array::from_fn(|_| Uuid::now_v7()); + let run = async { + // Locally generated UUIDs only. The keeper has a corrected end, but its + // original source was withdrawn; the duplicate has the only live source. + sqlx::raw_sql(&format!( + "INSERT INTO organizations(id,name) VALUES ('{org}','merge-statements'); + INSERT INTO workspaces(id,org_id,name) VALUES ('{ws}','{org}','merge-statements'); + INSERT INTO knowledge_bases(id,workspace_id,name) VALUES ('{kb}','{ws}','merge-statements'); + INSERT INTO entity_types(id,kb_id,key,label) VALUES ('{class}','{kb}','thing','Thing'); + INSERT INTO entities(id,kb_id,type_id,canonical_name) VALUES + ('{target}','{kb}','{class}','Harbor Bakery'), + ('{source}','{kb}','{class}','Harbor'), + ('{place}','{kb}','{class}','Port Ellen'); + INSERT INTO relation_types (id,kb_id,key,label,temporal) + VALUES ('{property}','{kb}','headquartered_in','headquartered in','state'); + INSERT INTO phrase_bindings + (id,kb_id,phrase,subject_type_id,object_type_id,object_is_value, + relation_type_id,direction,status) + VALUES ('{binding}','{kb}','is based in','{class}','{class}',false, + '{property}','forward','bound'); + INSERT INTO facts (id,kb_id,subject_id,object_id,layer,phrase,confidence) VALUES + ('{withdrawn}','{kb}','{target}','{place}','open','is based in',0.9), + ('{ships}','{kb}','{target}','{place}','open','ships to',0.9), + ('{receives}','{kb}','{target}','{place}','open','receives from',0.9), + ('{based}','{kb}','{source}','{place}','open','is based in',0.9); + UPDATE facts SET invalidated_at = now() WHERE id = '{withdrawn}'; + INSERT INTO facts + (id,kb_id,subject_id,predicate_id,object_id,confidence,from_statement_id, + recorded_at,valid_to,valid_to_precision,corrected_ends) VALUES + ('{kept}','{kb}','{target}','{property}','{place}',0.9,'{withdrawn}', + '2024-01-01','2025-01-01','year','end'), + ('{dropped}','{kb}','{source}','{property}','{place}',0.9,'{based}', + '2024-02-01',NULL,NULL,NULL); + INSERT INTO typed_fact_sources (fact_id,statement_id) VALUES + ('{kept}','{withdrawn}'), ('{dropped}','{based}');" + )) + .execute(&pool) + .await?; + + let merge = resolution::merge_entities(&pool, kb, source, target, None, "test").await?; + let live: Vec = sqlx::query_scalar( + "SELECT id FROM facts WHERE kb_id = $1 AND subject_id = $2 + AND layer = 'open' AND invalidated_at IS NULL ORDER BY id", + ) + .bind(kb) + .bind(target) + .fetch_all(&pool) + .await?; + let mut expected = vec![ships, receives, based]; + expected.sort(); + assert_eq!(live, expected, "different phrases, including those already on the target, survive"); + let invalidated: Vec = sqlx::query_scalar( + "SELECT invalidated_facts FROM entity_merges WHERE id = $1", + ) + .bind(merge) + .fetch_one(&pool) + .await?; + assert_eq!(invalidated, vec![dropped]); + let has_source: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM typed_fact_sources WHERE fact_id = $1 AND statement_id = $2)", + ) + .bind(kept) + .bind(based) + .fetch_one(&pool) + .await?; + assert!(has_source, "the surviving typed row inherits the duplicate's live source"); + + assert_eq!( + utopia_store::materialize::materialize(&pool, kb).await?, + utopia_store::materialize::Outcome::default(), + "materialization must neither retire the keeper nor rederive its uncorrected interval" + ); + let unchanged: bool = sqlx::query_scalar( + "SELECT invalidated_at IS NULL AND valid_to = '2025-01-01'::timestamptz + AND valid_to_precision = 'year' AND corrected_ends = 'end' + FROM facts WHERE id = $1", + ) + .bind(kept) + .fetch_one(&pool) + .await?; + assert!(unchanged); + anyhow::Ok(()) + } + .await; + sqlx::query("DELETE FROM knowledge_bases WHERE id = $1") + .bind(kb) + .execute(&pool) + .await?; + sqlx::query("DELETE FROM organizations WHERE id = $1") + .bind(org) + .execute(&pool) + .await?; + run +} diff --git a/crates/utopia-store/tests/store/main.rs b/crates/utopia-store/tests/store/main.rs index 2915505a2..45cfd7e11 100644 --- a/crates/utopia-store/tests/store/main.rs +++ b/crates/utopia-store/tests/store/main.rs @@ -55,6 +55,7 @@ mod a_judged_entity_waits_its_turn; mod a_kind_word_binds_to_a_class; mod a_late_value_takes_its_place_in_history; mod a_mapping_is_not_a_fact; +mod a_merge_keeps_distinct_statements; mod a_merge_rewinds_with_the_second_clock; mod a_merged_target_stays_out_of_the_export; mod a_moment_marks_its_state; From 6ac96324be073d26af842c80188ffb63c4201073 Mon Sep 17 00:00:00 2001 From: Nash Aschenbath Date: Mon, 19 Oct 2026 13:36:18 +0800 Subject: [PATCH 16/19] fix: reject review closes that invert a fact's interval (#1315) Signed-off-by: Aschenbath Co-authored-by: Aschenbath Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- .../src/api/graph_routes_time_tests.rs | 102 ++++++++++++++++++ crates/utopia-store/src/temporal.rs | 31 +++++- web/src/i18n/en.ts | 2 + web/src/i18n/zh.ts | 1 + web/src/pages/Review.tsx | 3 + 5 files changed, 136 insertions(+), 3 deletions(-) diff --git a/crates/utopia-server/src/api/graph_routes_time_tests.rs b/crates/utopia-server/src/api/graph_routes_time_tests.rs index 3ddaa6a33..2e26e35c3 100644 --- a/crates/utopia-server/src/api/graph_routes_time_tests.rs +++ b/crates/utopia-server/src/api/graph_routes_time_tests.rs @@ -166,3 +166,105 @@ async fn a_correction_keeps_the_mark_unless_the_request_changes_it() -> anyhow:: .await?; result } + +#[tokio::test] +async fn review_closes_refuse_inverted_intervals_without_settling_the_item() -> anyhow::Result<()> { + let Some(url) = utopia_store::test_db::url() else { + return Ok(()); + }; + let pool = sqlx::PgPool::connect(&url).await?; + let [org, ws, kb, editor, property, lin, meridian, orion, old, new, conflict, violation] = + std::array::from_fn(|_| Uuid::now_v7()); + // Only locally generated UUIDs and fixed timestamps enter this fixture SQL. + sqlx::raw_sql(&format!( + "INSERT INTO organizations(id,name) VALUES ('{org}','review-close-time'); + INSERT INTO workspaces(id,org_id,name) VALUES ('{ws}','{org}','review-close-time'); + INSERT INTO users(id,org_id,email,display_name,password_hash) + VALUES ('{editor}','{org}','{editor}@time.test','editor','unused'); + INSERT INTO knowledge_bases(id,workspace_id,name) VALUES ('{kb}','{ws}','times'); + INSERT INTO kb_members(kb_id,user_id,role) VALUES ('{kb}','{editor}','editor'); + INSERT INTO relation_types(id,kb_id,key,label,temporal) + VALUES ('{property}','{kb}','works_for','works for','state'); + INSERT INTO entities(id,kb_id,canonical_name) VALUES + ('{lin}','{kb}','Lin Zhao'), ('{meridian}','{kb}','Meridian'), ('{orion}','{kb}','Orion'); + INSERT INTO facts(id,kb_id,subject_id,predicate_id,object_id,confidence, + valid_from,valid_from_precision) VALUES + ('{old}','{kb}','{lin}','{property}','{meridian}',0.9,'{MARCH}','day'), + ('{new}','{kb}','{lin}','{property}','{orion}',0.9,NULL,NULL); + INSERT INTO fact_conflicts(id,kb_id,old_fact_id,new_fact_id,reason) + VALUES ('{conflict}','{kb}','{old}','{new}','no_time'); + INSERT INTO axiom_violations(id,kb_id,kind,left_fact,right_fact) + VALUES ('{violation}','{kb}','derived_contradiction','{old}','{new}');" + )) + .execute(&pool) + .await?; + let result = async { + let dir = tempfile::tempdir()?; + let config = utopia_core::config::AppConfig { + data_dir: dir.path().to_string_lossy().into_owned(), + ..Default::default() + }; + let search = Arc::new(utopia_search::SearchIndex::open(&dir.path().join("search"))?); + let state = crate::state::AppState::new(pool.clone(), &config, search, "test-only".into()); + let token = crate::auth::issue_token(&state, editor)?; + let app = crate::api::router(state, &config); + let prefix = format!("/api/v1/kbs/{kb}"); + // The first date is later as written, but earlier at year precision. + // The existing equal-state refusal must reach all three cards as well. + for (at, precision, code) in [ + ("2024-10-01T00:00:00Z", "year", "interval_inverted"), + (MARCH, "month", "empty_state_span"), + ] { + for (path, body) in [ + ( + format!("facts/{old}/close"), + json!({"valid_to": at, "valid_to_precision": precision}), + ), + ( + format!("conflicts/{conflict}"), + json!({"action": "close", "close_at": at, "close_at_precision": precision}), + ), + ( + format!("review/violations/{violation}"), + json!({"resolution": "fact_closed", "close_at": at, "close_at_precision": precision}), + ), + ] { + let (status, reply) = send(&app, &token, "POST", format!("{prefix}/{path}"), &body).await?; + assert_eq!((status, reply["code"].as_str()), (StatusCode::UNPROCESSABLE_ENTITY, Some(code)), "{path}: {reply}"); + } + } + // A conflict with no explicit date closes at the new row's start; it must + // get the same store guard, even though no date came through the route. + sqlx::query("UPDATE facts SET valid_from = '2024-01-01', valid_from_precision = 'year' WHERE id = $1") + .bind(new).execute(&pool).await?; + let (status, reply) = send(&app, &token, "POST", format!("{prefix}/conflicts/{conflict}"), &json!({"action": "close"})).await?; + assert_eq!((status, reply["code"].as_str()), (StatusCode::UNPROCESSABLE_ENTITY, Some("interval_inverted"))); + let unchanged: bool = sqlx::query_scalar( + "SELECT (SELECT count(*) = 2 FROM facts WHERE kb_id = $1) + AND EXISTS (SELECT 1 FROM facts WHERE id = $2 AND invalidated_at IS NULL AND valid_to IS NULL) + AND EXISTS (SELECT 1 FROM fact_conflicts WHERE id = $3 AND status = 'open' AND resolution IS NULL) + AND EXISTS (SELECT 1 FROM axiom_violations WHERE id = $4 AND status = 'open' AND resolution IS NULL)", + ).bind(kb).bind(old).bind(conflict).bind(violation).fetch_one(&pool).await?; + assert!(unchanged, "refused closes write no replacement and leave both queues open"); + + // A later date can still close the refused row, with the endpoint stored + // at its supplied precision, without translating it to a period's end. + let (status, _) = send(&app, &token, "POST", format!("{prefix}/facts/{old}/close"), + &json!({"valid_to": "2024-09-19T00:00:00Z", "valid_to_precision": "month"})).await?; + assert_eq!(status, StatusCode::OK); + let closed: (DateTime, String, bool) = sqlx::query_as( + "SELECT valid_to, valid_to_precision, end_after_period FROM facts WHERE supersedes = $1", + ).bind(old).fetch_one(&pool).await?; + assert_eq!(closed, ("2024-09-01T00:00:00Z".parse()?, "month".into(), false)); + anyhow::Ok(()) + }.await; + sqlx::query("DELETE FROM knowledge_bases WHERE id = $1") + .bind(kb) + .execute(&pool) + .await?; + sqlx::query("DELETE FROM organizations WHERE id = $1") + .bind(org) + .execute(&pool) + .await?; + result +} diff --git a/crates/utopia-store/src/temporal.rs b/crates/utopia-store/src/temporal.rs index 9b68dda3b..85e6d9853 100644 --- a/crates/utopia-store/src/temporal.rs +++ b/crates/utopia-store/src/temporal.rs @@ -873,16 +873,41 @@ pub async fn retract(pool: &PgPool, kb_id: Uuid, fact_id: Uuid) -> AppResult( pool: impl sqlx::Acquire<'a, Database = Postgres>, fact_id: Uuid, valid_to: DateTime, valid_to_precision: &str, ) -> AppResult> { + let valid_to = crate::graph::truncate_to(valid_to, Some(valid_to_precision)); + let mut tx = pool.begin().await?; + let row: Option<(Option>, Option)> = sqlx::query_as( + "SELECT valid_from, valid_from_precision FROM facts + WHERE id = $1 AND invalidated_at IS NULL FOR UPDATE", + ) + .bind(fact_id) + .fetch_optional(&mut *tx) + .await?; + let Some((from, from_precision)) = row else { + return Ok(None); + }; + // 锁住起点后再比，避免并发修正穿过检查；两端都按各自精度截断（#1294）。 + // 放在审核的 store 入口，也管住冲突默认取新事实起点的闭合，不改原文写入的规则。 + if from + .is_some_and(|from| valid_to < crate::graph::truncate_to(from, from_precision.as_deref())) + { + return Err(utopia_core::AppError::invalid( + "interval_inverted", + "The start of the interval is after its end.", + )); + } // 按日期关上的（审核里关、冲突的 close_at、违规的 fact_closed）给的就是终点那一刻，不标 - close_stated(pool, fact_id, valid_to, valid_to_precision, false).await + let end = End::At(valid_to, valid_to_precision.to_string()); + let corrected = rewrite_end_tx(&mut tx, fact_id, &end, false, false, false).await?; + tx.commit().await?; + Ok(corrected) } /// [`close_superseded`]，外加这个终点是不是原文所说时段之后的那一刻（`end_after_period`， diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts index 2eb3e0d88..a7109fcb7 100644 --- a/web/src/i18n/en.ts +++ b/web/src/i18n/en.ts @@ -140,6 +140,8 @@ export const en = { inference_off: "Materialized inference is off for this knowledge base. Turn it on in Settings.", bad_resolution: "That is not a valid decision.", + interval_inverted: + "The end is before the start. Give an end after the start.", empty_state_span: "A state that ends where it starts holds at no moment. Give an end after the start, or leave it open.", unknown_marks: "A single date marks the start, the end, or neither.", diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.ts index ddda99a15..9222d4edc 100644 --- a/web/src/i18n/zh.ts +++ b/web/src/i18n/zh.ts @@ -127,6 +127,7 @@ export const zh: Strings = { concurrency_range: "填一个 1 到 256 之间的数。", inference_off: "这个库没有打开物化推理。到设置里开。", bad_resolution: "这不是一个有效的裁决。", + interval_inverted: "终点早于起点。请给一个晚于起点的终点。", empty_state_span: "起止相同的状态任何时刻都不成立。给一个晚于起点的终点，或者让它开着。", unknown_marks: "单个日期标的是开始、结束，或者都不是。", marks_needs_state: "只有状态属性才说单个日期标的是什么，事件与恒常属性不说。", diff --git a/web/src/pages/Review.tsx b/web/src/pages/Review.tsx index 88e3a1eb7..ac6c213a6 100644 --- a/web/src/pages/Review.tsx +++ b/web/src/pages/Review.tsx @@ -1551,6 +1551,7 @@ export function Review() { resolution: ViolationResolution; opts?: DecideOpts; }) => api.decideViolation(kb!.id, id, resolution, opts), + onError: (e) => toast.error(e instanceof Error ? e.message : String(e)), onSettled: invalidate, }); // 检查是同步的纯计算,所以直接 mutate 不排队。跑完把报告留在按钮旁边—— @@ -1580,6 +1581,7 @@ export function Review() { close_at: closeAt, close_at_precision: closeAtPrecision, }), + onError: (e) => toast.error(e instanceof Error ? e.message : String(e)), onSettled: invalidate, }); @@ -1593,6 +1595,7 @@ export function Review() { validTo: string; precision: string; }) => api.closeFact(kb!.id, id, validTo, precision), + onError: (e) => toast.error(e instanceof Error ? e.message : String(e)), onSettled: invalidate, }); From 4c5221ad6bdca9fec1f1dd52405407b6ae5874c5 Mon Sep 17 00:00:00 2001 From: Szabo Bogdan Date: Mon, 19 Oct 2026 08:12:58 +0200 Subject: [PATCH 17/19] A push that names its external_id keeps that id's document (#1268) * A push that names its external_id keeps that id's document, so identical content under another id is reported as a duplicate instead of renaming the first, vanishing as unchanged, or failing the push Signed-off-by: Bogdan Szabo * Tombstone then push renames a declared id, and a duplicate answer leaves no restore or missing-flag change behind Signed-off-by: Bogdan Szabo --------- Signed-off-by: Bogdan Szabo Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- .../utopia-server/src/api/sources_routes.rs | 65 ++++--- .../src/api/sources_statements_tests.rs | 163 +++++++++++++++++- crates/utopia-server/src/ingest_sources.rs | 61 +++++-- crates/utopia-store/src/documents.rs | 23 ++- ...3-a-source-should-hand-over-its-history.md | 1 + web/src/docs/ingest.md | 6 +- 6 files changed, 284 insertions(+), 35 deletions(-) diff --git a/crates/utopia-server/src/api/sources_routes.rs b/crates/utopia-server/src/api/sources_routes.rs index 2c786394d..6e8cec739 100644 --- a/crates/utopia-server/src/api/sources_routes.rs +++ b/crates/utopia-server/src/api/sources_routes.rs @@ -404,6 +404,16 @@ pub struct IngestBody { pub deleted: bool, } +/// 推送的回执：动作，重复时再加上占着这份内容的文档 +fn action_reply(action: crate::ingest_sources::IngestAction) -> serde_json::Value { + match action { + crate::ingest_sources::IngestAction::Duplicate(id) => { + json!({ "action": action_str(action), "duplicate_of": id }) + } + _ => json!({ "action": action_str(action) }), + } +} + fn action_str(action: crate::ingest_sources::IngestAction) -> &'static str { match action { crate::ingest_sources::IngestAction::Created => "created", @@ -411,6 +421,7 @@ fn action_str(action: crate::ingest_sources::IngestAction) -> &'static str { crate::ingest_sources::IngestAction::Moved => "moved", crate::ingest_sources::IngestAction::Unchanged => "unchanged", crate::ingest_sources::IngestAction::Tombstoned => "marked_missing", + crate::ingest_sources::IngestAction::Duplicate(_) => "duplicate", } } @@ -444,7 +455,7 @@ pub async fn ingest( ) .await .map_err(utopia_core::AppError::Other)?; - Ok(Json(json!({ "action": action_str(action) }))) + Ok(Json(action_reply(action))) } /// 推送失败的两种性质。**调用方发错了**和**我们这边没接住**得分开：前者记进 @@ -474,13 +485,12 @@ async fn handle_push( ) -> Result { let body: IngestBody = serde_json::from_slice(bytes) .map_err(|e| PushError::Rejected(format!("Invalid JSON payload: {e}")))?; - let identity = body + let declared = body .external_id .as_deref() .map(str::trim) - .filter(|s| !s.is_empty()) - .unwrap_or_else(|| body.filename.trim()) - .to_string(); + .filter(|s| !s.is_empty()); + let identity = declared.unwrap_or_else(|| body.filename.trim()).to_string(); if identity.is_empty() { return Err(PushError::Rejected( "external_id or filename is required".into(), @@ -501,22 +511,31 @@ async fn handle_push( "filename and content are required".into(), )); } - let action = crate::ingest_sources::ingest_item( + // 带了 external_id 身份就是它：不走「同内容 = 移动」，不把另一个 ID 的文档改成它 + let action = crate::ingest_sources::ingest_item_with_outcome( state, source.kb_id, source.id, &key, + if declared.is_some() { + crate::ingest_sources::Identity::Declared + } else { + crate::ingest_sources::Identity::Path + }, body.filename.trim(), "text/plain", body.content.as_bytes(), body.doc_time, ) .await - .map_err(|e| PushError::Failed(e.to_string()))?; - // 失而复得：曾被墓碑标记的身份再次正常推送，摘掉 missing 标记 - utopia_store::documents::clear_missing_keys(&state.pool, source.id, &[key]) - .await - .map_err(|e| PushError::Failed(e.to_string()))?; + .map_err(|e| PushError::Failed(e.to_string()))? + .action; + // 失而复得：曾被墓碑标记的身份再次正常推送，摘掉 missing 标记；重复的推送什么也没落，不摘 + if !matches!(action, crate::ingest_sources::IngestAction::Duplicate(_)) { + utopia_store::documents::clear_missing_keys(&state.pool, source.id, &[key]) + .await + .map_err(|e| PushError::Failed(e.to_string()))?; + } Ok(action) } @@ -554,14 +573,15 @@ pub async fn push( crate::ingest_sources::IngestAction::Updated | crate::ingest_sources::IngestAction::Moved => (0, 1), crate::ingest_sources::IngestAction::Unchanged - | crate::ingest_sources::IngestAction::Tombstoned => (0, 0), + | crate::ingest_sources::IngestAction::Tombstoned + | crate::ingest_sources::IngestAction::Duplicate(_) => (0, 0), }; utopia_store::sources::finish_run(&state.pool, run, source.id, None, created, updated) .await?; // 来源行同步反映"上一次推送"：操作条/左栏状态点直接可用 utopia_store::sources::finish_sync(&state.pool, source.id, None, created).await?; state.emit_source(source.kb_id); - Ok(Json(json!({ "action": action_str(action) }))) + Ok(Json(action_reply(action))) } Err(err) => { let msg = err.message().to_string(); @@ -612,13 +632,14 @@ pub async fn push_statements( crate::ingest_sources::IngestAction::Updated | crate::ingest_sources::IngestAction::Moved => (0, 1), crate::ingest_sources::IngestAction::Unchanged - | crate::ingest_sources::IngestAction::Tombstoned => (0, 0), + | crate::ingest_sources::IngestAction::Tombstoned + | crate::ingest_sources::IngestAction::Duplicate(_) => (0, 0), }; utopia_store::sources::finish_run(&state.pool, run, source.id, None, created, updated) .await?; utopia_store::sources::finish_sync(&state.pool, source.id, None, created).await?; state.emit_source(source.kb_id); - Ok(Json(json!({ "action": action_str(action) }))) + Ok(Json(action_reply(action))) } Err(err) => { let msg = err.message().to_string(); @@ -797,21 +818,25 @@ async fn handle_statements_push( return Ok(crate::ingest_sources::IngestAction::Tombstoned); }; let filename = format!("{identity}.json"); - let action = crate::ingest_sources::ingest_item( + let action = crate::ingest_sources::ingest_item_with_outcome( state, source.kb_id, source.id, &key, + crate::ingest_sources::Identity::Declared, &filename, "application/json", content.as_bytes(), body.doc_time, ) .await - .map_err(|e| PushError::Failed(e.to_string()))?; - utopia_store::documents::clear_missing_keys(&state.pool, source.id, &[key]) - .await - .map_err(|e| PushError::Failed(e.to_string()))?; + .map_err(|e| PushError::Failed(e.to_string()))? + .action; + if !matches!(action, crate::ingest_sources::IngestAction::Duplicate(_)) { + utopia_store::documents::clear_missing_keys(&state.pool, source.id, &[key]) + .await + .map_err(|e| PushError::Failed(e.to_string()))?; + } Ok(action) } diff --git a/crates/utopia-server/src/api/sources_statements_tests.rs b/crates/utopia-server/src/api/sources_statements_tests.rs index 47350f256..9d97827aa 100644 --- a/crates/utopia-server/src/api/sources_statements_tests.rs +++ b/crates/utopia-server/src/api/sources_statements_tests.rs @@ -183,8 +183,24 @@ impl Fixture { token: Option<&str>, body: Vec, ) -> anyhow::Result<(StatusCode, Value)> { - let mut request = Request::post(format!("/api/v1/sources/{source}/statements")) - .header("Content-Type", "application/json"); + self.send(&format!("/api/v1/sources/{source}/statements"), token, body) + .await + } + + /// 推给 api 来源（`/ingest`），用它自己的密钥 + async fn push_api(&self, body: &Value) -> anyhow::Result<(StatusCode, Value)> { + let path = format!("/api/v1/sources/{}/ingest", self.api_source); + self.send(&path, Some(&self.api_token), body.to_string().into_bytes()) + .await + } + + async fn send( + &self, + path: &str, + token: Option<&str>, + body: Vec, + ) -> anyhow::Result<(StatusCode, Value)> { + let mut request = Request::post(path).header("Content-Type", "application/json"); if let Some(token) = token { request = request.header("Authorization", format!("Bearer {token}")); } @@ -577,3 +593,146 @@ async fn the_same_payload_under_a_new_identity_is_a_second_observation() -> anyh } f.cleanup().await } + +/// api 推送带了 `external_id`，身份就是这个 ID：另一个 ID 推来同样的字节不是把第一篇 +/// 改名过去（从前走「同内容出现在新路径 = 移动」，第一个 ID 就此消失），而是如实回一句 +/// 重复了谁，两个 ID 里已有的那篇原样留着 +#[tokio::test] +async fn a_new_id_with_the_same_content_does_not_take_the_first_ids_document() -> anyhow::Result<()> +{ + let Some(f) = Fixture::new().await? else { + return Ok(()); + }; + let record = |id: &str| json!({ "filename": "record.txt", "content": "colour = blue", "external_id": id }); + let (status, body) = f.push_api(&record("a")).await?; + assert_eq!( + (status, body["action"].as_str()), + (StatusCode::OK, Some("created")), + "{body}" + ); + let first = documents::find_by_external_key(&f.pool, f.api_source, "api:a") + .await? + .expect("a is stored"); + + let (status, body) = f.push_api(&record("b")).await?; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!( + body, + json!({ "action": "duplicate", "duplicate_of": first.id }) + ); + let kept = documents::find_by_external_key(&f.pool, f.api_source, "api:a").await?; + assert_eq!(kept.map(|d| d.id), Some(first.id), "a keeps its document"); + f.cleanup().await +} + +/// 同样的字节已经在库里别处（这里是一篇普通上传）：从前回 `unchanged`、什么也没建， +/// 调用方以为它的 ID 已经落库 +#[tokio::test] +async fn content_already_in_the_base_is_reported_as_a_duplicate() -> anyhow::Result<()> { + let Some(f) = Fixture::new().await? else { + return Ok(()); + }; + crate::ingest_sources::ingest_upload( + &f.state, + f.kb, + "upload.txt", + "text/plain", + b"colour = green", + None, + ) + .await?; + let upload: Uuid = + sqlx::query_scalar("SELECT id FROM documents WHERE kb_id = $1 AND source_id IS NULL") + .bind(f.kb) + .fetch_one(&f.pool) + .await?; + + let push = json!({ "filename": "record.txt", "content": "colour = green", "external_id": "c" }); + let (status, body) = f.push_api(&push).await?; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!( + body, + json!({ "action": "duplicate", "duplicate_of": upload }) + ); + f.cleanup().await +} + +/// 已有的 ID 改成另一篇文档的内容：从前撞上 `documents_kb_sha_idx`，推送以一条 SQL 错误 +/// 失败、来源被标成同步失败；现在回重复了谁，这个 ID 的文档保持原来的内容 +#[tokio::test] +async fn an_update_to_another_documents_content_is_a_duplicate_not_a_failure() -> anyhow::Result<()> +{ + let Some(f) = Fixture::new().await? else { + return Ok(()); + }; + let push = |id: &str, content: &str| json!({ "filename": "record.txt", "content": content, "external_id": id }); + f.push_api(&push("a", "colour = blue")).await?; + f.push_api(&push("b", "colour = red")).await?; + let a = documents::find_by_external_key(&f.pool, f.api_source, "api:a") + .await? + .expect("a is stored"); + + let (status, body) = f.push_api(&push("b", "colour = blue")).await?; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body, json!({ "action": "duplicate", "duplicate_of": a.id })); + let b = documents::find_by_external_key(&f.pool, f.api_source, "api:b") + .await? + .expect("b is still stored"); + assert_ne!(b.sha256, a.sha256, "b keeps its own content"); + f.cleanup().await +} + +/// 拿 ID 当路径用的调用方这样改名：先墓碑旧 ID，再用同样的字节推新 ID。新 ID 接手那篇 +/// 被墓碑标记的文档（同一篇、历史还在），旧 ID 不再留着一篇「不在来源中」的孤本 +#[tokio::test] +async fn a_new_id_with_a_tombstoned_ids_content_takes_over_its_document() -> anyhow::Result<()> { + let Some(f) = Fixture::new().await? else { + return Ok(()); + }; + let record = |id: &str| json!({ "filename": "record.txt", "content": "colour = blue", "external_id": id }); + f.push_api(&record("a")).await?; + let first = documents::find_by_external_key(&f.pool, f.api_source, "api:a") + .await? + .expect("a is stored"); + f.push_api(&json!({ "filename": "record.txt", "external_id": "a", "deleted": true })) + .await?; + + let (status, body) = f.push_api(&record("b")).await?; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body, json!({ "action": "moved" })); + let b = documents::find_by_external_key(&f.pool, f.api_source, "api:b") + .await? + .expect("b is stored"); + assert_eq!((b.id, b.missing_since), (first.id, None)); + let a = documents::find_by_external_key(&f.pool, f.api_source, "api:a").await?; + assert!(a.is_none(), "a is gone"); + f.cleanup().await +} + +/// 回收站里的 ID 推来另一篇文档的内容：先答重复，回收站里那篇不被恢复 +#[tokio::test] +async fn a_trashed_id_pushed_with_another_documents_content_stays_in_the_trash( +) -> anyhow::Result<()> { + let Some(f) = Fixture::new().await? else { + return Ok(()); + }; + let push = |id: &str, content: &str| json!({ "filename": "record.txt", "content": content, "external_id": id }); + f.push_api(&push("a", "colour = blue")).await?; + f.push_api(&push("b", "colour = red")).await?; + let a = documents::find_by_external_key(&f.pool, f.api_source, "api:a") + .await? + .expect("a is stored"); + let b = documents::find_by_external_key(&f.pool, f.api_source, "api:b") + .await? + .expect("b is stored"); + documents::delete(&f.pool, f.kb, a.id, None).await?; + + let (status, body) = f.push_api(&push("a", "colour = red")).await?; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body, json!({ "action": "duplicate", "duplicate_of": b.id })); + let a = documents::find_by_external_key(&f.pool, f.api_source, "api:a") + .await? + .expect("a is still stored"); + assert!(a.deleted_at.is_some(), "a stays in the trash"); + f.cleanup().await +} diff --git a/crates/utopia-server/src/ingest_sources.rs b/crates/utopia-server/src/ingest_sources.rs index 62f81c69d..17469be09 100644 --- a/crates/utopia-server/src/ingest_sources.rs +++ b/crates/utopia-server/src/ingest_sources.rs @@ -153,6 +153,17 @@ pub enum IngestAction { Unchanged, /// 墓碑：标记"不在来源中"（不删除文档，删不删由用户在 UI 决定） Tombstoned, + /// 库里已有一篇同内容的文档（`documents_kb_sha_idx`）：这次什么也没建、没改，指向那一篇 + Duplicate(Uuid), +} + +/// 身份从哪来。`Path`：来源按位置认文档，同内容出现在新位置就是移动/改名，历史跟着走； +/// `Declared`：调用方亲口给了 ID，身份就是它，绝不把另一个活着的 ID 的文档改成它； +/// 只有已被墓碑标记的那篇可以接手（改名）。 +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum Identity { + Path, + Declared, } #[derive(Debug, Clone, Copy)] @@ -231,6 +242,7 @@ pub async fn ingest_item( kb_id, source_id, external_key, + Identity::Path, filename, mime, bytes, @@ -246,6 +258,7 @@ pub async fn ingest_item_with_outcome( kb_id: Uuid, source_id: Uuid, external_key: &str, + identity: Identity, filename: &str, mime: &str, bytes: &[u8], @@ -281,6 +294,16 @@ pub async fn ingest_item_with_outcome( } if let Some(doc) = existing { + // 新内容已被库里另一篇占着：先答重复，免得下面恢复回收站、重建索引之后才发现 + if doc.sha256 != sha256 { + if let Some(holder) = + utopia_store::documents::find_by_kb_sha(&state.pool, kb_id, &sha256).await? + { + return Ok(IngestOutcome { + action: IngestAction::Duplicate(holder.id), + }); + } + } // Ordinary source reappearance retains upstream restore semantics, // including search and derivation repair, before comparing content. let revived = doc.deleted_at.is_some(); @@ -312,7 +335,7 @@ pub async fn ingest_item_with_outcome( } // 变更：原地替换，旧版本入 document_versions，并与处理任务一起提交 write_blob(state, &sha256, bytes).await?; - utopia_store::documents::replace_content_and_enqueue_processing( + match utopia_store::documents::replace_content_and_enqueue_processing( &state.pool, doc.id, filename, @@ -321,17 +344,26 @@ pub async fn ingest_item_with_outcome( &sha256, doc_time, ) - .await?; + .await + { + Ok(()) => {} + Err(utopia_core::AppError::Conflict(_)) => { + return duplicate_of(state, kb_id, &sha256).await + } + Err(e) => return Err(e.into()), + } state.emit_document(kb_id, doc.id); return Ok(IngestOutcome { action: IngestAction::Updated, }); } - // 同内容出现在新路径：识别为移动/改名，不重跑管道 - if let Some(doc) = - utopia_store::documents::find_by_source_sha(&state.pool, source_id, &sha256).await? - { + // 同内容出现在新路径：识别为移动/改名，不重跑管道。声明的 ID 只接手已被墓碑标记的那篇 + // （先墓碑旧 ID、再推新 ID = 改名），活着的绝不改走 + let moved = utopia_store::documents::find_by_source_sha(&state.pool, source_id, &sha256) + .await? + .filter(|doc| identity == Identity::Path || doc.missing_since.is_some()); + if let Some(doc) = moved { utopia_store::documents::update_location(&state.pool, doc.id, filename, external_key) .await?; state.emit_document(kb_id, doc.id); @@ -360,14 +392,23 @@ pub async fn ingest_item_with_outcome( action: IngestAction::Created, }) } - // KB 内已有同内容：不伪造新的身份，保留幂等结果 - Err(utopia_core::AppError::Conflict(_)) => Ok(IngestOutcome { - action: IngestAction::Unchanged, - }), + // KB 内已有同内容：不伪造新的身份，如实说重复了谁 + Err(utopia_core::AppError::Conflict(_)) => duplicate_of(state, kb_id, &sha256).await, Err(e) => Err(e.into()), } } +async fn duplicate_of( + state: &AppState, + kb_id: Uuid, + sha256: &str, +) -> anyhow::Result { + let holder = utopia_store::documents::find_by_kb_sha(&state.pool, kb_id, sha256).await?; + Ok(IngestOutcome { + action: holder.map_or(IngestAction::Unchanged, |d| IngestAction::Duplicate(d.id)), + }) +} + async fn sync_urls(state: &AppState, source: &Source) -> anyhow::Result { let urls: Vec = source.config["urls"] .as_array() diff --git a/crates/utopia-store/src/documents.rs b/crates/utopia-store/src/documents.rs index 3822fbad5..45c12acf1 100644 --- a/crates/utopia-store/src/documents.rs +++ b/crates/utopia-store/src/documents.rs @@ -282,7 +282,13 @@ pub async fn replace_content_and_enqueue_processing( .bind(sha256) .bind(doc_time) .execute(&mut *tx) - .await?; + .await + .map_err(|e| match &e { + sqlx::Error::Database(db) if db.is_unique_violation() => AppError::Conflict(format!( + "File already exists (identical content): {filename}" + )), + _ => AppError::Db(e), + })?; sqlx::query( "INSERT INTO document_versions (id, document_id, version, sha256, size_bytes, doc_time) VALUES ($1, $2, @@ -736,6 +742,21 @@ pub async fn find_by_source_sha( .await?) } +/// 库里占着这份内容的那篇文档（`documents_kb_sha_idx` 管的范围：未清除的，含回收站里的）。 +pub async fn find_by_kb_sha( + pool: &PgPool, + kb_id: Uuid, + sha256: &str, +) -> AppResult> { + Ok(sqlx::query_as( + "SELECT * FROM documents WHERE kb_id = $1 AND sha256 = $2 AND purged_at IS NULL", + ) + .bind(kb_id) + .bind(sha256) + .fetch_optional(pool) + .await?) +} + /// 迁移前的历史文档（无 external_key）按文件名认领——0008 之前建的行的一次性兜底。 pub async fn find_legacy_by_filename( pool: &PgPool, diff --git a/docs/decisions/0013-a-source-should-hand-over-its-history.md b/docs/decisions/0013-a-source-should-hand-over-its-history.md index 766040d0f..0fed62a3b 100644 --- a/docs/decisions/0013-a-source-should-hand-over-its-history.md +++ b/docs/decisions/0013-a-source-should-hand-over-its-history.md @@ -126,6 +126,7 @@ shape — a class with an IRI drawn as a circle is a picture that lies. - 2026-09-03: every connector's credentials stay on the server (#246). Until now only `auth_header` was stripped from responses; the object-storage, WebDAV and Notion keys went out to every viewer. The keys now live in one list, `SOURCE_SECRET_KEYS`, shared by the listing, the create / update responses and the update merge (blank or missing keeps the stored value, an explicit `null` removes it). Adding a connector means adding its keys there first. - 2026-09-03: the five connectors added under this record could not be created (#247): the store's hand-written `KINDS` allowlist stopped at seven kinds while the sync dispatcher and the UI knew twelve. The kinds now come from one enum, `SourceKind` in `utopia-core`; the allowlist is derived from it, the dispatcher matches it exhaustively, and a test compares the frontend's `web/src/sourceKinds.ts` against it, so the three can no longer drift apart. +- 2026-10-16: a push that names its `external_id` is identified by that id alone. The same bytes under a new id used to take the "same content at a new path is a move" branch, which re-keyed the existing document and silently dropped the first id. That branch now runs in full only for sources that identify by location (files, and an `api` push without `external_id`), where it keeps a renamed file's history. A push with `external_id` takes it over only from a document of the same source whose id was tombstoned, never from a live one: tombstoning the old id and then pushing the new id with the same bytes is how such a client renames, and the document keeps its history. A push whose content another document of the knowledge base already holds (any not purged, so `duplicate_of` can name a document in the trash), whether it is new or an update of an existing id, is answered `{"action": "duplicate", "duplicate_of": }`; it used to answer `unchanged` for a new id and fail the push with a database error for an update. One document per content per base (`documents_kb_sha_idx`) stands: storing the second id as its own document would need that index relaxed, which this revision does not do. ## Status history diff --git a/web/src/docs/ingest.md b/web/src/docs/ingest.md index d5cbaac44..e7bce0500 100644 --- a/web/src/docs/ingest.md +++ b/web/src/docs/ingest.md @@ -107,7 +107,9 @@ The response tells you what happened: { "action": "created" } ``` -`created` · `updated` · `moved` (same content, new name) · `unchanged` · `marked_missing`. +`created` · `updated` · `moved` (same content, new name) · `unchanged` · `marked_missing` · `duplicate` (the base already holds this exact content as another document, named by `duplicate_of`, which may be a document in the trash; nothing was created or changed). + +With `external_id`, the id is the identity: the same content under a new id never takes over a live id's document. To rename an id, tombstone the old one (`"deleted": true`), then push the new id with the same content; the new id takes over the document, with its history, and answers `moved`. Example with curl: @@ -156,7 +158,7 @@ Two things to know before wiring a system to it: - **Send events, not state.** A row that *is* the current state of a system belongs on a mounted database, read at query time. Pushing it here copies state into the ledger and the two drift. - **A new push under the same identity does not close the old statement.** The earlier one becomes stale and goes to review; deciding that an interval ended is done on the typed layer, by alignment and the temporal engine, not by the push. -The response is the API push's: `{ "action": "created" }` · `updated` · `unchanged` · `marked_missing`. +The response is the API push's: `{ "action": "created" }` · `updated` · `unchanged` · `marked_missing` · `duplicate` (with `duplicate_of`). It is never `moved`: the stored document carries its `external_id`, so two identities never share content. --- From 59b5b6de6a9a8fdc2935053a4a9fc042c9003bf6 Mon Sep 17 00:00:00 2001 From: Bogdan Szabo Date: Mon, 19 Oct 2026 08:14:19 +0200 Subject: [PATCH 18/19] Revert "The typed recompute plans its statements for the base it is given, so a generic plan cannot turn its source cleanup into hours." This reverts commit 10fa50b2f2080138045920c715938b09d13d59aa. --- crates/utopia-store/src/materialize.rs | 12 ---------- .../src/materialize_plan_tests.rs | 23 ------------------- 2 files changed, 35 deletions(-) delete mode 100644 crates/utopia-store/src/materialize_plan_tests.rs diff --git a/crates/utopia-store/src/materialize.rs b/crates/utopia-store/src/materialize.rs index a07f8749a..cd81523b5 100644 --- a/crates/utopia-store/src/materialize.rs +++ b/crates/utopia-store/src/materialize.rs @@ -156,14 +156,6 @@ async fn materialize_in_tx( tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, kb_id: Uuid, ) -> AppResult<(Outcome, Vec)> { - // Plan every statement of the recompute for this base. Prepared statements switch to a - // generic plan after a few runs, and a generic plan cannot see how many rows `kb_id = $1` - // selects: on a production base of 200,000 typed-fact sources the source cleanup below - // chose a nested loop over a materialized anti-join and ran for hours, where the plan for - // the base's own id takes two seconds. LOCAL ends with the transaction. - sqlx::query("SET LOCAL plan_cache_mode = force_custom_plan") - .execute(&mut **tx) - .await?; // 这一轮写下的行（新建的和并入的），提交后对账 let mut written: Vec = Vec::new(); // 0. 起止同值的物化状态行（#966 之前照抄一刻写下的）任何时刻都不成立：作废，它的陈述 @@ -706,7 +698,3 @@ pub async fn count(pool: &PgPool, kb_id: Uuid) -> AppResult { #[cfg(test)] #[path = "materialize_delivery_tests.rs"] mod delivery_tests; - -#[cfg(test)] -#[path = "materialize_plan_tests.rs"] -mod plan_tests; diff --git a/crates/utopia-store/src/materialize_plan_tests.rs b/crates/utopia-store/src/materialize_plan_tests.rs deleted file mode 100644 index efa06d92c..000000000 --- a/crates/utopia-store/src/materialize_plan_tests.rs +++ /dev/null @@ -1,23 +0,0 @@ -//! The recompute plans each statement for the base it is given (#1298: a generic plan of the -//! source cleanup ran for hours on a production base where the custom plan takes two seconds). -use super::materialize_in_tx; -use sqlx::PgPool; -use uuid::Uuid; - -#[tokio::test] -async fn a_recompute_plans_its_statements_for_the_base_it_is_given() -> anyhow::Result<()> { - let Some(url) = crate::test_db::url() else { - return Ok(()); - }; - let pool = PgPool::connect(&url).await?; - let mut tx = pool.begin().await?; - - materialize_in_tx(&mut tx, Uuid::now_v7()).await?; - let mode: String = sqlx::query_scalar("SELECT current_setting('plan_cache_mode')") - .fetch_one(&mut *tx) - .await?; - tx.rollback().await?; - - assert_eq!(mode, "force_custom_plan"); - Ok(()) -} From 0f8fad376748cecd8d571dad322c98a9f1d8c374 Mon Sep 17 00:00:00 2001 From: Bogdan Szabo Date: Mon, 19 Oct 2026 08:19:59 +0200 Subject: [PATCH 19/19] Sync: drop the duplicate keyed() and KbMenuList imports both sides added --- crates/utopia-server/src/phrase_alignment.rs | 18 ------------------ web/src/pages/Chat.tsx | 1 - web/src/pages/KbSwitcher.tsx | 1 - 3 files changed, 20 deletions(-) diff --git a/crates/utopia-server/src/phrase_alignment.rs b/crates/utopia-server/src/phrase_alignment.rs index 6610e24e1..ba83160db 100644 --- a/crates/utopia-server/src/phrase_alignment.rs +++ b/crates/utopia-server/src/phrase_alignment.rs @@ -229,24 +229,6 @@ fn undecided( .collect() } -/// 只有键的签名：计数与例句为空，够 `settled` / `declared` 判 -fn keyed( - (phrase, subject_type_id, object_type_id, object_is_value): &SignatureKey, -) -> PhraseSignature { - PhraseSignature { - phrase: phrase.clone(), - subject_type_id: *subject_type_id, - subject_type_key: None, - object_type_id: *object_type_id, - object_type_key: None, - object_is_value: *object_is_value, - count: 0, - examples: Vec::new(), - quotes: Vec::new(), - example_statement_ids: Vec::new(), - } -} - /// 推送来的文档抽完（0054）：它的陈述落在的签名里，短语正好是一条声明了的关系的，由本体 /// 直接绑上，记成本体判的（#1242）。不问模型，没配对话模型也照判；别的签名照旧留给对齐器。 /// 判定照旧是签名上的一行（0044 决定 3），人看得见、改得了；人的判定不动（`decide_on` diff --git a/web/src/pages/Chat.tsx b/web/src/pages/Chat.tsx index e3d07760b..5abe02f1e 100644 --- a/web/src/pages/Chat.tsx +++ b/web/src/pages/Chat.tsx @@ -76,7 +76,6 @@ import { stepDetail, stepLabel } from "../steps"; import { NextStep, nextStep, useReadiness } from "./NextStep"; import { KbMenuList } from "./KbMenuList"; import { DRAFT_KEY, fetchMe, lastKey } from "../authSession"; -import { KbMenuList } from "./KbMenuList"; /* `Turn` 定义在 liveAnswer 里：进行中的那一次也是一串 Turn， 而它必须活得比这个组件长（见那个文件顶上的说明） */ diff --git a/web/src/pages/KbSwitcher.tsx b/web/src/pages/KbSwitcher.tsx index 6fe8423c1..9ce534f62 100644 --- a/web/src/pages/KbSwitcher.tsx +++ b/web/src/pages/KbSwitcher.tsx @@ -20,7 +20,6 @@ import { } from "@/components/ui/popover"; import { KbMenuList } from "./KbMenuList"; import { fetchMe } from "../authSession"; -import { KbMenuList } from "./KbMenuList"; export function KbSwitcher({ kb,