Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions crates/utopia-core/src/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,8 @@ pub struct Source {
pub sync_interval_minutes: Option<i32>,
/// 标准 5 段 cron(服务器本地时区;与 sync_interval_minutes 互斥)
pub sync_cron: Option<String>,
/// NULL = 文档不衰减;对象的形状见 `utopia_store::decay::DecaySettings`(RC-003)
pub decay: Option<serde_json::Value>,
pub last_sync_at: Option<DateTime<Utc>>,
/// never | queued | running | ok | failed
pub last_sync_status: String,
Expand Down Expand Up @@ -197,6 +199,7 @@ mod source_extracts_tests {
icon: None,
sync_interval_minutes: None,
sync_cron: None,
decay: None,
last_sync_at: None,
last_sync_status: "never".into(),
last_sync_error: None,
Expand Down Expand Up @@ -353,6 +356,7 @@ pub struct SourceView {
pub icon: Option<String>,
pub sync_interval_minutes: Option<i32>,
pub sync_cron: Option<String>,
pub decay: Option<serde_json::Value>,
pub last_sync_at: Option<DateTime<Utc>>,
pub last_sync_status: String,
pub last_sync_error: Option<String>,
Expand Down
33 changes: 31 additions & 2 deletions crates/utopia-server/src/api/sources_routes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,18 @@ pub struct UpdateBody {
/// 出现 schedule 字段即整体覆盖调度(interval 与 cron 互斥;两者皆 null = 关闭定时)
#[serde(default)]
pub schedule: Option<ScheduleBody>,
/// 缺省 = 不动;`null` = 关掉衰减;对象 = 打开(RC-003)
#[serde(default, deserialize_with = "present_even_when_null")]
pub decay: Option<Option<utopia_store::decay::DecaySettings>>,
}

/// 区分「没写这个字段」和「写成了 null」:前者外层是 `None`,后者是 `Some(None)`。
fn present_even_when_null<'de, D, T>(deserializer: D) -> Result<Option<Option<T>>, D::Error>
where
D: serde::Deserializer<'de>,
T: Deserialize<'de>,
{
Option::<T>::deserialize(deserializer).map(Some)
}

#[derive(Deserialize)]
Expand Down Expand Up @@ -281,6 +293,7 @@ pub async fn update(
body.icon.as_deref(),
body.schedule
.map(|s| (s.sync_interval_minutes, s.sync_cron)),
body.decay.as_ref().map(Option::as_ref),
)
.await?;
state.emit_source(kb_id);
Expand All @@ -292,7 +305,11 @@ pub async fn update(
"source.updated",
"source",
Some(source_id),
json!({ "name": body.name, "config_changed": config.is_some() }),
json!({
"name": body.name,
"config_changed": config.is_some(),
"decay_changed": body.decay.is_some()
}),
)
.await;
Ok(Json(json!({ "source": mask_secrets(source) })))
Expand Down Expand Up @@ -893,9 +910,20 @@ mod statements_tests;

#[cfg(test)]
mod tests {
use super::{keep_secrets, validate_rss_config};
use super::{keep_secrets, validate_rss_config, UpdateBody};
use serde_json::json;

#[test]
fn an_update_tells_a_missing_decay_from_a_null_one() {
let absent: UpdateBody = serde_json::from_value(json!({ "name": "n" })).unwrap();
let null: UpdateBody = serde_json::from_value(json!({ "decay": null })).unwrap();
let object: UpdateBody =
serde_json::from_value(json!({ "decay": { "max_items": 9 } })).unwrap();
assert_eq!(absent.decay, None);
assert_eq!(null.decay, Some(None));
assert_eq!(object.decay.unwrap().unwrap().max_items, 9);
}

/// 认证判断写反是这类改动唯一值得担心的事(`ct_eq` 给的是 `Choice` 不是
/// `bool`,判反一次就是全放行且一切看起来正常),所以两个方向都钉住
#[test]
Expand Down Expand Up @@ -955,6 +983,7 @@ mod tests {
icon: None,
sync_interval_minutes: None,
sync_cron: None,
decay: None,
last_sync_at: None,
last_sync_status: "never".into(),
last_sync_error: None,
Expand Down
40 changes: 40 additions & 0 deletions crates/utopia-store/src/decay.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use utopia_core::AppError;

pub const DEFAULT_HALF_LIFE_DAYS: f64 = 60.0;
pub const MAX_HALF_LIFE_DAYS: f64 = 365.0;
Expand Down Expand Up @@ -37,6 +38,20 @@ impl Default for DecaySettings {
}
}

impl DecaySettings {
/// 数据库不拦的范围:半衰期 1 到 365 天,容量和陈旧天数至少 1。
pub fn validate(&self) -> Result<(), AppError> {
let half_life_ok = (1..=MAX_HALF_LIFE_DAYS as u32).contains(&self.half_life_days);
if !half_life_ok || self.max_items == 0 || self.stale_after_days == 0 {
return Err(AppError::Validation(format!(
"decay needs max_items >= 1, stale_after_days >= 1 and half_life_days within 1 to {}",
MAX_HALF_LIFE_DAYS as u32
)));
}
Ok(())
}
}

/// 打分用到的一个文档:`key` 是文档名,`value` 是它的文字。
pub struct Memory<'a> {
pub key: &'a str,
Expand Down Expand Up @@ -212,6 +227,31 @@ mod tests {
);
}

#[test]
fn a_setting_out_of_range_is_refused() {
let ok = DecaySettings::default();
assert!(ok.validate().is_ok());
let bad = [
DecaySettings {
max_items: 0,
..ok.clone()
},
DecaySettings {
half_life_days: 0,
..ok.clone()
},
DecaySettings {
half_life_days: 366,
..ok.clone()
},
DecaySettings {
stale_after_days: 0,
..ok.clone()
},
];
assert!(bad.iter().all(|settings| settings.validate().is_err()));
}

#[test]
fn a_misspelled_setting_is_refused() {
let parsed = serde_json::from_str::<DecaySettings>(r#"{"max_item": 5}"#);
Expand Down
20 changes: 18 additions & 2 deletions crates/utopia-store/src/sources.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ use utopia_core::models::{Role, Source, SourceKind, SourceView, SyncRun, SOURCE_
use utopia_core::{secrets, AppError, AppResult};
use uuid::Uuid;

use crate::decay::DecaySettings;

/// folder = 纯容器(上传/拖拽入内,无同步语义);url/rss = 拉取型;api = 推送型。
/// 本机目录监听(watch_folder)已否决——自部署用户看不到服务器磁盘;
/// 对象存储 / WebDAV / Notion 是它的替代形态(0013)。
Expand Down Expand Up @@ -89,7 +91,7 @@ pub async fn list(pool: &PgPool, kb_id: Uuid) -> AppResult<Vec<SourceView>> {
let rows: Vec<SourceView> = sqlx::query_as(&format!(
"SELECT listed_source.id, listed_source.kind, listed_source.name,
listed_source.config - $2::text[] AS config, listed_source.icon,
listed_source.sync_interval_minutes, listed_source.sync_cron,
listed_source.sync_interval_minutes, listed_source.sync_cron, listed_source.decay,
listed_source.last_sync_at, listed_source.last_sync_status,
listed_source.last_sync_error, listed_source.last_sync_added,
(SELECT count(*) FROM documents d
Expand Down Expand Up @@ -249,7 +251,18 @@ pub async fn update(
config: Option<&serde_json::Value>,
icon: Option<&str>,
schedule: Option<(Option<i32>, Option<String>)>,
decay: Option<Option<&DecaySettings>>,
) -> AppResult<Source> {
let decay = decay
.map(|settings| {
settings
.map(|settings| {
settings.validate()?;
serde_json::to_value(settings).map_err(|e| AppError::Other(e.into()))
})
.transpose()
})
.transpose()?;
let schedule = match schedule {
Some((interval, cron)) => {
let cron_norm = cron.as_deref().map(validate_cron).transpose()?;
Expand Down Expand Up @@ -287,7 +300,8 @@ pub async fn update(
config = COALESCE($3, config),
icon = COALESCE($4, icon),
sync_interval_minutes = CASE WHEN $5 THEN $6 ELSE sync_interval_minutes END,
sync_cron = CASE WHEN $5 THEN $7 ELSE sync_cron END
sync_cron = CASE WHEN $5 THEN $7 ELSE sync_cron END,
decay = CASE WHEN $8 THEN $9 ELSE decay END
WHERE id = $1 RETURNING *",
)
.bind(id)
Expand All @@ -297,6 +311,8 @@ pub async fn update(
.bind(schedule.is_some())
.bind(schedule.as_ref().and_then(|(i, _)| *i))
.bind(schedule.as_ref().and_then(|(_, c)| c.clone()))
.bind(decay.is_some())
.bind(decay.flatten())
.fetch_one(&mut *tx)
.await?;

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
//! 改来源时设置、保留、关掉衰减(RC-003),打在真库上。
//!
//! 要钉住的行为:
//!
//! - **不写就不动**:更新别的字段,衰减原样
//! - **写对象就打开**:存下完整的四个字段,之后默认值再变也不会动它
//! - **写 null 就关掉**
//! - **越界的值被拒绝**,库里的原值不变
//! - **列表看得到**
//!
//! 没有 `UTOPIA_DATABASE_URL` 时跳过而不是失败。自建自拆,绝不碰已有的库。

use sqlx::PgPool;
use utopia_store::decay::DecaySettings;
use utopia_store::sources;
use uuid::Uuid;

#[tokio::test]
async fn a_source_decay_is_set_and_cleared_by_update() -> anyhow::Result<()> {
let Some(url) = utopia_store::test_db::url() else {
return Ok(());
};
let pool = PgPool::connect(&url).await?;
utopia_store::db::migrate(&pool).await?;

let (org, ws, kb, source) = (
Uuid::now_v7(),
Uuid::now_v7(),
Uuid::now_v7(),
Uuid::now_v7(),
);
sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, 'source-decay-update-test')")
.bind(org)
.execute(&pool)
.await?;
sqlx::query(
"INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, 'source-decay-update-test')",
)
.bind(ws)
.bind(org)
.execute(&pool)
.await?;
sqlx::query(
"INSERT INTO knowledge_bases (id, workspace_id, name)
VALUES ($1, $2, 'source-decay-update-test')",
)
.bind(kb)
.bind(ws)
.execute(&pool)
.await?;
sqlx::query("INSERT INTO sources (id, kb_id, kind, name) VALUES ($1, $2, 'folder', 'f')")
.bind(source)
.bind(kb)
.execute(&pool)
.await?;

let run = async {
let renamed = sources::update(&pool, source, Some("g"), None, None, None, None).await?;
assert_eq!(renamed.decay, None, "a source starts without decay");

let settings = DecaySettings {
max_items: 3,
..DecaySettings::default()
};
let on =
sources::update(&pool, source, None, None, None, None, Some(Some(&settings))).await?;
assert_eq!(
on.decay,
Some(serde_json::json!({
"max_items": 3, "half_life_days": 60, "stale_after_days": 30, "min_age_days": 30
}))
);

let kept = sources::update(&pool, source, Some("h"), None, None, None, None).await?;
assert_eq!(kept.decay, on.decay, "another field leaves decay alone");

let too_long = DecaySettings {
half_life_days: 400,
..DecaySettings::default()
};
let refused =
sources::update(&pool, source, None, None, None, None, Some(Some(&too_long))).await;
assert!(refused.is_err());
let listed = sources::list(&pool, kb).await?;
assert_eq!(listed[0].decay, on.decay, "a refused value changes nothing");

let off = sources::update(&pool, source, None, None, None, None, Some(None)).await?;
assert_eq!(off.decay, None, "null turns decay off");
anyhow::Ok(())
}
.await;

sqlx::query("DELETE FROM organizations WHERE id = $1")
.bind(org)
.execute(&pool)
.await?;
run
}
1 change: 1 addition & 0 deletions crates/utopia-store/tests/store/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ mod a_signature_holds_on_every_path;
mod a_signature_vector_lasts_while_its_text_does;
mod a_similar_name_is_proposed_not_merged;
mod a_single_name_candidate_is_not_counted;
mod a_source_decay_is_set_and_cleared_by_update;
mod a_source_decays_only_when_asked;
mod a_source_kind_is_listed_once;
mod a_source_reaches_only_where_it_was_granted;
Expand Down
7 changes: 7 additions & 0 deletions docs/design/sources.md
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,13 @@ indexed and never extracted (`sources.config.extract`) [0036 d7].
HTML tables now are [0039].
- Backup and restore (#712), a 100k-document benchmark (#713) [0016 E].

## Decay

A source may let its documents decay [RC-003](../decisions/re-cinq/RC-003-a-source-may-decay.md).
`PATCH` a source with `decay` to set it: an object turns it on (a missing field takes Lore's value, so
`{}` is Lore's behaviour), `null` turns it off, and leaving the field out changes nothing. A new
source decays nothing.

## Open questions

- Whether low engine confidence on an OCR or transcript segment should cap facts [0040].
Expand Down
Loading