diff --git a/Cargo.lock b/Cargo.lock index f2912d7..2ecf0c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1098,15 +1098,6 @@ dependencies = [ "shlex 2.0.1", ] -[[package]] -name = "cedarwood" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d910bedd62c24733263d0bed247460853c9d22e8956bd4cd964302095e04e90" -dependencies = [ - "smallvec", -] - [[package]] name = "cexpr" version = "0.6.0" @@ -4200,24 +4191,19 @@ checksum = "47f142fe24a9c9944451e8349de0a56af5f3e7226dc46f3ed4d4ecc0b85af75e" [[package]] name = "jieba-macros" -version = "0.8.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "348294e44ee7e3c42685da656490f8febc7359632544019621588902216da95c" -dependencies = [ - "phf_codegen 0.13.1", -] +checksum = "2987cf45b1bfdb4ff1f6a814b1aa2ad6361f4ff43bf353a9bd29f391a9568f70" [[package]] name = "jieba-rs" -version = "0.8.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "766bd7012aa5ba49411ebdf4e93bddd59b182d2918e085d58dec5bb9b54b7105" +checksum = "0a3a7d02ea31e6fc6f2fe10495dc7c5efcf381e1f21d7172f58ba792efa1c2fd" dependencies = [ - "cedarwood", + "bytecount", "include-flate", "jieba-macros", - "phf 0.13.1", - "regex", "rustc-hash", ] @@ -5626,16 +5612,6 @@ dependencies = [ "phf_shared 0.11.3", ] -[[package]] -name = "phf_codegen" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1" -dependencies = [ - "phf_generator 0.13.1", - "phf_shared 0.13.1", -] - [[package]] name = "phf_codegen" version = "0.14.0" diff --git a/Cargo.toml b/Cargo.toml index 9437838..1be3fe2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ candle-core = "=0.11.0" candle-transformers = "=0.11.0" candle-nn = "=0.11.0" -jieba-rs = "0.8" +jieba-rs = "0.11" dashmap = "6.2.1" rand = "0.10" parking_lot = "0.12" diff --git a/crates/soul-mem-algo/src/algo/forget/mask.rs b/crates/soul-mem-algo/src/algo/forget/mask.rs index b3ba684..643dc74 100644 --- a/crates/soul-mem-algo/src/algo/forget/mask.rs +++ b/crates/soul-mem-algo/src/algo/forget/mask.rs @@ -1,4 +1,4 @@ -use jieba_rs::Jieba; +use jieba_rs::{Jieba, Token}; use rand::SeedableRng; use rand::rngs::StdRng; use rand::seq::SliceRandom; @@ -24,7 +24,7 @@ pub struct MaskResult { /// 遮罩是确定性的:同一文本 + 同一缺失度 → 同一遮罩结果。 /// 使用 jieba 分词后,随机选取 `n` 个词替换为 `[masked]`。 pub fn mask_text(text: &str, missing_degree: f32, jieba: &Jieba) -> MaskResult { - let words: Vec<&str> = jieba.cut(text, true); + let words: Vec> = jieba.cut(text, true); let total = words.len(); if total == 0 || missing_degree <= 0.0 { @@ -61,7 +61,7 @@ pub fn mask_text(text: &str, missing_degree: f32, jieba: &Jieba) -> MaskResult { if masked_set.contains(&i) { MASK_WORD.to_string() } else { - w.to_string() + w.word.to_string() } }) .collect();