From a353b111578bcb3905e53b9d88e4bf6d6a5d62a7 Mon Sep 17 00:00:00 2001 From: YPARK Date: Thu, 1 Oct 2026 16:14:39 -0700 Subject: [PATCH] Frozen dictionaries: match lowercased locus keys to case-kept rows data-beans 0.6.5 to 0.6.23 wrote locus row keys with the chromosome lowercased (x_0_100 for chrX:0-100); 0.6.24 keeps case. A frozen feature dictionary saved from the old keys no longer matched the new rows, so X, Y, M and contig peaks dropped out of a frozen load. After the exact pass, a second pass matches a still-unmatched target row to the unused source row with the same case-folded locus key. Keys come from the shared locus grammar (feature_names::folded_locus_key), so only names that parse as loci take part. A folded key that names more than one source row or more than one target spelling is left unmatched and reported; exact matches are never replaced. The pass is skipped under FeatureNameKind::Exact. Release 0.6.25. --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/aux/feature_names.rs | 7 ++ src/aux/frozen_features.rs | 177 +++++++++++++++++++++++++++++++++++-- 4 files changed, 177 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fa5dc81..1135729 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -930,7 +930,7 @@ dependencies = [ [[package]] name = "data-beans" -version = "0.6.24" +version = "0.6.25" dependencies = [ "anyhow", "approx", diff --git a/Cargo.toml b/Cargo.toml index 1b4782d..8f155e7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ edition = "2021" readme = "README.md" # Own version (decoupled from workspace.version): data-beans evolves its CLI / # QC API independently of the shared utility crates. -version = "0.6.24" +version = "0.6.25" rust-version = "1.91" # Ship only the sources, README and license: nothing else in the working tree # (data stores, saved name lists, scratch files) can end up on crates.io. diff --git a/src/aux/feature_names.rs b/src/aux/feature_names.rs index 1ae3dcf..71fedd3 100644 --- a/src/aux/feature_names.rs +++ b/src/aux/feature_names.rs @@ -191,6 +191,13 @@ fn locus_key(name: &str) -> Option> { coordinates::parse_interval(name).map(|l| l.locus_key()) } +/// [`locus_key`] with its case folded: the form data-beans 0.6.5 to 0.6.23 +/// wrote (`x_0_100` for `chrX:0-100`). Only for matching such saved keys +/// back to case-kept rows; never a row name. +pub(crate) fn folded_locus_key(name: &str) -> Option> { + locus_key(name).map(|k| k.to_ascii_lowercase().into_boxed_str()) +} + /// Per-name rule of [`FeatureNameKind::Mixed`]: locus key, else the gene /// rule for `_` names, else the name unchanged. fn mixed_canonicalize(name: &str) -> Box { diff --git a/src/aux/frozen_features.rs b/src/aux/frozen_features.rs index b712fe8..6463c29 100644 --- a/src/aux/frozen_features.rs +++ b/src/aux/frozen_features.rs @@ -16,7 +16,7 @@ //! Name resolution goes through [`FeatureNameKind`] so `TGFB1` and //! `ENSG00000105329_TGFB1` resolve to the same row. -use crate::aux::feature_names::FeatureNameKind; +use crate::aux::feature_names::{folded_locus_key, FeatureNameKind}; use legume_numeric::matrix::traits::IoOps; use nalgebra::DMatrix; use rustc_hash::{FxHashMap, FxHashSet}; @@ -139,15 +139,33 @@ pub fn load_frozen_feature_host(args: FrozenLoadArgs) -> anyhow::Result> = args + .target_feature_names + .iter() + .map(|n| args.name_kind.canonicalize(n)) + .collect(); + let mut matched: Vec> = target_canon + .iter() + .map(|c| src_by_canon.get(c).copied()) + .collect(); + let n_legacy = if args.name_kind.is_exact() { + 0 + } else { + match_legacy_locus_case(&src_names, &target_canon, &mut matched) + }; + if n_legacy > 0 { + log::info!( + "{}: {} locus rows matched only with the chromosome case ignored \ + (a dictionary saved with lowercased locus names)", + args.dictionary_path, + n_legacy + ); } + let (keep_target_indices, keep_src_indices): (Vec, Vec) = matched + .iter() + .enumerate() + .filter_map(|(t, m)| m.map(|s| (t, s))) + .unzip(); anyhow::ensure!( !keep_target_indices.is_empty(), "No feature names matched between {} (n={}) and target axis (n={}) under {:?} \ @@ -210,6 +228,70 @@ pub fn load_frozen_feature_host(args: FrozenLoadArgs) -> anyhow::Result], + target_canon: &[Box], + matched: &mut [Option], +) -> usize { + // Folded key -> the unmatched target rows with their case-kept spelling. + let mut targets: FxHashMap, Vec<(usize, Box)>> = FxHashMap::default(); + for (t, canon) in target_canon.iter().enumerate() { + if matched[t].is_none() { + if let Some(folded) = folded_locus_key(canon) { + targets.entry(folded).or_default().push((t, canon.clone())); + } + } + } + if targets.is_empty() { + return 0; + } + let mut used = vec![false; src_names.len()]; + for &s in matched.iter().flatten() { + used[s] = true; + } + // Folded key -> unused source rows, for keys some target row needs. + let mut src: FxHashMap, Vec> = FxHashMap::default(); + for (i, name) in src_names.iter().enumerate() { + if !used[i] { + if let Some(folded) = folded_locus_key(name).filter(|k| targets.contains_key(k)) { + src.entry(folded).or_default().push(i); + } + } + } + let (mut n_matched, mut n_ambiguous) = (0usize, 0usize); + for (folded, rows) in &targets { + let Some(src_rows) = src.get(folded) else { + continue; + }; + // Any number of target rows may share one spelling, as in the exact pass. + let one_spelling = rows.iter().all(|(_, k)| *k == rows[0].1); + match src_rows.as_slice() { + &[s] if one_spelling => { + for &(t, _) in rows { + matched[t] = Some(s); + n_matched += 1; + } + } + _ => n_ambiguous += 1, + } + } + if n_ambiguous > 0 { + log::warn!( + "{n_ambiguous} locus keys fold onto more than one source row or target spelling \ + once the chromosome case is ignored; left unmatched rather than guessed" + ); + } + n_matched +} + #[cfg(test)] mod tests { use super::*; @@ -386,6 +468,83 @@ mod tests { assert!(err.to_string().contains("No feature names matched")); } + /// Load a one-column dictionary with rows `dict` onto `target`. + fn load_onto(dict: &[&str], target: &[&str], kind: FeatureNameKind) -> FrozenFeatureHost { + let dir = tempfile::tempdir().unwrap(); + let dict_path = dir.path().join("d.parquet").to_str().unwrap().to_string(); + let src = DMatrix::::from_fn(dict.len(), 1, |i, _| i as f32); + write_test_parquet(&dict_path, dict, "peak", &["h0"], &src); + let target: Vec> = target.iter().map(|s| (*s).into()).collect(); + load_frozen_feature_host(FrozenLoadArgs { + dictionary_path: &dict_path, + bias_path: None, + target_feature_names: &target, + name_kind: kind, + source_name_map: None, + }) + .unwrap() + } + + #[test] + fn lowercased_locus_dictionary_matches_case_kept_rows() { + let host = load_onto( + &["x_0_100", "1_0_100", "m_0_50", "un_ctg1v1_0_10", "y_0_100"], + &[ + "chrM:0-50", + "chrX:0-100", + "chr1:0-100", + "chrUn_CTG1v1:0-10", + "GENE1", + "chrY:0-100", + "Y_0_100", + ], + FeatureNameKind::Mixed, + ); + // chr1 matches exactly; M, X and the contig match with case ignored; + // both spellings of chrY share one key, so both take `y_0_100`. + assert_eq!(host.keep_target_indices, vec![0, 1, 2, 3, 5, 6]); + assert_eq!(host.keep_src_indices, vec![2, 0, 1, 3, 4, 4]); + } + + #[test] + fn a_key_with_two_case_spellings_is_not_guessed() { + let locus = FeatureNameKind::Locus { + merge_overlapping: false, + }; + // Two target spellings fold onto one source row. + let host = load_onto( + &["un_ctg_0_10", "A"], + &["chrUn_CTG:0-10", "UN_ctg_0_10", "A"], + locus.clone(), + ); + assert_eq!(host.keep_target_indices, vec![2]); + // Two source spellings fold onto one target row. + let host = load_onto( + &["Un_ctg_0_10", "un_CTG_0_10", "A"], + &["chrUn_CTG:0-10", "A"], + locus, + ); + assert_eq!(host.keep_target_indices, vec![1]); + } + + #[test] + fn an_exact_match_is_never_replaced_by_the_case_fallback() { + let host = load_onto( + &["X_0_100", "x_0_100"], + &["chrX:0-100"], + FeatureNameKind::Locus { + merge_overlapping: false, + }, + ); + assert_eq!(host.keep_src_indices, vec![0]); + } + + #[test] + fn the_exact_kind_skips_the_case_fallback() { + let host = load_onto(&["x_0_100", "A"], &["X_0_100", "A"], FeatureNameKind::Exact); + assert_eq!(host.keep_target_indices, vec![1]); + } + #[test] fn bias_loaded_when_provided() { let dir = tempfile::tempdir().unwrap();