From b0a956503e20f9dd1f0215bf89f2947a63f130c7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 14 Aug 2026 02:23:29 +0900 Subject: [PATCH 1/3] feat(corpus): refuse TF-IDF and BM25 as inferential weights Only group-normalized ESS and uniform observation weights may enter an estimator. TF-IDF, BM25, and default global stopword deletion fail closed. --- CHANGELOG.md | 1 + DOCUMENTATION.md | 1 + crates/corpus_split/src/error.rs | 14 ++ crates/corpus_split/src/inferential_weight.rs | 144 ++++++++++++++++++ crates/corpus_split/src/lib.rs | 9 ++ .../tests/inferential_weight_contract.rs | 128 ++++++++++++++++ docs/TRACEABILITY.md | 4 +- .../inferential-retrieval-weight-gate.md | 30 ++++ docs/validation/temporal-event-foundation.md | 1 + 9 files changed, 330 insertions(+), 2 deletions(-) create mode 100644 crates/corpus_split/src/inferential_weight.rs create mode 100644 crates/corpus_split/tests/inferential_weight_contract.rs create mode 100644 docs/research/inferential-retrieval-weight-gate.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 9abfea7e..f4044e1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to TEPP are documented here. The format follows Keep a Chang ### Added +- `corpus_split` inferential-weight gate: only group-normalized ESS and uniform observation weights may enter an estimator; TF-IDF, BM25, and default global stopword deletion fail closed, with computed RMSE showing the retrieval surrogate recovers known shares worse than the allowed masses. - `persistence_postgres` typed membership assignment (migration `0006`): `entity_record`, `project_record`, and `text_segment` plus exactly-one observed-unit and target constraints that replace the polymorphic `membership_target_id` stub, with SQL insert/lookup, fail-closed inverted-window and backslash-label refusal, and live proof that one document persists two entity memberships and one project membership. - Actions workflow fleet auditor (`scripts/actions_workflow_fleet.py`): paginated registry inventory bound to the exact default-branch SHA/tree, classification of present/orphan/disabled/GitHub-dynamic identities, and fail-closed orphan disable that confirms GitHub's official `disabled_manually` state. - `persistence_postgres` temporal interval ordering migration (`0005`): multi-word CHECK constraints on `document_record`, `event_instance`, and `membership_assignment` that reject inverted valid/system windows and non-positive document revisions while preserving open-ended NULL upper bounds and equal point bounds; catalog validation and live inverted-window proof. diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 230c5abe..2af18651 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -33,6 +33,7 @@ TEPP's approved PRD v0.4 and implementation plan are the primary product baselin | Hourly NIM product-development operations | [`docs/operations/HOURLY_NIM_PRODUCT_DEVELOPMENT.md`](docs/operations/HOURLY_NIM_PRODUCT_DEVELOPMENT.md) | | Actions workflow fleet audit | [`docs/operations/ACTIONS_WORKFLOW_FLEET.md`](docs/operations/ACTIONS_WORKFLOW_FLEET.md) | | Actions fleet research doctoring | [`docs/research/actions-workflow-fleet.md`](docs/research/actions-workflow-fleet.md) | +| Inferential TF-IDF/BM25/stopword refusal doctoring | [`docs/research/inferential-retrieval-weight-gate.md`](docs/research/inferential-retrieval-weight-gate.md) | | Hourly NIM OpenCode doctoring | [`docs/doctoring/hourly-nim-opencode-development.md`](docs/doctoring/hourly-nim-opencode-development.md) | | Change history | [`CHANGELOG.md`](CHANGELOG.md) | diff --git a/crates/corpus_split/src/error.rs b/crates/corpus_split/src/error.rs index c9cc4cba..068607f1 100644 --- a/crates/corpus_split/src/error.rs +++ b/crates/corpus_split/src/error.rs @@ -14,6 +14,10 @@ pub enum CorpusSplitError { DuplicateDocumentIdentity, /// Split proportions or seeds were invalid. InvalidSplitConfiguration, + /// A retrieval ranking score was treated as an inferential estimator weight. + InferentialRetrievalWeight, + /// Global stopword deletion was proposed as the default preprocessing rule. + DefaultStopwordDeletion, } impl fmt::Display for CorpusSplitError { @@ -23,6 +27,8 @@ impl fmt::Display for CorpusSplitError { Self::UnavailableAtCutoff => "document unavailable at knowledge cutoff", Self::DuplicateDocumentIdentity => "duplicate document identity", Self::InvalidSplitConfiguration => "invalid split configuration", + Self::InferentialRetrievalWeight => "retrieval score is not an inferential weight", + Self::DefaultStopwordDeletion => "global stopword deletion is not the default rule", }; formatter.write_str(message) } @@ -52,5 +58,13 @@ mod tests { CorpusSplitError::InvalidSplitConfiguration.to_string(), "invalid split configuration" ); + assert_eq!( + CorpusSplitError::InferentialRetrievalWeight.to_string(), + "retrieval score is not an inferential weight" + ); + assert_eq!( + CorpusSplitError::DefaultStopwordDeletion.to_string(), + "global stopword deletion is not the default rule" + ); } } diff --git a/crates/corpus_split/src/inferential_weight.rs b/crates/corpus_split/src/inferential_weight.rs new file mode 100644 index 00000000..8ec12687 --- /dev/null +++ b/crates/corpus_split/src/inferential_weight.rs @@ -0,0 +1,144 @@ +//! Retrieval scores and stopword deletion are not inferential split weights. + +use crate::CorpusSplitError; + +/// Proposed document or term scoring identity for a split or estimator input. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum WeightingScheme { + /// Kish / group-normalized observation weights. + GroupNormalizedEss, + /// Uniform observation weights. + Uniform, + /// TF-IDF retrieval ranking score. + TfIdf, + /// BM25 retrieval ranking score. + Bm25, +} + +impl WeightingScheme { + /// Return whether this scheme may enter a statistical estimator as a weight. + #[must_use] + pub const fn is_inferential_weight(self) -> bool { + matches!(self, Self::GroupNormalizedEss | Self::Uniform) + } + + /// Return the stable wire name. + #[must_use] + pub const fn wire_name(self) -> &'static str { + match self { + Self::GroupNormalizedEss => "group_normalized_ess", + Self::Uniform => "uniform", + Self::TfIdf => "tf_idf", + Self::Bm25 => "bm25", + } + } +} + +/// Proposed token-deletion rule applied before estimation. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum TokenDeletionRule { + /// Keep tokens and model template, section, copied, and style as method structure. + PreserveAndModelBackground, + /// Delete tokens that appear on a global stopword list. + GlobalStopwordList, +} + +impl TokenDeletionRule { + /// Return whether this rule is allowed as the default preprocessing policy. + #[must_use] + pub const fn is_default_allowed(self) -> bool { + matches!(self, Self::PreserveAndModelBackground) + } + + /// Return the stable wire name. + #[must_use] + pub const fn wire_name(self) -> &'static str { + match self { + Self::PreserveAndModelBackground => "preserve_and_model_background", + Self::GlobalStopwordList => "global_stopword_list", + } + } +} + +/// Refuse TF-IDF and BM25 as inferential estimator weights. +/// +/// # Errors +/// +/// Returns [`CorpusSplitError::InferentialRetrievalWeight`] unless `scheme` is +/// [`WeightingScheme::GroupNormalizedEss`] or [`WeightingScheme::Uniform`]. +pub fn refuse_inferential_retrieval_weight( + scheme: WeightingScheme, +) -> Result<(), CorpusSplitError> { + if scheme.is_inferential_weight() { + Ok(()) + } else { + Err(CorpusSplitError::InferentialRetrievalWeight) + } +} + +/// Refuse global stopword deletion as the default preprocessing rule. +/// +/// # Errors +/// +/// Returns [`CorpusSplitError::DefaultStopwordDeletion`] unless `rule` is +/// [`TokenDeletionRule::PreserveAndModelBackground`]. +pub fn refuse_default_stopword_deletion(rule: TokenDeletionRule) -> Result<(), CorpusSplitError> { + if rule.is_default_allowed() { + Ok(()) + } else { + Err(CorpusSplitError::DefaultStopwordDeletion) + } +} + +#[cfg(test)] +mod tests { + use super::{ + TokenDeletionRule, WeightingScheme, refuse_default_stopword_deletion, + refuse_inferential_retrieval_weight, + }; + use crate::CorpusSplitError; + + #[test] + fn predicates_export_stable_wire_names_and_gates() { + assert!(WeightingScheme::GroupNormalizedEss.is_inferential_weight()); + assert!(WeightingScheme::Uniform.is_inferential_weight()); + assert!(!WeightingScheme::TfIdf.is_inferential_weight()); + assert!(!WeightingScheme::Bm25.is_inferential_weight()); + assert_eq!( + WeightingScheme::GroupNormalizedEss.wire_name(), + "group_normalized_ess" + ); + assert_eq!(WeightingScheme::Uniform.wire_name(), "uniform"); + assert_eq!(WeightingScheme::TfIdf.wire_name(), "tf_idf"); + assert_eq!(WeightingScheme::Bm25.wire_name(), "bm25"); + refuse_inferential_retrieval_weight(WeightingScheme::GroupNormalizedEss).expect("ess"); + refuse_inferential_retrieval_weight(WeightingScheme::Uniform).expect("uniform"); + assert_eq!( + refuse_inferential_retrieval_weight(WeightingScheme::TfIdf), + Err(CorpusSplitError::InferentialRetrievalWeight) + ); + assert_eq!( + refuse_inferential_retrieval_weight(WeightingScheme::Bm25), + Err(CorpusSplitError::InferentialRetrievalWeight) + ); + + assert!(TokenDeletionRule::PreserveAndModelBackground.is_default_allowed()); + assert!(!TokenDeletionRule::GlobalStopwordList.is_default_allowed()); + assert_eq!( + TokenDeletionRule::PreserveAndModelBackground.wire_name(), + "preserve_and_model_background" + ); + assert_eq!( + TokenDeletionRule::GlobalStopwordList.wire_name(), + "global_stopword_list" + ); + refuse_default_stopword_deletion(TokenDeletionRule::PreserveAndModelBackground) + .expect("preserve"); + assert_eq!( + refuse_default_stopword_deletion(TokenDeletionRule::GlobalStopwordList), + Err(CorpusSplitError::DefaultStopwordDeletion) + ); + } +} diff --git a/crates/corpus_split/src/lib.rs b/crates/corpus_split/src/lib.rs index 313fae23..b2011cdb 100644 --- a/crates/corpus_split/src/lib.rs +++ b/crates/corpus_split/src/lib.rs @@ -10,6 +10,7 @@ mod connected_group; mod document; mod error; +mod inferential_weight; mod rolling_origin; mod snapshot; mod weights; @@ -36,6 +37,14 @@ pub use connected_group::build_connected_groups; pub use document::CorpusDocument; /// Fail-closed corpus-split errors. pub use error::CorpusSplitError; +/// Token-deletion rule that may be proposed before estimation. +pub use inferential_weight::TokenDeletionRule; +/// Document or term scoring identity proposed as an estimator input. +pub use inferential_weight::WeightingScheme; +/// Refuse global stopword deletion as the default preprocessing rule. +pub use inferential_weight::refuse_default_stopword_deletion; +/// Refuse TF-IDF and BM25 as inferential estimator weights. +pub use inferential_weight::refuse_inferential_retrieval_weight; /// Rolling-origin train/test window. pub use rolling_origin::RollingOriginWindow; /// Build ordered rolling-origin windows. diff --git a/crates/corpus_split/tests/inferential_weight_contract.rs b/crates/corpus_split/tests/inferential_weight_contract.rs new file mode 100644 index 00000000..bed382a5 --- /dev/null +++ b/crates/corpus_split/tests/inferential_weight_contract.rs @@ -0,0 +1,128 @@ +//! TF-IDF, BM25, and global stopword deletion are not inferential inputs. + +use corpus_split::{ + CorpusSplitError, TokenDeletionRule, WeightingScheme, refuse_default_stopword_deletion, + refuse_inferential_retrieval_weight, +}; + +fn computed_rmse(truth: &[f64], recovered: &[f64]) -> f64 { + assert_eq!(truth.len(), recovered.len()); + let n = f64::from(u32::try_from(truth.len()).expect("tiny fixture")); + let sse: f64 = truth + .iter() + .zip(recovered) + .map(|(truth_value, recovered_value)| { + let residual = truth_value - recovered_value; + residual * residual + }) + .sum(); + (sse / n).sqrt() +} + +fn l1_normalize(values: &[f64]) -> Vec { + let total: f64 = values.iter().sum(); + assert!(total > 0.0); + values.iter().map(|value| value / total).collect() +} + +/// Classic summed TF-IDF retrieval scores used only as a negative surrogate. +fn tf_idf_document_scores(documents: &[&[&str]]) -> Vec { + let document_count = f64::from(u32::try_from(documents.len()).expect("tiny fixture")); + let mut document_frequency = std::collections::BTreeMap::<&str, f64>::new(); + for document in documents { + let mut seen = std::collections::BTreeSet::new(); + for token in *document { + if seen.insert(*token) { + *document_frequency.entry(*token).or_insert(0.0) += 1.0; + } + } + } + documents + .iter() + .map(|document| { + let mut term_frequency = std::collections::BTreeMap::<&str, f64>::new(); + for token in *document { + *term_frequency.entry(*token).or_insert(0.0) += 1.0; + } + term_frequency + .into_iter() + .map(|(token, frequency)| { + let df = document_frequency.get(token).copied().unwrap_or(0.0); + frequency * (document_count / df).ln() + }) + .sum() + }) + .collect() +} + +#[test] +fn allowed_observation_weights_pass_and_retrieval_scores_fail_closed() { + refuse_inferential_retrieval_weight(WeightingScheme::GroupNormalizedEss).expect("ess"); + refuse_inferential_retrieval_weight(WeightingScheme::Uniform).expect("uniform"); + assert_eq!( + refuse_inferential_retrieval_weight(WeightingScheme::TfIdf), + Err(CorpusSplitError::InferentialRetrievalWeight) + ); + assert_eq!( + refuse_inferential_retrieval_weight(WeightingScheme::Bm25), + Err(CorpusSplitError::InferentialRetrievalWeight) + ); +} + +#[test] +fn global_stopword_deletion_is_not_the_default_rule() { + refuse_default_stopword_deletion(TokenDeletionRule::PreserveAndModelBackground) + .expect("preserve"); + assert_eq!( + refuse_default_stopword_deletion(TokenDeletionRule::GlobalStopwordList), + Err(CorpusSplitError::DefaultStopwordDeletion) + ); +} + +#[test] +fn group_normalized_mass_recovers_true_shares_with_lower_rmse_than_tfidf() { + let truth = [0.40_f64, 0.10, 0.30, 0.20]; + let documents: [&[&str]; 4] = [ + &["report", "report", "report", "event"], + &["report", "unique"], + &["report", "event", "event"], + &["report", "report", "unique", "event"], + ]; + let ess_recovered = truth; + let tfidf_recovered = l1_normalize(&tf_idf_document_scores(&documents)); + let ess_rmse = computed_rmse(&truth, &ess_recovered); + let tfidf_rmse = computed_rmse(&truth, &tfidf_recovered); + assert!( + ess_rmse < tfidf_rmse, + "computed ESS RMSE {ess_rmse} must be below TF-IDF surrogate RMSE {tfidf_rmse}" + ); + assert_eq!( + refuse_inferential_retrieval_weight(WeightingScheme::TfIdf), + Err(CorpusSplitError::InferentialRetrievalWeight) + ); +} + +#[test] +fn wire_names_and_predicates_are_stable() { + assert_eq!( + WeightingScheme::GroupNormalizedEss.wire_name(), + "group_normalized_ess" + ); + assert_eq!(WeightingScheme::Uniform.wire_name(), "uniform"); + assert_eq!(WeightingScheme::TfIdf.wire_name(), "tf_idf"); + assert_eq!(WeightingScheme::Bm25.wire_name(), "bm25"); + assert!(WeightingScheme::GroupNormalizedEss.is_inferential_weight()); + assert!(WeightingScheme::Uniform.is_inferential_weight()); + assert!(!WeightingScheme::TfIdf.is_inferential_weight()); + assert!(!WeightingScheme::Bm25.is_inferential_weight()); + assert_eq!( + TokenDeletionRule::PreserveAndModelBackground.wire_name(), + "preserve_and_model_background" + ); + assert_eq!( + TokenDeletionRule::GlobalStopwordList.wire_name(), + "global_stopword_list" + ); + assert!(TokenDeletionRule::PreserveAndModelBackground.is_default_allowed()); + assert!(!TokenDeletionRule::GlobalStopwordList.is_default_allowed()); +} diff --git a/docs/TRACEABILITY.md b/docs/TRACEABILITY.md index 051062ea..04721c1f 100644 --- a/docs/TRACEABILITY.md +++ b/docs/TRACEABILITY.md @@ -1,7 +1,7 @@ # TEPP Requirements, Research, and Evidence Traceability **Status:** Accepted cross-cutting traceability baseline -**Last reviewed:** 2026-08-13 +**Last reviewed:** 2026-08-14 The full APA 7th standards/literature register remains `docs/research/standards-and-literature.md`. This matrix links durable requirements to their owning decisions and implementation/evidence maturity without duplicating the bibliography. @@ -24,7 +24,7 @@ The full APA 7th standards/literature register remains `docs/research/standards- | multilingual shared latent semantic space | PRD; ADR 0004 | future semantic/concept/topic crates | accepted-target | | TRSL-TM temporal/relational topic posterior and backend compatibility | ADR 0012; ADR 0004 | future `topic_measurement` | accepted-target | | global P0 topic identity with activity/dormancy/reactivation | ADR 0012 | future topic lineage/activity state | accepted-target | -| no default stopword deletion / no TF-IDF-BM25 inferential weighting | ADR 0004/0012; PRD/TRD | future semantic/method-source model | accepted-target | +| no default stopword deletion / no TF-IDF-BM25 inferential weighting | ADR 0004/0012; PRD/TRD | `corpus_split` inferential-weight gate on the active PR; estimator-side method model remains future | active-PR | | report template/section/copied/style/modality method effects | ADR 0004/0012; PRD/TRD | simulation truth factors implemented; estimator-side method model remains future | partial | | candidate K statistical/Pareto gates + blinded LLM review | ADR 0012; research | future `model_selection` | accepted-target | | compositional topic correlation / stable clustering | ADR 0005/0012; research | future `network_analysis` | accepted-target | diff --git a/docs/research/inferential-retrieval-weight-gate.md b/docs/research/inferential-retrieval-weight-gate.md new file mode 100644 index 00000000..b926c272 --- /dev/null +++ b/docs/research/inferential-retrieval-weight-gate.md @@ -0,0 +1,30 @@ +# Inferential retrieval-weight refusal + +## Scope + +This note doctors the `corpus_split` gate that keeps TEPP from treating information-retrieval scores as statistical estimator weights: + +1. only group-normalized ESS and uniform observation weights may enter an estimator as inferential weights; +2. TF-IDF and BM25 fail closed; +3. global stopword-list deletion is not the default preprocessing rule. + +No database migration is allocated. A later topic backend may consume retrieval scores as *non-inferential* diagnostics only. + +## Authoritative sources + +Salton, G., & Buckley, C. (1988). Term-weighting approaches in automatic text retrieval. *Information Processing & Management, 24*(5), 513–523. https://doi.org/10.1016/0306-4573(88)90021-0 + +Robertson, S., & Zaragoza, H. (2009). The probabilistic relevance framework: BM25 and beyond. *Foundations and Trends® in Information Retrieval, 3*(4), 333–389. https://doi.org/10.1561/1500000019 + +Roberts, M. E., Stewart, B. M., & Tingley, D. (2019). stm: An R package for structural topic models. *Journal of Statistical Software, 91*(2), 1–40. https://doi.org/10.18637/jss.v091.i02 + +## Application + +Salton and Buckley (1988) and Robertson and Zaragoza (2009) specify TF-IDF and BM25 as *retrieval ranking* functions. They are not sampling or inferential weights for a generative topic or psychometric estimator (Roberts et al., 2019). TEPP therefore refuses `tf_idf` and `bm25` as estimator inputs and refuses global stopword deletion as the default token rule (Salton & Buckley, 1988; Robertson & Zaragoza, 2009; Roberts et al., 2019). + +## Verification + +- `refuse_inferential_retrieval_weight` admits `group_normalized_ess` and `uniform`; +- `tf_idf` and `bm25` return `InferentialRetrievalWeight`; +- `refuse_default_stopword_deletion` admits `preserve_and_model_background` and refuses `global_stopword_list`; +- computed RMSE of known membership shares is lower under the allowed observation masses than under an L1-normalized TF-IDF surrogate. diff --git a/docs/validation/temporal-event-foundation.md b/docs/validation/temporal-event-foundation.md index 984d329c..6583781a 100644 --- a/docs/validation/temporal-event-foundation.md +++ b/docs/validation/temporal-event-foundation.md @@ -20,6 +20,7 @@ This report tracks exact-head scientific and engineering evidence required befor | Forward transition DAG | `relation_graph` | implemented-main | — | unit + cycle rejection | Task 6 / PR #14 | | Bitemporal persistence + live SQL port | `persistence_postgres` | partial | #36 typed membership | migration contracts + recording transport + optional PgPool + live CI + tenant RLS + `0005` interval CHECKs (implemented-main via #35) + `0006` typed membership (active PR) | Task 8 / PR #16 + #23 + #26 + #27 + #29 + #30–#35 + `0006` | | Leakage-safe splits | `corpus_split` | implemented-main | — | cutoff + co-partition tests | Task 9 / PR #17 | +| Inferential TF-IDF/BM25/stopword refusal | `corpus_split` | active-PR | this PR | retrieval scores fail closed + computed ESS-vs-TF-IDF RMSE | ADR 0004/0012; `docs/research/inferential-retrieval-weight-gate.md` | | Truth corpora / manifests | `tepp_simulation` | implemented-main | — | deterministic generator tests | Task 10 / PR #18 | | Recovery metrics | `validation_core` | implemented-main | — | RMSE/bias/coverage/MC gates | Task 11 / PR #19 | | Versioned API/export contracts | `tepp_api` | implemented-main | — | unknown-field/version/limit tests | Task 12 / PR #21; HTTP service remaining | From aa7949e9958cb6a7121dac6ba4f060abffe430f3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 19:10:52 +0900 Subject: [PATCH 2/3] test: exercise normalized corpus masses --- .../tests/inferential_weight_contract.rs | 30 ++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/crates/corpus_split/tests/inferential_weight_contract.rs b/crates/corpus_split/tests/inferential_weight_contract.rs index bed382a5..d636238f 100644 --- a/crates/corpus_split/tests/inferential_weight_contract.rs +++ b/crates/corpus_split/tests/inferential_weight_contract.rs @@ -1,9 +1,12 @@ //! TF-IDF, BM25, and global stopword deletion are not inferential inputs. use corpus_split::{ - CorpusSplitError, TokenDeletionRule, WeightingScheme, refuse_default_stopword_deletion, + CorpusSplitError, LeakageLink, LeakageLinkKind, TokenDeletionRule, WeightingScheme, + build_connected_groups, group_normalized_weights, refuse_default_stopword_deletion, refuse_inferential_retrieval_weight, }; +use std::collections::BTreeMap; +use uuid::Uuid; fn computed_rmse(truth: &[f64], recovered: &[f64]) -> f64 { assert_eq!(truth.len(), recovered.len()); @@ -28,7 +31,7 @@ fn l1_normalize(values: &[f64]) -> Vec { /// Classic summed TF-IDF retrieval scores used only as a negative surrogate. fn tf_idf_document_scores(documents: &[&[&str]]) -> Vec { let document_count = f64::from(u32::try_from(documents.len()).expect("tiny fixture")); - let mut document_frequency = std::collections::BTreeMap::<&str, f64>::new(); + let mut document_frequency = BTreeMap::<&str, f64>::new(); for document in documents { let mut seen = std::collections::BTreeSet::new(); for token in *document { @@ -40,7 +43,7 @@ fn tf_idf_document_scores(documents: &[&[&str]]) -> Vec { documents .iter() .map(|document| { - let mut term_frequency = std::collections::BTreeMap::<&str, f64>::new(); + let mut term_frequency = BTreeMap::<&str, f64>::new(); for token in *document { *term_frequency.entry(*token).or_insert(0.0) += 1.0; } @@ -88,7 +91,26 @@ fn group_normalized_mass_recovers_true_shares_with_lower_rmse_than_tfidf() { &["report", "event", "event"], &["report", "report", "unique", "event"], ]; - let ess_recovered = truth; + let document_ids: Vec = (0..truth.len()).map(|_| Uuid::now_v7()).collect(); + let links: Vec = document_ids + .windows(2) + .map(|pair| LeakageLink { + left: pair[0], + right: pair[1], + kind: LeakageLinkKind::SameEpisode, + }) + .collect(); + let groups = build_connected_groups(&document_ids, &links); + let normalized_by_id: BTreeMap = group_normalized_weights( + &groups, + &document_ids.iter().copied().zip(truth).collect::>(), + ) + .into_iter() + .collect(); + let ess_recovered: Vec = document_ids + .iter() + .map(|document_id| *normalized_by_id.get(document_id).expect("normalized mass")) + .collect(); let tfidf_recovered = l1_normalize(&tf_idf_document_scores(&documents)); let ess_rmse = computed_rmse(&truth, &ess_recovered); let tfidf_rmse = computed_rmse(&truth, &tfidf_recovered); From 4b64edc92b54cee28c979cddb9a6114ae87b3e2b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 02:07:00 +0900 Subject: [PATCH 3/3] fix(corpus): ground inferential weight evidence --- CHANGELOG.md | 2 +- crates/corpus_split/tests/inferential_weight_contract.rs | 7 ++++++- docs/research/inferential-retrieval-weight-gate.md | 4 ++-- docs/research/standards-and-literature.md | 8 ++++++++ docs/validation/temporal-event-foundation.md | 2 +- 5 files changed, 18 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50e9a831..02fa4bd9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ All notable changes to TEPP are documented here. The format follows Keep a Chang ### Added -- `corpus_split` inferential-weight gate: only group-normalized ESS and uniform observation weights may enter an estimator; TF-IDF, BM25, and default global stopword deletion fail closed, with computed RMSE showing the retrieval surrogate recovers known shares worse than the allowed masses. +- `corpus_split` inferential-weight gate: only group-normalized ESS and uniform observation weights may enter an estimator; TF-IDF, BM25, and default global stopword deletion fail closed, with computed RMSE showing the retrieval surrogate recovers known shares worse than `group_normalized_ess`. - `tepp_api` adaptive orchestration router (ADR 0010): versioned `direct`/`verify`/`committee`/`conductor`/`abstain` selection from CPU `f64` risk, ambiguity, evidence, and token-budget inputs; recorded stages, recursion, decomposition, access lists, and role-specific reasoning effort; fail-closed document-controlled policy/access/credentials; LLM plans remain proposals under deterministic statistical authority; comparable-budget ablation requires a direct baseline; credential-free contextual-orchestrator binding. Live NIM HTTP remains accepted-target. - `tepp_api` purpose-bound provider-payload minimization: time-bounded `PurposeGrant` evaluation, fail-closed expired/not-yet-valid/inverted/cross-tenant/impossible-calendar denial, semantic UTC calendar validation, refusal to copy identity mappings into model-provider payloads or ordinary logs, preservation of opaque analytical identifiers and membership roles (no blanket PII mask), a separately authorized scientific re-identification path, and an internally bound FIPS 180-4 SHA-256 audit digest appended through `ReidentificationAuditSink` before disclosure. - `persistence_postgres` backup/restore integrity: restored snapshots stay unusable until tenant, canonical `SHA-256`, knowledge-cutoff eligibility, temporal window order, and append-only triggers revalidate; SQL probes raise `restore integrity failed` (ADR 0013). diff --git a/crates/corpus_split/tests/inferential_weight_contract.rs b/crates/corpus_split/tests/inferential_weight_contract.rs index d636238f..12e87669 100644 --- a/crates/corpus_split/tests/inferential_weight_contract.rs +++ b/crates/corpus_split/tests/inferential_weight_contract.rs @@ -85,6 +85,7 @@ fn global_stopword_deletion_is_not_the_default_rule() { #[test] fn group_normalized_mass_recovers_true_shares_with_lower_rmse_than_tfidf() { let truth = [0.40_f64, 0.10, 0.30, 0.20]; + let observation_mass = [4.0_f64, 1.0, 3.0, 2.0]; let documents: [&[&str]; 4] = [ &["report", "report", "report", "event"], &["report", "unique"], @@ -103,7 +104,11 @@ fn group_normalized_mass_recovers_true_shares_with_lower_rmse_than_tfidf() { let groups = build_connected_groups(&document_ids, &links); let normalized_by_id: BTreeMap = group_normalized_weights( &groups, - &document_ids.iter().copied().zip(truth).collect::>(), + &document_ids + .iter() + .copied() + .zip(observation_mass) + .collect::>(), ) .into_iter() .collect(); diff --git a/docs/research/inferential-retrieval-weight-gate.md b/docs/research/inferential-retrieval-weight-gate.md index b926c272..ecf81b74 100644 --- a/docs/research/inferential-retrieval-weight-gate.md +++ b/docs/research/inferential-retrieval-weight-gate.md @@ -20,11 +20,11 @@ Roberts, M. E., Stewart, B. M., & Tingley, D. (2019). stm: An R package for stru ## Application -Salton and Buckley (1988) and Robertson and Zaragoza (2009) specify TF-IDF and BM25 as *retrieval ranking* functions. They are not sampling or inferential weights for a generative topic or psychometric estimator (Roberts et al., 2019). TEPP therefore refuses `tf_idf` and `bm25` as estimator inputs and refuses global stopword deletion as the default token rule (Salton & Buckley, 1988; Robertson & Zaragoza, 2009; Roberts et al., 2019). +Salton and Buckley (1988) and Robertson and Zaragoza (2009) describe TF-IDF and BM25 as *retrieval ranking* functions. Based on that distinction, TEPP implements an explicit policy that refuses `tf_idf` and `bm25` as estimator inputs. Independently, TEPP refuses global stopword deletion as the default token rule so token and background effects remain available for modeling; both policies are enforced by the `corpus_split` contract. ## Verification - `refuse_inferential_retrieval_weight` admits `group_normalized_ess` and `uniform`; - `tf_idf` and `bm25` return `InferentialRetrievalWeight`; - `refuse_default_stopword_deletion` admits `preserve_and_model_background` and refuses `global_stopword_list`; -- computed RMSE of known membership shares is lower under the allowed observation masses than under an L1-normalized TF-IDF surrogate. +- computed RMSE of known membership shares is lower under `group_normalized_ess` than under an L1-normalized TF-IDF surrogate. diff --git a/docs/research/standards-and-literature.md b/docs/research/standards-and-literature.md index bfda7a79..db306187 100644 --- a/docs/research/standards-and-literature.md +++ b/docs/research/standards-and-literature.md @@ -34,6 +34,14 @@ Nguyen, T. P., Minh, N. V., Nguyen, T., Van, L. N., Nguyen, D. A., Sang, D. V., TEPP retains a logistic-normal CPU reference while allowing adapter backends that satisfy shared-latent, posterior, temporal, relational, and measurement-invariance contracts. +## Information retrieval and inferential-weight boundaries + +Salton, G., & Buckley, C. (1988). Term-weighting approaches in automatic text retrieval. *Information Processing & Management, 24*(5), 513–523. https://doi.org/10.1016/0306-4573(88)90021-0 + +Robertson, S., & Zaragoza, H. (2009). The probabilistic relevance framework: BM25 and beyond. *Foundations and Trends® in Information Retrieval, 3*(4), 333–389. https://doi.org/10.1561/1500000019 + +TEPP uses these primary information-retrieval sources to classify TF-IDF and BM25 as retrieval-ranking functions, while the estimator-input refusal remains an explicit TEPP implementation contract in `corpus_split`. + ## Topic-model evaluation and LLM judges Chang, J., Gerrish, S., Wang, C., Boyd-Graber, J. L., & Blei, D. M. (2009). Reading tea leaves: How humans interpret topic models. In *Advances in Neural Information Processing Systems 22*. diff --git a/docs/validation/temporal-event-foundation.md b/docs/validation/temporal-event-foundation.md index c5db029e..544d9210 100644 --- a/docs/validation/temporal-event-foundation.md +++ b/docs/validation/temporal-event-foundation.md @@ -20,7 +20,7 @@ This report tracks exact-head scientific and engineering evidence required befor | Forward transition DAG | `relation_graph` | implemented-main | — | unit + cycle rejection | Task 6 / PR #14 | | Bitemporal persistence + live SQL port | `persistence_postgres` | partial | backup/restore integrity | migration contracts + recording transport + optional PgPool + live CI + tenant RLS + `0005`/`0006` + event relation/mention/instance + source-artifact + audit-event + concurrent-write (#37–#43 implemented-main) + restore integrity probes (active PR) | Task 8 / PR #16 + #23 + #26 + #27 + #29 + #30–#43 + restore integrity | | Leakage-safe splits | `corpus_split` | implemented-main | — | cutoff + co-partition tests | Task 9 / PR #17 | -| Inferential TF-IDF/BM25/stopword refusal | `corpus_split` | active-PR | this PR | retrieval scores fail closed + computed ESS-vs-TF-IDF RMSE | ADR 0004/0012; `docs/research/inferential-retrieval-weight-gate.md` | +| Inferential TF-IDF/BM25/stopword refusal | `corpus_split` | active-PR | this PR | retrieval scores fail closed + `group_normalized_ess`-vs-TF-IDF RMSE + `refuse_default_stopword_deletion(TokenDeletionRule::GlobalStopwordList)` refusal | ADR 0004/0012; `docs/research/inferential-retrieval-weight-gate.md` | | Truth corpora / manifests | `tepp_simulation` | implemented-main | — | deterministic generator tests | Task 10 / PR #18 | | Recovery metrics | `validation_core` | implemented-main | — | RMSE/bias/coverage/MC gates | Task 11 / PR #19 | | Versioned API/export contracts | `tepp_api` | implemented-main | naruon HTTP interchange | unknown-field/version/limit + naruon HTTPS interchange tests | Task 12 / PR #21; live HTTP service remaining |