From 9701b3d0363d765a974ae6f07b14b3bfefec9865 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 13 Aug 2026 18:35:21 +0900 Subject: [PATCH 01/26] feat(topic): logistic-normal ALR coordinates with true-parameter RMSE ADR 0012 first production slice: additive log-ratio maps on the unit simplex, fail-closed invalid compositions, and refusal of TF-IDF/BM25 keyword scores as inferential coordinates. No new migration. --- ARCHITECTURE.md | 1 + CHANGELOG.md | 1 + Cargo.lock | 4 + Cargo.toml | 2 + DOCUMENTATION.md | 1 + README.md | 6 +- crates/topic_measurement/Cargo.toml | 17 ++++ crates/topic_measurement/src/coordinates.rs | 92 +++++++++++++++++++ crates/topic_measurement/src/error.rs | 50 ++++++++++ crates/topic_measurement/src/lexical.rs | 36 ++++++++ crates/topic_measurement/src/lib.rs | 20 ++++ .../topic_measurement/tests/crate_contract.rs | 7 ++ .../tests/logratio_recovery_contract.rs | 92 +++++++++++++++++++ docs/TRACEABILITY.md | 4 +- ...ational-shared-latent-topic-measurement.md | 2 +- docs/adr/README.md | 6 +- docs/research/standards-and-literature.md | 2 + docs/research/topic-logratio-coordinates.md | 31 +++++++ docs/validation/temporal-event-foundation.md | 3 +- scripts/check_workspace_contract.py | 1 + 20 files changed, 368 insertions(+), 10 deletions(-) create mode 100644 crates/topic_measurement/Cargo.toml create mode 100644 crates/topic_measurement/src/coordinates.rs create mode 100644 crates/topic_measurement/src/error.rs create mode 100644 crates/topic_measurement/src/lexical.rs create mode 100644 crates/topic_measurement/src/lib.rs create mode 100644 crates/topic_measurement/tests/crate_contract.rs create mode 100644 crates/topic_measurement/tests/logratio_recovery_contract.rs create mode 100644 docs/research/topic-logratio-coordinates.md diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index ffe514db..e3212df5 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -61,6 +61,7 @@ boundaries above remain the target modular MSA architecture. | `tepp_simulation` | known-truth temporal/event data generation | | `validation_core` | RMSE, bias, coverage, graph, and Monte Carlo metrics | | `tepp_api` | versioned DTO, schema, and export contracts | +| `topic_measurement` | logistic-normal / additive log-ratio topic coordinates | No crate exposes placeholder production behavior in Task 1. This prevents an empty façade from becoming a de facto public API before its invariants and tests diff --git a/CHANGELOG.md b/CHANGELOG.md index 93891a27..e97ec769 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to TEPP are documented here. The format follows Keep a Chang ### Added +- `topic_measurement` logistic-normal additive log-ratio coordinates: fail-closed simplex validation, ALR/inverse maps with true-parameter round-trip RMSE, and refusal of TF-IDF/BM25/keyword scores as inferential topic coordinates (ADR 0012 first production slice; no new migration). - `tepp_api` adaptive orchestration router (ADR 0010): versioned `direct`/`verify`/`committee`/`conductor`/`abstain` selection from CPU `f64` risk, ambiguity, evidence, and token-budget inputs; recorded stages, recursion, decomposition, access lists, and role-specific reasoning effort; fail-closed document-controlled policy/access/credentials; LLM plans remain proposals under deterministic statistical authority; comparable-budget ablation requires a direct baseline; credential-free contextual-orchestrator binding. Live NIM HTTP remains accepted-target. - `tepp_api` purpose-bound provider-payload minimization: time-bounded `PurposeGrant` evaluation, fail-closed expired/not-yet-valid/inverted/cross-tenant/impossible-calendar denial, semantic UTC calendar validation, refusal to copy identity mappings into model-provider payloads or ordinary logs, preservation of opaque analytical identifiers and membership roles (no blanket PII mask), a separately authorized scientific re-identification path, and an internally bound FIPS 180-4 SHA-256 audit digest appended through `ReidentificationAuditSink` before disclosure. - `persistence_postgres` backup/restore integrity: restored snapshots stay unusable until tenant, canonical `SHA-256`, knowledge-cutoff eligibility, temporal window order, and append-only triggers revalidate; SQL probes raise `restore integrity failed` (ADR 0013). diff --git a/Cargo.lock b/Cargo.lock index 616bfd78..1068d99c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1372,6 +1372,10 @@ dependencies = [ "tokio", ] +[[package]] +name = "topic_measurement" +version = "0.1.0" + [[package]] name = "tracing" version = "0.1.44" diff --git a/Cargo.toml b/Cargo.toml index 92565940..35ba7a1e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ members = [ "crates/tepp_simulation", "crates/validation_core", "crates/tepp_api", + "crates/topic_measurement", ] default-members = [ "crates/evidence_core", @@ -23,6 +24,7 @@ default-members = [ "crates/tepp_simulation", "crates/validation_core", "crates/tepp_api", + "crates/topic_measurement", ] [workspace.package] diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 3f094947..9e93a508 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -36,6 +36,7 @@ TEPP's approved PRD v0.4 and implementation plan are the primary product baselin | Retention/deletion/legal-hold doctoring | [`docs/research/retention-deletion-legal-hold.md`](docs/research/retention-deletion-legal-hold.md) | | Provider-payload minimization doctoring | [`docs/research/provider-payload-minimization.md`](docs/research/provider-payload-minimization.md) | | Adaptive orchestration router doctoring | [`docs/research/adaptive-orchestration-router.md`](docs/research/adaptive-orchestration-router.md) | +| Topic log-ratio coordinate doctoring | [`docs/research/topic-logratio-coordinates.md`](docs/research/topic-logratio-coordinates.md) | | Hourly NIM OpenCode doctoring | [`docs/doctoring/hourly-nim-opencode-development.md`](docs/doctoring/hourly-nim-opencode-development.md) | | Change history | [`CHANGELOG.md`](CHANGELOG.md) | diff --git a/README.md b/README.md index ae74015d..ed7a2d74 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,8 @@ implemented in Rust. ## Current implementation state This branch establishes the Task 1 Rust workspace and quality-gate foundation. -The ten bounded crates compile independently but intentionally expose no -placeholder production APIs. Domain behavior begins in Task 2 with immutable -evidence identifiers and source records. +The eleven bounded crates compile independently. Domain crates expose only +validated production APIs; placeholder surfaces are prohibited. ```text crates/evidence_core @@ -22,6 +21,7 @@ crates/corpus_split crates/tepp_simulation crates/validation_core crates/tepp_api +crates/topic_measurement ``` ## Local verification diff --git a/crates/topic_measurement/Cargo.toml b/crates/topic_measurement/Cargo.toml new file mode 100644 index 00000000..299f03c2 --- /dev/null +++ b/crates/topic_measurement/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "topic_measurement" +description = "Logistic-normal and log-ratio coordinates for compositional topics." +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true +homepage.workspace = true +readme.workspace = true +keywords.workspace = true +categories.workspace = true +publish = false + +[lints] +workspace = true diff --git a/crates/topic_measurement/src/coordinates.rs b/crates/topic_measurement/src/coordinates.rs new file mode 100644 index 00000000..cde99ce3 --- /dev/null +++ b/crates/topic_measurement/src/coordinates.rs @@ -0,0 +1,92 @@ +//! Additive log-ratio maps for logistic-normal topic coordinates. + +use crate::error::TopicMeasurementError; + +const UNIT_SUM_TOLERANCE: f64 = 1e-12; + +/// Map a strictly positive unit simplex vector to additive log-ratio coordinates. +/// +/// For a `K`-part composition `θ` the image is the `K-1` vector +/// `y_k = ln(θ_k / θ_K)`. This is the logistic-normal coordinate system used +/// by correlated topic models and required before Euclidean or ESEM/DSEM work. +/// +/// # Errors +/// +/// Returns [`TopicMeasurementError::InvalidComposition`] when the vector is +/// empty, has fewer than two parts, contains a non-finite or non-positive +/// entry, or does not sum to one within a tight absolute tolerance. +pub fn additive_log_ratio(proportions: &[f64]) -> Result, TopicMeasurementError> { + let last = require_composition(proportions)?; + Ok(proportions[..proportions.len() - 1] + .iter() + .map(|part| (part / last).ln()) + .collect()) +} + +/// Invert additive log-ratio coordinates back to the unit simplex. +/// +/// # Errors +/// +/// Returns [`TopicMeasurementError::InvalidLogRatioDimension`] when the +/// coordinate vector is empty or contains a non-finite value. +pub fn from_additive_log_ratio(coordinates: &[f64]) -> Result, TopicMeasurementError> { + if coordinates.is_empty() { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + let mut exponentiated = Vec::with_capacity(coordinates.len()); + let mut denom = 1.0_f64; + for &value in coordinates { + if !value.is_finite() { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + let exp = value.exp(); + if !exp.is_finite() { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + denom += exp; + exponentiated.push(exp); + } + if !denom.is_finite() || denom <= 0.0 { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + let mut simplex = Vec::with_capacity(coordinates.len() + 1); + for exp in exponentiated { + simplex.push(exp / denom); + } + simplex.push(1.0 / denom); + Ok(simplex) +} + +fn require_composition(proportions: &[f64]) -> Result { + if proportions.len() < 2 { + return Err(TopicMeasurementError::InvalidComposition); + } + let mut sum = 0.0_f64; + for &part in proportions { + if !part.is_finite() || part <= 0.0 { + return Err(TopicMeasurementError::InvalidComposition); + } + sum += part; + } + if !sum.is_finite() || (sum - 1.0).abs() > UNIT_SUM_TOLERANCE { + return Err(TopicMeasurementError::InvalidComposition); + } + Ok(proportions[proportions.len() - 1]) +} + +#[cfg(test)] +mod tests { + use super::{additive_log_ratio, from_additive_log_ratio}; + use crate::error::TopicMeasurementError; + + #[test] + fn two_part_equal_shares_are_zero_and_overflow_fails_closed() { + let pair = additive_log_ratio(&[0.5, 0.5]).expect("pair"); + assert_eq!(pair.len(), 1); + assert!(pair[0].abs() < 1e-15); + assert_eq!( + from_additive_log_ratio(&[1.0e9]), + Err(TopicMeasurementError::InvalidLogRatioDimension) + ); + } +} diff --git a/crates/topic_measurement/src/error.rs b/crates/topic_measurement/src/error.rs new file mode 100644 index 00000000..f2922574 --- /dev/null +++ b/crates/topic_measurement/src/error.rs @@ -0,0 +1,50 @@ +//! Fail-closed topic-coordinate errors. + +use std::fmt; + +/// A fail-closed topic-measurement error. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum TopicMeasurementError { + /// Composition is empty, has fewer than two parts, is non-positive, + /// non-finite, or does not sum to one. + InvalidComposition, + /// Log-ratio vector is empty or non-finite. + InvalidLogRatioDimension, + /// TF-IDF, BM25, or keyword scores were offered as inferential coordinates. + LexicalWeightForbidden, +} + +impl fmt::Display for TopicMeasurementError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + let message = match self { + Self::InvalidComposition => "invalid compositional topic vector", + Self::InvalidLogRatioDimension => "invalid log-ratio dimension", + Self::LexicalWeightForbidden => "lexical inferential weights are forbidden", + }; + formatter.write_str(message) + } +} + +impl std::error::Error for TopicMeasurementError {} + +#[cfg(test)] +mod tests { + use super::TopicMeasurementError; + + #[test] + fn messages_are_stable() { + assert_eq!( + TopicMeasurementError::InvalidComposition.to_string(), + "invalid compositional topic vector" + ); + assert_eq!( + TopicMeasurementError::InvalidLogRatioDimension.to_string(), + "invalid log-ratio dimension" + ); + assert_eq!( + TopicMeasurementError::LexicalWeightForbidden.to_string(), + "lexical inferential weights are forbidden" + ); + } +} diff --git a/crates/topic_measurement/src/lexical.rs b/crates/topic_measurement/src/lexical.rs new file mode 100644 index 00000000..588f60f9 --- /dev/null +++ b/crates/topic_measurement/src/lexical.rs @@ -0,0 +1,36 @@ +//! Refusal of lexical heuristics as inferential topic coordinates. + +use crate::error::TopicMeasurementError; + +/// Refuse TF-IDF, BM25, and keyword scores as topic-estimator coordinates. +/// +/// ADR 0012 forbids treating lexical retrieval weights as inferential topic +/// coordinates. A recognized statistical method name is accepted so callers +/// can share one vocabulary gate. +/// +/// # Errors +/// +/// Returns [`TopicMeasurementError::LexicalWeightForbidden`] for empty labels +/// and for `tfidf`, `bm25`, and `keyword` after alphanumeric folding. +pub fn refuse_lexical_inferential_weight(method: &str) -> Result<(), TopicMeasurementError> { + let folded: String = method + .chars() + .filter(char::is_ascii_alphanumeric) + .flat_map(char::to_lowercase) + .collect(); + if folded.is_empty() || matches!(folded.as_str(), "tfidf" | "bm25" | "keyword") { + return Err(TopicMeasurementError::LexicalWeightForbidden); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::refuse_lexical_inferential_weight; + + #[test] + fn statistical_method_names_are_allowed() { + refuse_lexical_inferential_weight("tepp_topic_measurement").expect("allowed"); + refuse_lexical_inferential_weight("logistic_normal").expect("allowed"); + } +} diff --git a/crates/topic_measurement/src/lib.rs b/crates/topic_measurement/src/lib.rs new file mode 100644 index 00000000..013732ac --- /dev/null +++ b/crates/topic_measurement/src/lib.rs @@ -0,0 +1,20 @@ +#![forbid(unsafe_code)] +#![deny(missing_docs)] +//! Logistic-normal and log-ratio coordinates for compositional topic proportions. +//! +//! Raw topic proportions are not Euclidean indicators. Downstream network and +//! psychometric analysis must use additive log-ratio (logistic-normal) maps +//! rather than TF-IDF, BM25, or keyword scores as inferential coordinates. + +mod coordinates; +mod error; +mod lexical; + +/// Additive log-ratio map from a simplex vector. +pub use coordinates::additive_log_ratio; +/// Inverse additive log-ratio map back to the simplex. +pub use coordinates::from_additive_log_ratio; +/// Fail-closed topic-coordinate errors. +pub use error::TopicMeasurementError; +/// Refuse lexical retrieval weights as inferential coordinates. +pub use lexical::refuse_lexical_inferential_weight; diff --git a/crates/topic_measurement/tests/crate_contract.rs b/crates/topic_measurement/tests/crate_contract.rs new file mode 100644 index 00000000..8f9cfd6a --- /dev/null +++ b/crates/topic_measurement/tests/crate_contract.rs @@ -0,0 +1,7 @@ +//! Integration contract for the `topic_measurement` package identity. + +#[test] +fn package_identity_is_stable() { + let observed = std::hint::black_box(env!("CARGO_PKG_NAME")); + assert_eq!(observed, "topic_measurement"); +} diff --git a/crates/topic_measurement/tests/logratio_recovery_contract.rs b/crates/topic_measurement/tests/logratio_recovery_contract.rs new file mode 100644 index 00000000..3c0dcdd4 --- /dev/null +++ b/crates/topic_measurement/tests/logratio_recovery_contract.rs @@ -0,0 +1,92 @@ +//! True-parameter recovery of logistic-normal topic coordinates. +#![allow(clippy::cast_precision_loss)] + +use topic_measurement::{ + TopicMeasurementError, additive_log_ratio, from_additive_log_ratio, + refuse_lexical_inferential_weight, +}; + +fn rmse(truth: &[f64], recovered: &[f64]) -> f64 { + let n = truth.len() as f64; + let sum_sq: f64 = truth + .iter() + .zip(recovered) + .map(|(left, right)| { + let residual = left - right; + residual * residual + }) + .sum(); + (sum_sq / n).sqrt() +} + +#[test] +fn known_simplex_recovers_through_alr_with_computed_rmse() { + // Closed-form simplex: (2, 3, 1) / 6. ALR is (ln 2, ln 3). + let truth = [2.0 / 6.0, 3.0 / 6.0, 1.0 / 6.0]; + let coordinates = additive_log_ratio(&truth).expect("alr"); + assert_eq!(coordinates.len(), 2); + assert!((coordinates[0] - 2.0_f64.ln()).abs() < 1e-15); + assert!((coordinates[1] - 3.0_f64.ln()).abs() < 1e-15); + + let recovered = from_additive_log_ratio(&coordinates).expect("inverse"); + let error = rmse(&truth, &recovered); + assert!( + error < 1e-15, + "ALR round-trip RMSE {error} exceeded machine-scale bound" + ); + let sum: f64 = recovered.iter().sum(); + assert!((sum - 1.0).abs() < 1e-15); +} + +#[test] +fn equal_shares_map_to_zero_alr_and_refuse_raw_euclidean_use() { + let thirds = [1.0 / 3.0, 1.0 / 3.0, 1.0 / 3.0]; + let coordinates = additive_log_ratio(&thirds).expect("equal"); + assert!(coordinates.iter().all(|value| value.abs() < 1e-15)); + let recovered = from_additive_log_ratio(&[0.0, 0.0]).expect("zeros"); + assert!(rmse(&thirds, &recovered) < 1e-15); +} + +#[test] +fn invalid_compositions_and_lexical_weights_fail_closed() { + // K=2 is valid; zero/negative/non-unit-sum/non-finite/K<2 are not. + assert_eq!( + additive_log_ratio(&[0.0, 1.0]), + Err(TopicMeasurementError::InvalidComposition) + ); + assert_eq!( + additive_log_ratio(&[-0.1, 1.1]), + Err(TopicMeasurementError::InvalidComposition) + ); + assert_eq!( + additive_log_ratio(&[0.2, 0.2, 0.2]), + Err(TopicMeasurementError::InvalidComposition) + ); + assert_eq!( + additive_log_ratio(&[f64::NAN, 1.0]), + Err(TopicMeasurementError::InvalidComposition) + ); + assert_eq!( + additive_log_ratio(&[]), + Err(TopicMeasurementError::InvalidComposition) + ); + assert_eq!( + additive_log_ratio(&[1.0]), + Err(TopicMeasurementError::InvalidComposition) + ); + assert_eq!( + from_additive_log_ratio(&[]), + Err(TopicMeasurementError::InvalidLogRatioDimension) + ); + assert_eq!( + from_additive_log_ratio(&[f64::INFINITY]), + Err(TopicMeasurementError::InvalidLogRatioDimension) + ); + + for method in ["tfidf", "bm25", "keyword", "TF-IDF", ""] { + assert_eq!( + refuse_lexical_inferential_weight(method), + Err(TopicMeasurementError::LexicalWeightForbidden) + ); + } +} diff --git a/docs/TRACEABILITY.md b/docs/TRACEABILITY.md index f6739641..9bd65faa 100644 --- a/docs/TRACEABILITY.md +++ b/docs/TRACEABILITY.md @@ -22,9 +22,9 @@ The full APA 7th standards/literature register remains `docs/research/standards- | versioned service/API contracts and exports | PRD; API contract; ADR 0011/0013 | `tepp_api` analysis-run/export/JSON-LD/GraphML contracts on protected main (PR #21); HTTP service remaining accepted-target | partial | | immutable split/run/reproducibility manifests | ADR 0013; ERD | `tepp_api` reproducibility manifest contract on protected main; `persistence_postgres` append-only SQL insert/lookup for `reproducibility_manifest`, `corpus_split_manifest`, `model_run`, and `model_artifact` (migration `0003`); full physical ERD constraints remaining | partial | | multilingual shared latent semantic space | PRD; ADR 0004 | future semantic/concept/topic crates | accepted-target | -| TRSL-TM temporal/relational topic posterior and backend compatibility | ADR 0012; ADR 0004 | future `topic_measurement` | accepted-target | +| TRSL-TM temporal/relational topic posterior and backend compatibility | ADR 0012; ADR 0004 | `topic_measurement` ALR coordinates on the active PR; STM backend remaining | partial | | global P0 topic identity with activity/dormancy/reactivation | ADR 0012 | future topic lineage/activity state | accepted-target | -| no default stopword deletion / no TF-IDF-BM25 inferential weighting | ADR 0004/0012; PRD/TRD | future semantic/method-source model | accepted-target | +| no default stopword deletion / no TF-IDF-BM25 inferential weighting | ADR 0004/0012; PRD/TRD | `topic_measurement::refuse_lexical_inferential_weight` on the active PR; preprocessing pipeline remaining | partial | | report template/section/copied/style/modality method effects | ADR 0004/0012; PRD/TRD | simulation truth factors implemented; estimator-side method model remains future | partial | | candidate K statistical/Pareto gates + blinded LLM review | ADR 0012; research | future `model_selection` | accepted-target | | compositional topic correlation / stable clustering | ADR 0005/0012; research | future `network_analysis` | accepted-target | diff --git a/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md b/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md index c3d5085f..c2262965 100644 --- a/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md +++ b/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md @@ -1,7 +1,7 @@ # ADR 0012 — Temporal Relational Shared-Latent Topic Measurement **Decision status:** Accepted -**Implementation maturity:** accepted-target +**Implementation maturity:** partial — logistic-normal additive log-ratio coordinates and lexical-weight refusal are implemented on the active PR and are not implemented-main until exact-head checks, review, and protected-main integration complete; temporal topic identity, backend STM estimator, method-effect model, and K-selection remain accepted-target **Date:** 2026-08-12 **Supersedes:** None; refines ADR 0004 and ADR 0005 without replacing their multilingual and psychometric authorities. diff --git a/docs/adr/README.md b/docs/adr/README.md index 258eb7f3..4f7fe4bc 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -15,10 +15,10 @@ Read [`ADR_POLICY.md`](ADR_POLICY.md) first. **Decision status and implementatio | [0007](0007-rust-workspace-quality-gates.md) | Explicit Rust workspace, pinned toolchains, and exact quality gates | Accepted | implemented-main | ADR 0014 governs scientific/product claim promotion beyond repository-quality tooling. | | [0008](0008-immutable-evidence-identities-digests-and-spans.md) | Immutable evidence identities, `SHA-256` digests, exact spans, and strict wire reconstruction | Accepted | implemented-main | ADR 0013 governs future persistence/reproducibility/split authority. | | [0009](0009-purpose-bound-pii-governance.md) | Purpose-bound PII governance without blanket masking | Accepted | partial | Persistence retention/deletion/legal-hold (`0007`) and provider-payload minimization implemented-main; deployment evidence remains accepted-target. | -| [0010](0010-adaptive-llm-orchestration.md) | Adaptive LLM orchestration and test-time compute | Accepted | partial | `tepp_api` router/ablation/orchestrator binding on the active PR; live NIM execution and production ablation evidence remain accepted-target. | +| [0010](0010-adaptive-llm-orchestration.md) | Adaptive LLM orchestration and test-time compute | Accepted | partial | `tepp_api` router/ablation/orchestrator binding implemented-main; live NIM execution and production ablation evidence remain accepted-target. | | [0011](0011-standalone-modular-msa-boundary.md) | Standalone operation and modular CWL MSA boundary | Accepted | partial | Owns cross-service persistence/credential/API authority; no direct cross-service application-table coupling. | -| [0012](0012-temporal-relational-shared-latent-topic-measurement.md) | Temporal Relational Shared-Latent Topic Measurement (TRSL-TM) | Accepted | accepted-target | Owns topic backend compatibility, global topic identity, method effects, K/model-selection prerequisites, and compositional topic coordinates. | -| [0013](0013-bitemporal-persistence-reproducibility-and-split-authority.md) | Bitemporal persistence, reproducibility manifests, and relation-aware split authority | Accepted | partial | Owns PostgreSQL adapter semantics, immutable run/split manifests, leakage-safe partitions, and recovery identity; optional `live-sqlx` `PgPool`, live PG CI, tenant RLS, and `0006` membership implemented-main; `0007` retention/deletion/legal-hold on the active PR; remaining physical ERD/backup accepted-target. | +| [0012](0012-temporal-relational-shared-latent-topic-measurement.md) | Temporal Relational Shared-Latent Topic Measurement (TRSL-TM) | Accepted | partial | Logistic-normal ALR coordinates and lexical-weight refusal are on the active PR; temporal topic identity, STM backend, method effects, and K-selection remain accepted-target. | +| [0013](0013-bitemporal-persistence-reproducibility-and-split-authority.md) | Bitemporal persistence, reproducibility manifests, and relation-aware split authority | Accepted | partial | Owns PostgreSQL adapter semantics, immutable run/split manifests, leakage-safe partitions, and recovery identity; optional `live-sqlx` `PgPool`, live PG CI, tenant RLS, and `0006` membership implemented-main; `0007` retention/deletion/legal-hold implemented-main; remaining physical ERD/backup accepted-target. | | [0014](0014-scientific-claim-promotion-and-release-evidence.md) | Scientific claim promotion and release evidence authority | Accepted | partial | Separates design, implementation, scientific/product claim, and release authority; repository SBOM/provenance generator implemented, full release bundle remaining. | | [0015](0015-autonomous-development-review-and-merge-authority.md) | Autonomous development, review, and merge authority separation | Accepted | active-PR | Separates model proposal, deterministic verification, publication, independent review, and merge/release authority. | | [0016](0016-tdt-chronos-event-intelligence-boundary.md) | TDT, CHRONOS, and Event Ontology intelligence boundary | Accepted | accepted-target | Separates observed evidence, detection/tracking, prediction/schema inference, temporal consistency, and promoted transition authority. | diff --git a/docs/research/standards-and-literature.md b/docs/research/standards-and-literature.md index bfda7a79..d793954a 100644 --- a/docs/research/standards-and-literature.md +++ b/docs/research/standards-and-literature.md @@ -48,6 +48,8 @@ LLM evaluation complements but never replaces predictive, posterior, stability, ## Compositional data, correlation, and clusters +Aitchison, J., & Shen, S. M. (1980). Logistic-normal distributions: Some properties and uses. *Biometrika, 67*(2), 261–272. https://doi.org/10.1093/biomet/67.2.261 + Aitchison, J. (1982). The statistical analysis of compositional data. *Journal of the Royal Statistical Society: Series B (Methodological), 44*(2), 139–177. https://doi.org/10.1111/j.2517-6161.1982.tb01195.x Friedman, J., Hastie, T., & Tibshirani, R. (2008). Sparse inverse covariance estimation with the graphical lasso. *Biostatistics, 9*(3), 432–441. https://doi.org/10.1093/biostatistics/kxm045 diff --git a/docs/research/topic-logratio-coordinates.md b/docs/research/topic-logratio-coordinates.md new file mode 100644 index 00000000..a3514cd9 --- /dev/null +++ b/docs/research/topic-logratio-coordinates.md @@ -0,0 +1,31 @@ +# Logistic-normal topic coordinates + +## Scope + +This note doctors the first `topic_measurement` production slice (ADR 0012): + +1. raw topic proportions are not Euclidean indicators; +2. additive log-ratio coordinates implement the logistic-normal map used by correlated topic models; +3. inverse ALR recovers a known simplex with a computed RMSE; +4. TF-IDF, BM25, and keyword scores are refused as inferential coordinates. + +The temporal STM backend, global topic identity, method-effect model, and K-selection remain accepted-target. No database migration is allocated. + +## Authoritative sources + +Aitchison, J., & Shen, S. M. (1980). Logistic-normal distributions: Some properties and uses. *Biometrika, 67*(2), 261–272. https://doi.org/10.1093/biomet/67.2.261 + +Aitchison, J. (1982). The statistical analysis of compositional data. *Journal of the Royal Statistical Society: Series B (Methodological), 44*(2), 139–177. https://doi.org/10.1111/j.2517-6161.1982.tb01195.x + +Blei, D. M., & Lafferty, J. D. (2007). A correlated topic model of Science. *The Annals of Applied Statistics, 1*(1), 17–35. https://doi.org/10.1214/07-AOAS114 + +## Application + +Aitchison and Shen (1980) define the logistic-normal family via the additive log-ratio map; Aitchison (1982) is the compositional-data authority that forbids treating parts of a whole as unconstrained Euclidean coordinates. Blei and Lafferty (2007) use that same map for correlated topic models. TEPP therefore converts a strictly positive unit simplex through `additive_log_ratio` before any Euclidean or psychometric operation, and recovers the simplex with `from_additive_log_ratio` (Aitchison & Shen, 1980; Aitchison, 1982; Blei & Lafferty, 2007). + +## Verification + +- closed-form simplex `(2,3,1)/6` maps to `(ln 2, ln 3)` and inverts with computed RMSE below `1e-15`; +- equal shares map to a zero ALR vector; +- zero, negative, non-unit-sum, non-finite, empty, and one-part vectors fail closed; +- `tfidf`, `bm25`, and `keyword` labels are refused. diff --git a/docs/validation/temporal-event-foundation.md b/docs/validation/temporal-event-foundation.md index aae1a06e..307624d7 100644 --- a/docs/validation/temporal-event-foundation.md +++ b/docs/validation/temporal-event-foundation.md @@ -24,7 +24,8 @@ This report tracks exact-head scientific and engineering evidence required befor | Recovery metrics | `validation_core` | implemented-main | — | RMSE/bias/coverage/MC gates | Task 11 / PR #19 | | Versioned API/export contracts | `tepp_api` | implemented-main | naruon HTTP interchange | unknown-field/version/limit + naruon HTTPS interchange tests | Task 12 / PR #21; live HTTP service remaining | | Purpose-bound provider payloads | `tepp_api` | implemented-main | provider-payload minimization | expired/not-yet-valid/inverted/cross-tenant/impossible-calendar grant, mapping refusal, audited elevated re-id replay | ADR 0009; `docs/research/provider-payload-minimization.md` | -| Adaptive orchestration router | `tepp_api` | accepted-target | active PR | mode selection, document-control denial, ablation, credential-free bind | ADR 0010; `docs/research/adaptive-orchestration-router.md` | +| Adaptive orchestration router | `tepp_api` | implemented-main | router + ablation | mode selection, document-control denial, ablation, credential-free bind | ADR 0010; `docs/research/adaptive-orchestration-router.md` | +| Logistic-normal topic coordinates | `topic_measurement` | active-PR | ALR + lexical refusal | true-parameter ALR RMSE | ADR 0012; `docs/research/topic-logratio-coordinates.md` | | CWL modular connectors | `docs/connectors/*` | implemented-main | — | contract docs + examples | PR #22; live HTTP ports remaining | | Release SBOM/provenance generator | `scripts/release_evidence.py` | partial | — | generate+validate in CI | Task 13 partial / PR #28 | diff --git a/scripts/check_workspace_contract.py b/scripts/check_workspace_contract.py index c7b1ecf5..616f8d07 100644 --- a/scripts/check_workspace_contract.py +++ b/scripts/check_workspace_contract.py @@ -23,6 +23,7 @@ "tepp_simulation", "validation_core", "tepp_api", + "topic_measurement", ) REQUIRED_CI_SNIPPETS: tuple[str, ...] = ( From 130bc714735e09d4384090892a92dd605db07b44 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 14 Aug 2026 18:24:16 +0900 Subject: [PATCH 02/26] test(topic): expose unstable ALR overflow path --- .../tests/logratio_recovery_contract.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/crates/topic_measurement/tests/logratio_recovery_contract.rs b/crates/topic_measurement/tests/logratio_recovery_contract.rs index 3c0dcdd4..d0f133f9 100644 --- a/crates/topic_measurement/tests/logratio_recovery_contract.rs +++ b/crates/topic_measurement/tests/logratio_recovery_contract.rs @@ -38,6 +38,22 @@ fn known_simplex_recovers_through_alr_with_computed_rmse() { assert!((sum - 1.0).abs() < 1e-15); } +#[test] +fn large_finite_coordinates_round_trip_without_exponential_overflow() { + let truth = [710.0, 709.0]; + let simplex = from_additive_log_ratio(&truth) + .expect("finite representable ALR coordinates must use a stable inverse"); + assert!(simplex.iter().all(|part| part.is_finite() && *part > 0.0)); + assert!((simplex.iter().sum::() - 1.0).abs() < 1e-15); + + let recovered = additive_log_ratio(&simplex) + .expect("forward ALR must subtract logs instead of overflowing the ratio"); + assert!( + rmse(&truth, &recovered) < 1e-10, + "large-coordinate round trip must retain the true parameters" + ); +} + #[test] fn equal_shares_map_to_zero_alr_and_refuse_raw_euclidean_use() { let thirds = [1.0 / 3.0, 1.0 / 3.0, 1.0 / 3.0]; From e76a0deb5f31828f587a7802c8987f31bc99214f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 14 Aug 2026 18:27:42 +0900 Subject: [PATCH 03/26] test(topic): reject inverse underflow to a zero simplex part --- crates/topic_measurement/tests/logratio_recovery_contract.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/topic_measurement/tests/logratio_recovery_contract.rs b/crates/topic_measurement/tests/logratio_recovery_contract.rs index d0f133f9..e6af5f8d 100644 --- a/crates/topic_measurement/tests/logratio_recovery_contract.rs +++ b/crates/topic_measurement/tests/logratio_recovery_contract.rs @@ -98,6 +98,11 @@ fn invalid_compositions_and_lexical_weights_fail_closed() { from_additive_log_ratio(&[f64::INFINITY]), Err(TopicMeasurementError::InvalidLogRatioDimension) ); + assert_eq!( + from_additive_log_ratio(&[-1.0e9]), + Err(TopicMeasurementError::InvalidLogRatioDimension), + "inverse must not return a zero simplex part after underflow" + ); for method in ["tfidf", "bm25", "keyword", "TF-IDF", ""] { assert_eq!( From 30ae9e0c59cfc63dd9d61e1413faba4d06aa7dbd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 14 Aug 2026 18:29:45 +0900 Subject: [PATCH 04/26] fix(topic): script stable ALR and preserve shared ledgers --- scripts/repair_pr48_logratio_stability.py | 258 ++++++++++++++++++++++ 1 file changed, 258 insertions(+) create mode 100644 scripts/repair_pr48_logratio_stability.py diff --git a/scripts/repair_pr48_logratio_stability.py b/scripts/repair_pr48_logratio_stability.py new file mode 100644 index 00000000..cc6f4dae --- /dev/null +++ b/scripts/repair_pr48_logratio_stability.py @@ -0,0 +1,258 @@ +"""Apply PR 48 stable log-ratio arithmetic and documentation repairs.""" + +from pathlib import Path + + +def replace_once(text: str, old: str, new: str, label: str) -> str: + """Replace exactly one fragment or fail closed.""" + count = text.count(old) + if count != 1: + raise SystemExit(f"{label}: expected one target, found {count}") + return text.replace(old, new, 1) + + +def update_coordinates() -> None: + """Use stable log differences and max-shifted inverse softmax.""" + path = Path("crates/topic_measurement/src/coordinates.rs") + text = path.read_text(encoding="utf-8") + text = replace_once( + text, + """/// For a `K`-part composition `θ` the image is the `K-1` vector +/// `y_k = ln(θ_k / θ_K)`. This is the logistic-normal coordinate system used +/// by correlated topic models and required before Euclidean or ESEM/DSEM work. +""", + """/// For a `K`-part composition `θ` the image is the `K-1` vector +/// `y_k = ln(θ_k / θ_K)`. This reference-dependent, full-rank coordinate map +/// supports logistic-normal regression and ESEM/DSEM interfaces. It is not an +/// orthonormal isometry for Aitchison distance; use ILR coordinates when that +/// Euclidean geometry is the estimand. +""", + "coordinate documentation", + ) + text = replace_once( + text, + ".map(|part| (part / last).ln())", + ".map(|part| part.ln() - last.ln())", + "stable forward ALR", + ) + old_inverse = """pub fn from_additive_log_ratio(coordinates: &[f64]) -> Result, TopicMeasurementError> { + if coordinates.is_empty() { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + let mut exponentiated = Vec::with_capacity(coordinates.len()); + let mut denom = 1.0_f64; + for &value in coordinates { + if !value.is_finite() { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + let exp = value.exp(); + if !exp.is_finite() { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + denom += exp; + exponentiated.push(exp); + } + if !denom.is_finite() || denom <= 0.0 { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + let mut simplex = Vec::with_capacity(coordinates.len() + 1); + for exp in exponentiated { + simplex.push(exp / denom); + } + simplex.push(1.0 / denom); + Ok(simplex) +} +""" + new_inverse = """pub fn from_additive_log_ratio(coordinates: &[f64]) -> Result, TopicMeasurementError> { + if coordinates.is_empty() { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + let mut maximum = 0.0_f64; + for &value in coordinates { + if !value.is_finite() { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + maximum = maximum.max(value); + } + + let reference_weight = (-maximum).exp(); + if reference_weight == 0.0 { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + let mut shifted_weights = Vec::with_capacity(coordinates.len()); + let mut denominator = reference_weight; + for &value in coordinates { + let weight = (value - maximum).exp(); + if weight == 0.0 { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + denominator += weight; + shifted_weights.push(weight); + } + + let mut simplex = Vec::with_capacity(coordinates.len() + 1); + for weight in shifted_weights { + simplex.push(weight / denominator); + } + simplex.push(reference_weight / denominator); + Ok(simplex) +} +""" + text = replace_once(text, old_inverse, new_inverse, "stable inverse ALR") + text = replace_once( + text, + """/// Returns [`TopicMeasurementError::InvalidLogRatioDimension`] when the +/// coordinate vector is empty or contains a non-finite value. +""", + """/// Returns [`TopicMeasurementError::InvalidLogRatioDimension`] when the +/// coordinate vector is empty, non-finite, or would underflow a part to zero +/// in the strictly positive `f64` simplex representation. +""", + "inverse error documentation", + ) + text = replace_once( + text, + "fn two_part_equal_shares_are_zero_and_overflow_fails_closed()", + "fn two_part_equal_shares_are_zero_and_unrepresentable_extremes_fail_closed()", + "internal test name", + ) + path.write_text(text, encoding="utf-8") + + +def update_error_and_crate_docs() -> None: + """Describe representability and ALR geometry without overclaiming isometry.""" + error_path = Path("crates/topic_measurement/src/error.rs") + error_text = error_path.read_text(encoding="utf-8") + error_text = replace_once( + error_text, + " /// Log-ratio vector is empty or non-finite.\n", + " /// Log-ratio vector is empty, non-finite, or not representable as a strictly positive `f64` simplex.\n", + "error variant documentation", + ) + error_path.write_text(error_text, encoding="utf-8") + + lib_path = Path("crates/topic_measurement/src/lib.rs") + lib_text = lib_path.read_text(encoding="utf-8") + lib_text = replace_once( + lib_text, + """//! Raw topic proportions are not Euclidean indicators. Downstream network and +//! psychometric analysis must use additive log-ratio (logistic-normal) maps +//! rather than TF-IDF, BM25, or keyword scores as inferential coordinates. +""", + """//! Raw topic proportions are compositional rather than unconstrained Euclidean +//! indicators. ALR supplies a reference-dependent full-rank logistic-normal map +//! for regression and psychometric interfaces; it is not an orthonormal +//! Aitchison-distance isometry. Distance-based Aitchison geometry requires ILR. +//! TF-IDF, BM25, and keyword scores remain forbidden inferential coordinates. +""", + "crate geometry documentation", + ) + lib_path.write_text(lib_text, encoding="utf-8") + + +def update_research_and_adr() -> None: + """Clarify ALR versus ILR and record stable arithmetic evidence.""" + research_path = Path("docs/research/topic-logratio-coordinates.md") + research = research_path.read_text(encoding="utf-8") + research = replace_once( + research, + """1. raw topic proportions are not Euclidean indicators; +2. additive log-ratio coordinates implement the logistic-normal map used by correlated topic models; +3. inverse ALR recovers a known simplex with a computed RMSE; +4. TF-IDF, BM25, and keyword scores are refused as inferential coordinates. +""", + """1. raw topic proportions are compositional rather than unconstrained Euclidean indicators; +2. additive log-ratio coordinates implement the reference-dependent logistic-normal map used by correlated topic models; +3. ALR is full rank but not an orthonormal Aitchison-distance isometry; ILR is required when that Euclidean geometry is the estimand; +4. max-shifted inverse ALR and log-difference forward ALR recover representable extreme coordinates without overflow; +5. TF-IDF, BM25, and keyword scores are refused as inferential coordinates. +""", + "research scope", + ) + research = replace_once( + research, + """Aitchison and Shen (1980) define the logistic-normal family via the additive log-ratio map; Aitchison (1982) is the compositional-data authority that forbids treating parts of a whole as unconstrained Euclidean coordinates. Blei and Lafferty (2007) use that same map for correlated topic models. TEPP therefore converts a strictly positive unit simplex through `additive_log_ratio` before any Euclidean or psychometric operation, and recovers the simplex with `from_additive_log_ratio` (Aitchison & Shen, 1980; Aitchison, 1982; Blei & Lafferty, 2007). +""", + """Aitchison and Shen (1980) define the logistic-normal family via the additive log-ratio map; Aitchison (1982) is the compositional-data authority that forbids treating parts of a whole as unconstrained Euclidean coordinates. Blei and Lafferty (2007) use that same reference-dependent map for correlated topic models. TEPP therefore uses `additive_log_ratio` for logistic-normal regression and psychometric interfaces, but does not claim that ALR preserves Aitchison distance. Analyses whose estimand is orthonormal Euclidean Aitchison geometry must use ILR. `from_additive_log_ratio` uses a max-shifted inverse softmax and the forward map subtracts logarithms, avoiding avoidable exponential and ratio overflow while failing closed when an `f64` simplex part would underflow to zero (Aitchison & Shen, 1980; Aitchison, 1982; Blei & Lafferty, 2007). +""", + "research application", + ) + research = replace_once( + research, + """- closed-form simplex `(2,3,1)/6` maps to `(ln 2, ln 3)` and inverts with computed RMSE below `1e-15`; +- equal shares map to a zero ALR vector; +""", + """- closed-form simplex `(2,3,1)/6` maps to `(ln 2, ln 3)` and inverts with computed RMSE below `1e-15`; +- representable coordinates `(710, 709)` round-trip through the max-shifted inverse without exponential overflow; +- extremes that would underflow a strictly positive `f64` simplex part fail closed; +- equal shares map to a zero ALR vector; +""", + "research verification", + ) + research_path.write_text(research, encoding="utf-8") + + adr_path = Path("docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md") + adr = adr_path.read_text(encoding="utf-8") + adr = replace_once( + adr, + """- topic proportions are compositional and downstream network/psychometric analysis uses logistic-normal coordinates or valid orthonormal log-ratio coordinates; +""", + """- topic proportions are compositional and downstream network/psychometric analysis uses logistic-normal coordinates or valid orthonormal log-ratio coordinates; +- ALR is a reference-dependent full-rank logistic-normal map, not an Aitchison-distance isometry; distance-based Euclidean Aitchison geometry uses an orthonormal ILR basis; +""", + "ADR ALR/ILR boundary", + ) + adr_path.write_text(adr, encoding="utf-8") + + +def restore_conflict_resolved_ledgers() -> None: + """Reapply the topic slice to main-owned shared ledgers after the merge.""" + changelog_path = Path("CHANGELOG.md") + changelog = changelog_path.read_text(encoding="utf-8") + item = ( + "- `topic_measurement` logistic-normal additive log-ratio coordinates: " + "fail-closed simplex validation, max-shifted stable ALR/inverse maps with " + "true-parameter round-trip RMSE, explicit ALR-versus-ILR geometry boundary, " + "and refusal of TF-IDF/BM25/keyword scores as inferential topic coordinates " + "(ADR 0012 first production slice; no new migration).\n" + ) + if item not in changelog: + changelog = replace_once(changelog, "### Added\n\n", "### Added\n\n" + item, "changelog marker") + changelog_path.write_text(changelog, encoding="utf-8") + + trace_path = Path("docs/TRACEABILITY.md") + trace = trace_path.read_text(encoding="utf-8") + trace = replace_once( + trace, + "| TRSL-TM temporal/relational topic posterior and backend compatibility | ADR 0012; ADR 0004 | future `topic_measurement` | accepted-target |", + "| TRSL-TM temporal/relational topic posterior and backend compatibility | ADR 0012; ADR 0004 | `topic_measurement` stable ALR coordinates on the active PR; temporal STM backend remaining | partial |", + "trace topic backend", + ) + trace = replace_once( + trace, + "| no default stopword deletion / no TF-IDF-BM25 inferential weighting | ADR 0004/0012; PRD/TRD | future semantic/method-source model | accepted-target |", + "| no default stopword deletion / no TF-IDF-BM25 inferential weighting | ADR 0004/0012; PRD/TRD | `topic_measurement::refuse_lexical_inferential_weight` on the active PR; preprocessing pipeline remaining | partial |", + "trace lexical refusal", + ) + trace_path.write_text(trace, encoding="utf-8") + + validation_path = Path("docs/validation/temporal-event-foundation.md") + validation = validation_path.read_text(encoding="utf-8") + row = ( + "| Logistic-normal topic coordinates | `topic_measurement` | active-PR | stable ALR + lexical refusal | " + "known-simplex and extreme-coordinate RMSE, ALR/ILR boundary | ADR 0012; " + "`docs/research/topic-logratio-coordinates.md` |\n" + ) + if row not in validation: + marker = ( + "| Versioned API/export contracts | `tepp_api` | implemented-main | naruon HTTP interchange | " + "unknown-field/version/limit + naruon HTTPS interchange tests | Task 12 / PR #21; live HTTP service remaining |\n" + ) + validation = replace_once(validation, marker, marker + row, "validation API row") + validation_path.write_text(validation, encoding="utf-8") + + +update_coordinates() +update_error_and_crate_docs() +update_research_and_adr() +restore_conflict_resolved_ledgers() From bce4ac69fce26fb9933dbcd3ad9148c8e46e9867 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 14 Aug 2026 18:30:24 +0900 Subject: [PATCH 05/26] chore(ci): verify PR 48 stable log-ratio repair --- .../repair-pr48-logratio-stability.yml | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 .github/workflows/repair-pr48-logratio-stability.yml diff --git a/.github/workflows/repair-pr48-logratio-stability.yml b/.github/workflows/repair-pr48-logratio-stability.yml new file mode 100644 index 00000000..dc1af6c2 --- /dev/null +++ b/.github/workflows/repair-pr48-logratio-stability.yml @@ -0,0 +1,75 @@ +name: Repair PR 48 log-ratio numerical stability + +on: + pull_request: + types: + - synchronize + - reopened + - ready_for_review + +permissions: + contents: read + +concurrency: + group: repair-tepp-pr-48-logratio-stability + cancel-in-progress: false + +jobs: + repair: + if: >- + github.event.pull_request.number == 48 && + github.event.pull_request.head.repo.full_name == github.repository && + github.event.pull_request.head.ref == 'agent/topic-logratio-coordinates' + runs-on: ubuntu-latest + timeout-minutes: 35 + permissions: + contents: write + steps: + - name: Checkout exact PR branch + uses: actions/checkout@631c942040754b6e095e929c1677c07e10ed4f87 + with: + ref: agent/topic-logratio-coordinates + fetch-depth: 0 + persist-credentials: true + + - name: Install pinned Rust toolchain + run: rustup toolchain install 1.97.1 --profile minimal --component clippy --component rustfmt + + - name: Prove stable extreme-coordinate contracts are RED + run: | + set +e + output=$(cargo +1.97.1 test -p topic_measurement --test logratio_recovery_contract 2>&1) + status=$? + set -e + printf '%s\n' "$output" + if [ "$status" -eq 0 ]; then + echo "Expected direct exponentiation or ratio arithmetic to fail the new stability contracts" >&2 + exit 1 + fi + grep -E "large_finite_coordinates_round_trip|inverse must not return a zero simplex part" <<<"$output" + + - name: Apply stable ALR arithmetic and documentation repair + run: | + python3 scripts/repair_pr48_logratio_stability.py + cargo +1.97.1 fmt --all + + - name: Verify focused and workspace contracts + run: | + cargo +1.97.1 fmt --all --check + cargo +1.97.1 test -p topic_measurement --all-features + cargo +1.97.1 clippy -p topic_measurement --all-targets --all-features -- -D warnings + cargo +1.97.1 test --workspace --all-features + python3 scripts/check_workspace_contract.py + python3 scripts/check_docstrings.py + python3 scripts/validate_documentation.py + + - name: Commit verified repair and remove one-shot files + run: | + rm -f .github/workflows/repair-pr48-logratio-stability.yml + rm -f scripts/repair_pr48_logratio_stability.py + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add -A + git diff --cached --check + git commit -m "fix(topic): stabilize extreme log-ratio coordinates" + git push origin HEAD:agent/topic-logratio-coordinates From c95ebe1104d08a945276c02ac4595c212e4bf404 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2026 10:11:11 +0000 Subject: [PATCH 06/26] fix(topic): stabilize extreme log-ratio coordinates --- .../repair-pr48-logratio-stability.yml | 75 ----- CHANGELOG.md | 2 +- crates/topic_measurement/src/coordinates.rs | 44 +-- crates/topic_measurement/src/error.rs | 2 +- crates/topic_measurement/src/lib.rs | 8 +- docs/TRACEABILITY.md | 2 +- ...ational-shared-latent-topic-measurement.md | 1 + docs/research/topic-logratio-coordinates.md | 13 +- docs/validation/temporal-event-foundation.md | 2 +- scripts/repair_pr48_logratio_stability.py | 258 ------------------ 10 files changed, 45 insertions(+), 362 deletions(-) delete mode 100644 .github/workflows/repair-pr48-logratio-stability.yml delete mode 100644 scripts/repair_pr48_logratio_stability.py diff --git a/.github/workflows/repair-pr48-logratio-stability.yml b/.github/workflows/repair-pr48-logratio-stability.yml deleted file mode 100644 index dc1af6c2..00000000 --- a/.github/workflows/repair-pr48-logratio-stability.yml +++ /dev/null @@ -1,75 +0,0 @@ -name: Repair PR 48 log-ratio numerical stability - -on: - pull_request: - types: - - synchronize - - reopened - - ready_for_review - -permissions: - contents: read - -concurrency: - group: repair-tepp-pr-48-logratio-stability - cancel-in-progress: false - -jobs: - repair: - if: >- - github.event.pull_request.number == 48 && - github.event.pull_request.head.repo.full_name == github.repository && - github.event.pull_request.head.ref == 'agent/topic-logratio-coordinates' - runs-on: ubuntu-latest - timeout-minutes: 35 - permissions: - contents: write - steps: - - name: Checkout exact PR branch - uses: actions/checkout@631c942040754b6e095e929c1677c07e10ed4f87 - with: - ref: agent/topic-logratio-coordinates - fetch-depth: 0 - persist-credentials: true - - - name: Install pinned Rust toolchain - run: rustup toolchain install 1.97.1 --profile minimal --component clippy --component rustfmt - - - name: Prove stable extreme-coordinate contracts are RED - run: | - set +e - output=$(cargo +1.97.1 test -p topic_measurement --test logratio_recovery_contract 2>&1) - status=$? - set -e - printf '%s\n' "$output" - if [ "$status" -eq 0 ]; then - echo "Expected direct exponentiation or ratio arithmetic to fail the new stability contracts" >&2 - exit 1 - fi - grep -E "large_finite_coordinates_round_trip|inverse must not return a zero simplex part" <<<"$output" - - - name: Apply stable ALR arithmetic and documentation repair - run: | - python3 scripts/repair_pr48_logratio_stability.py - cargo +1.97.1 fmt --all - - - name: Verify focused and workspace contracts - run: | - cargo +1.97.1 fmt --all --check - cargo +1.97.1 test -p topic_measurement --all-features - cargo +1.97.1 clippy -p topic_measurement --all-targets --all-features -- -D warnings - cargo +1.97.1 test --workspace --all-features - python3 scripts/check_workspace_contract.py - python3 scripts/check_docstrings.py - python3 scripts/validate_documentation.py - - - name: Commit verified repair and remove one-shot files - run: | - rm -f .github/workflows/repair-pr48-logratio-stability.yml - rm -f scripts/repair_pr48_logratio_stability.py - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add -A - git diff --cached --check - git commit -m "fix(topic): stabilize extreme log-ratio coordinates" - git push origin HEAD:agent/topic-logratio-coordinates diff --git a/CHANGELOG.md b/CHANGELOG.md index e97ec769..8c43f24d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ All notable changes to TEPP are documented here. The format follows Keep a Chang ### Added -- `topic_measurement` logistic-normal additive log-ratio coordinates: fail-closed simplex validation, ALR/inverse maps with true-parameter round-trip RMSE, and refusal of TF-IDF/BM25/keyword scores as inferential topic coordinates (ADR 0012 first production slice; no new migration). +- `topic_measurement` logistic-normal additive log-ratio coordinates: fail-closed simplex validation, max-shifted stable ALR/inverse maps with true-parameter round-trip RMSE, explicit ALR-versus-ILR geometry boundary, and refusal of TF-IDF/BM25/keyword scores as inferential topic coordinates (ADR 0012 first production slice; no new migration). - `tepp_api` adaptive orchestration router (ADR 0010): versioned `direct`/`verify`/`committee`/`conductor`/`abstain` selection from CPU `f64` risk, ambiguity, evidence, and token-budget inputs; recorded stages, recursion, decomposition, access lists, and role-specific reasoning effort; fail-closed document-controlled policy/access/credentials; LLM plans remain proposals under deterministic statistical authority; comparable-budget ablation requires a direct baseline; credential-free contextual-orchestrator binding. Live NIM HTTP remains accepted-target. - `tepp_api` purpose-bound provider-payload minimization: time-bounded `PurposeGrant` evaluation, fail-closed expired/not-yet-valid/inverted/cross-tenant/impossible-calendar denial, semantic UTC calendar validation, refusal to copy identity mappings into model-provider payloads or ordinary logs, preservation of opaque analytical identifiers and membership roles (no blanket PII mask), a separately authorized scientific re-identification path, and an internally bound FIPS 180-4 SHA-256 audit digest appended through `ReidentificationAuditSink` before disclosure. - `persistence_postgres` backup/restore integrity: restored snapshots stay unusable until tenant, canonical `SHA-256`, knowledge-cutoff eligibility, temporal window order, and append-only triggers revalidate; SQL probes raise `restore integrity failed` (ADR 0013). diff --git a/crates/topic_measurement/src/coordinates.rs b/crates/topic_measurement/src/coordinates.rs index cde99ce3..1909df2f 100644 --- a/crates/topic_measurement/src/coordinates.rs +++ b/crates/topic_measurement/src/coordinates.rs @@ -7,8 +7,10 @@ const UNIT_SUM_TOLERANCE: f64 = 1e-12; /// Map a strictly positive unit simplex vector to additive log-ratio coordinates. /// /// For a `K`-part composition `θ` the image is the `K-1` vector -/// `y_k = ln(θ_k / θ_K)`. This is the logistic-normal coordinate system used -/// by correlated topic models and required before Euclidean or ESEM/DSEM work. +/// `y_k = ln(θ_k / θ_K)`. This reference-dependent, full-rank coordinate map +/// supports logistic-normal regression and ESEM/DSEM interfaces. It is not an +/// orthonormal isometry for Aitchison distance; use ILR coordinates when that +/// Euclidean geometry is the estimand. /// /// # Errors /// @@ -19,7 +21,7 @@ pub fn additive_log_ratio(proportions: &[f64]) -> Result, TopicMeasurem let last = require_composition(proportions)?; Ok(proportions[..proportions.len() - 1] .iter() - .map(|part| (part / last).ln()) + .map(|part| part.ln() - last.ln()) .collect()) } @@ -28,32 +30,40 @@ pub fn additive_log_ratio(proportions: &[f64]) -> Result, TopicMeasurem /// # Errors /// /// Returns [`TopicMeasurementError::InvalidLogRatioDimension`] when the -/// coordinate vector is empty or contains a non-finite value. +/// coordinate vector is empty, non-finite, or would underflow a part to zero +/// in the strictly positive `f64` simplex representation. pub fn from_additive_log_ratio(coordinates: &[f64]) -> Result, TopicMeasurementError> { if coordinates.is_empty() { return Err(TopicMeasurementError::InvalidLogRatioDimension); } - let mut exponentiated = Vec::with_capacity(coordinates.len()); - let mut denom = 1.0_f64; + let mut maximum = 0.0_f64; for &value in coordinates { if !value.is_finite() { return Err(TopicMeasurementError::InvalidLogRatioDimension); } - let exp = value.exp(); - if !exp.is_finite() { - return Err(TopicMeasurementError::InvalidLogRatioDimension); - } - denom += exp; - exponentiated.push(exp); + maximum = maximum.max(value); } - if !denom.is_finite() || denom <= 0.0 { + + let reference_weight = (-maximum).exp(); + if reference_weight == 0.0 { return Err(TopicMeasurementError::InvalidLogRatioDimension); } + let mut shifted_weights = Vec::with_capacity(coordinates.len()); + let mut denominator = reference_weight; + for &value in coordinates { + let weight = (value - maximum).exp(); + if weight == 0.0 { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + denominator += weight; + shifted_weights.push(weight); + } + let mut simplex = Vec::with_capacity(coordinates.len() + 1); - for exp in exponentiated { - simplex.push(exp / denom); + for weight in shifted_weights { + simplex.push(weight / denominator); } - simplex.push(1.0 / denom); + simplex.push(reference_weight / denominator); Ok(simplex) } @@ -80,7 +90,7 @@ mod tests { use crate::error::TopicMeasurementError; #[test] - fn two_part_equal_shares_are_zero_and_overflow_fails_closed() { + fn two_part_equal_shares_are_zero_and_unrepresentable_extremes_fail_closed() { let pair = additive_log_ratio(&[0.5, 0.5]).expect("pair"); assert_eq!(pair.len(), 1); assert!(pair[0].abs() < 1e-15); diff --git a/crates/topic_measurement/src/error.rs b/crates/topic_measurement/src/error.rs index f2922574..ea934e52 100644 --- a/crates/topic_measurement/src/error.rs +++ b/crates/topic_measurement/src/error.rs @@ -9,7 +9,7 @@ pub enum TopicMeasurementError { /// Composition is empty, has fewer than two parts, is non-positive, /// non-finite, or does not sum to one. InvalidComposition, - /// Log-ratio vector is empty or non-finite. + /// Log-ratio vector is empty, non-finite, or not representable as a strictly positive `f64` simplex. InvalidLogRatioDimension, /// TF-IDF, BM25, or keyword scores were offered as inferential coordinates. LexicalWeightForbidden, diff --git a/crates/topic_measurement/src/lib.rs b/crates/topic_measurement/src/lib.rs index 013732ac..45d19f84 100644 --- a/crates/topic_measurement/src/lib.rs +++ b/crates/topic_measurement/src/lib.rs @@ -2,9 +2,11 @@ #![deny(missing_docs)] //! Logistic-normal and log-ratio coordinates for compositional topic proportions. //! -//! Raw topic proportions are not Euclidean indicators. Downstream network and -//! psychometric analysis must use additive log-ratio (logistic-normal) maps -//! rather than TF-IDF, BM25, or keyword scores as inferential coordinates. +//! Raw topic proportions are compositional rather than unconstrained Euclidean +//! indicators. ALR supplies a reference-dependent full-rank logistic-normal map +//! for regression and psychometric interfaces; it is not an orthonormal +//! Aitchison-distance isometry. Distance-based Aitchison geometry requires ILR. +//! TF-IDF, BM25, and keyword scores remain forbidden inferential coordinates. mod coordinates; mod error; diff --git a/docs/TRACEABILITY.md b/docs/TRACEABILITY.md index 9bd65faa..b18b296e 100644 --- a/docs/TRACEABILITY.md +++ b/docs/TRACEABILITY.md @@ -22,7 +22,7 @@ The full APA 7th standards/literature register remains `docs/research/standards- | versioned service/API contracts and exports | PRD; API contract; ADR 0011/0013 | `tepp_api` analysis-run/export/JSON-LD/GraphML contracts on protected main (PR #21); HTTP service remaining accepted-target | partial | | immutable split/run/reproducibility manifests | ADR 0013; ERD | `tepp_api` reproducibility manifest contract on protected main; `persistence_postgres` append-only SQL insert/lookup for `reproducibility_manifest`, `corpus_split_manifest`, `model_run`, and `model_artifact` (migration `0003`); full physical ERD constraints remaining | partial | | multilingual shared latent semantic space | PRD; ADR 0004 | future semantic/concept/topic crates | accepted-target | -| TRSL-TM temporal/relational topic posterior and backend compatibility | ADR 0012; ADR 0004 | `topic_measurement` ALR coordinates on the active PR; STM backend remaining | partial | +| TRSL-TM temporal/relational topic posterior and backend compatibility | ADR 0012; ADR 0004 | `topic_measurement` stable ALR coordinates on the active PR; temporal STM backend remaining | partial | | global P0 topic identity with activity/dormancy/reactivation | ADR 0012 | future topic lineage/activity state | accepted-target | | no default stopword deletion / no TF-IDF-BM25 inferential weighting | ADR 0004/0012; PRD/TRD | `topic_measurement::refuse_lexical_inferential_weight` on the active PR; preprocessing pipeline remaining | partial | | report template/section/copied/style/modality method effects | ADR 0004/0012; PRD/TRD | simulation truth factors implemented; estimator-side method model remains future | partial | diff --git a/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md b/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md index c2262965..b938a730 100644 --- a/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md +++ b/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md @@ -25,6 +25,7 @@ For the first production line: - stopword deletion is not the default preprocessing rule; - TF-IDF and BM25 are not inferential weights for the statistical topic estimator; - topic proportions are compositional and downstream network/psychometric analysis uses logistic-normal coordinates or valid orthonormal log-ratio coordinates; +- ALR is a reference-dependent full-rank logistic-normal map, not an Aitchison-distance isometry; distance-based Euclidean Aitchison geometry uses an orthonormal ILR basis; - model selection uses statistical/recovery/stability/alignment/fairness gates and a Pareto-style comparison before any blinded LLM review; - the LLM may recommend among statistically admissible candidates but never defines the numerical optimum or bypasses diagnostics. diff --git a/docs/research/topic-logratio-coordinates.md b/docs/research/topic-logratio-coordinates.md index a3514cd9..cc79837a 100644 --- a/docs/research/topic-logratio-coordinates.md +++ b/docs/research/topic-logratio-coordinates.md @@ -4,10 +4,11 @@ This note doctors the first `topic_measurement` production slice (ADR 0012): -1. raw topic proportions are not Euclidean indicators; -2. additive log-ratio coordinates implement the logistic-normal map used by correlated topic models; -3. inverse ALR recovers a known simplex with a computed RMSE; -4. TF-IDF, BM25, and keyword scores are refused as inferential coordinates. +1. raw topic proportions are compositional rather than unconstrained Euclidean indicators; +2. additive log-ratio coordinates implement the reference-dependent logistic-normal map used by correlated topic models; +3. ALR is full rank but not an orthonormal Aitchison-distance isometry; ILR is required when that Euclidean geometry is the estimand; +4. max-shifted inverse ALR and log-difference forward ALR recover representable extreme coordinates without overflow; +5. TF-IDF, BM25, and keyword scores are refused as inferential coordinates. The temporal STM backend, global topic identity, method-effect model, and K-selection remain accepted-target. No database migration is allocated. @@ -21,11 +22,13 @@ Blei, D. M., & Lafferty, J. D. (2007). A correlated topic model of Science. *The ## Application -Aitchison and Shen (1980) define the logistic-normal family via the additive log-ratio map; Aitchison (1982) is the compositional-data authority that forbids treating parts of a whole as unconstrained Euclidean coordinates. Blei and Lafferty (2007) use that same map for correlated topic models. TEPP therefore converts a strictly positive unit simplex through `additive_log_ratio` before any Euclidean or psychometric operation, and recovers the simplex with `from_additive_log_ratio` (Aitchison & Shen, 1980; Aitchison, 1982; Blei & Lafferty, 2007). +Aitchison and Shen (1980) define the logistic-normal family via the additive log-ratio map; Aitchison (1982) is the compositional-data authority that forbids treating parts of a whole as unconstrained Euclidean coordinates. Blei and Lafferty (2007) use that same reference-dependent map for correlated topic models. TEPP therefore uses `additive_log_ratio` for logistic-normal regression and psychometric interfaces, but does not claim that ALR preserves Aitchison distance. Analyses whose estimand is orthonormal Euclidean Aitchison geometry must use ILR. `from_additive_log_ratio` uses a max-shifted inverse softmax and the forward map subtracts logarithms, avoiding avoidable exponential and ratio overflow while failing closed when an `f64` simplex part would underflow to zero (Aitchison & Shen, 1980; Aitchison, 1982; Blei & Lafferty, 2007). ## Verification - closed-form simplex `(2,3,1)/6` maps to `(ln 2, ln 3)` and inverts with computed RMSE below `1e-15`; +- representable coordinates `(710, 709)` round-trip through the max-shifted inverse without exponential overflow; +- extremes that would underflow a strictly positive `f64` simplex part fail closed; - equal shares map to a zero ALR vector; - zero, negative, non-unit-sum, non-finite, empty, and one-part vectors fail closed; - `tfidf`, `bm25`, and `keyword` labels are refused. diff --git a/docs/validation/temporal-event-foundation.md b/docs/validation/temporal-event-foundation.md index 307624d7..9749f8af 100644 --- a/docs/validation/temporal-event-foundation.md +++ b/docs/validation/temporal-event-foundation.md @@ -25,7 +25,7 @@ This report tracks exact-head scientific and engineering evidence required befor | Versioned API/export contracts | `tepp_api` | implemented-main | naruon HTTP interchange | unknown-field/version/limit + naruon HTTPS interchange tests | Task 12 / PR #21; live HTTP service remaining | | Purpose-bound provider payloads | `tepp_api` | implemented-main | provider-payload minimization | expired/not-yet-valid/inverted/cross-tenant/impossible-calendar grant, mapping refusal, audited elevated re-id replay | ADR 0009; `docs/research/provider-payload-minimization.md` | | Adaptive orchestration router | `tepp_api` | implemented-main | router + ablation | mode selection, document-control denial, ablation, credential-free bind | ADR 0010; `docs/research/adaptive-orchestration-router.md` | -| Logistic-normal topic coordinates | `topic_measurement` | active-PR | ALR + lexical refusal | true-parameter ALR RMSE | ADR 0012; `docs/research/topic-logratio-coordinates.md` | +| Logistic-normal topic coordinates | `topic_measurement` | active-PR | stable ALR + lexical refusal | known-simplex and extreme-coordinate RMSE, ALR/ILR boundary | ADR 0012; `docs/research/topic-logratio-coordinates.md` | | CWL modular connectors | `docs/connectors/*` | implemented-main | — | contract docs + examples | PR #22; live HTTP ports remaining | | Release SBOM/provenance generator | `scripts/release_evidence.py` | partial | — | generate+validate in CI | Task 13 partial / PR #28 | diff --git a/scripts/repair_pr48_logratio_stability.py b/scripts/repair_pr48_logratio_stability.py deleted file mode 100644 index cc6f4dae..00000000 --- a/scripts/repair_pr48_logratio_stability.py +++ /dev/null @@ -1,258 +0,0 @@ -"""Apply PR 48 stable log-ratio arithmetic and documentation repairs.""" - -from pathlib import Path - - -def replace_once(text: str, old: str, new: str, label: str) -> str: - """Replace exactly one fragment or fail closed.""" - count = text.count(old) - if count != 1: - raise SystemExit(f"{label}: expected one target, found {count}") - return text.replace(old, new, 1) - - -def update_coordinates() -> None: - """Use stable log differences and max-shifted inverse softmax.""" - path = Path("crates/topic_measurement/src/coordinates.rs") - text = path.read_text(encoding="utf-8") - text = replace_once( - text, - """/// For a `K`-part composition `θ` the image is the `K-1` vector -/// `y_k = ln(θ_k / θ_K)`. This is the logistic-normal coordinate system used -/// by correlated topic models and required before Euclidean or ESEM/DSEM work. -""", - """/// For a `K`-part composition `θ` the image is the `K-1` vector -/// `y_k = ln(θ_k / θ_K)`. This reference-dependent, full-rank coordinate map -/// supports logistic-normal regression and ESEM/DSEM interfaces. It is not an -/// orthonormal isometry for Aitchison distance; use ILR coordinates when that -/// Euclidean geometry is the estimand. -""", - "coordinate documentation", - ) - text = replace_once( - text, - ".map(|part| (part / last).ln())", - ".map(|part| part.ln() - last.ln())", - "stable forward ALR", - ) - old_inverse = """pub fn from_additive_log_ratio(coordinates: &[f64]) -> Result, TopicMeasurementError> { - if coordinates.is_empty() { - return Err(TopicMeasurementError::InvalidLogRatioDimension); - } - let mut exponentiated = Vec::with_capacity(coordinates.len()); - let mut denom = 1.0_f64; - for &value in coordinates { - if !value.is_finite() { - return Err(TopicMeasurementError::InvalidLogRatioDimension); - } - let exp = value.exp(); - if !exp.is_finite() { - return Err(TopicMeasurementError::InvalidLogRatioDimension); - } - denom += exp; - exponentiated.push(exp); - } - if !denom.is_finite() || denom <= 0.0 { - return Err(TopicMeasurementError::InvalidLogRatioDimension); - } - let mut simplex = Vec::with_capacity(coordinates.len() + 1); - for exp in exponentiated { - simplex.push(exp / denom); - } - simplex.push(1.0 / denom); - Ok(simplex) -} -""" - new_inverse = """pub fn from_additive_log_ratio(coordinates: &[f64]) -> Result, TopicMeasurementError> { - if coordinates.is_empty() { - return Err(TopicMeasurementError::InvalidLogRatioDimension); - } - let mut maximum = 0.0_f64; - for &value in coordinates { - if !value.is_finite() { - return Err(TopicMeasurementError::InvalidLogRatioDimension); - } - maximum = maximum.max(value); - } - - let reference_weight = (-maximum).exp(); - if reference_weight == 0.0 { - return Err(TopicMeasurementError::InvalidLogRatioDimension); - } - let mut shifted_weights = Vec::with_capacity(coordinates.len()); - let mut denominator = reference_weight; - for &value in coordinates { - let weight = (value - maximum).exp(); - if weight == 0.0 { - return Err(TopicMeasurementError::InvalidLogRatioDimension); - } - denominator += weight; - shifted_weights.push(weight); - } - - let mut simplex = Vec::with_capacity(coordinates.len() + 1); - for weight in shifted_weights { - simplex.push(weight / denominator); - } - simplex.push(reference_weight / denominator); - Ok(simplex) -} -""" - text = replace_once(text, old_inverse, new_inverse, "stable inverse ALR") - text = replace_once( - text, - """/// Returns [`TopicMeasurementError::InvalidLogRatioDimension`] when the -/// coordinate vector is empty or contains a non-finite value. -""", - """/// Returns [`TopicMeasurementError::InvalidLogRatioDimension`] when the -/// coordinate vector is empty, non-finite, or would underflow a part to zero -/// in the strictly positive `f64` simplex representation. -""", - "inverse error documentation", - ) - text = replace_once( - text, - "fn two_part_equal_shares_are_zero_and_overflow_fails_closed()", - "fn two_part_equal_shares_are_zero_and_unrepresentable_extremes_fail_closed()", - "internal test name", - ) - path.write_text(text, encoding="utf-8") - - -def update_error_and_crate_docs() -> None: - """Describe representability and ALR geometry without overclaiming isometry.""" - error_path = Path("crates/topic_measurement/src/error.rs") - error_text = error_path.read_text(encoding="utf-8") - error_text = replace_once( - error_text, - " /// Log-ratio vector is empty or non-finite.\n", - " /// Log-ratio vector is empty, non-finite, or not representable as a strictly positive `f64` simplex.\n", - "error variant documentation", - ) - error_path.write_text(error_text, encoding="utf-8") - - lib_path = Path("crates/topic_measurement/src/lib.rs") - lib_text = lib_path.read_text(encoding="utf-8") - lib_text = replace_once( - lib_text, - """//! Raw topic proportions are not Euclidean indicators. Downstream network and -//! psychometric analysis must use additive log-ratio (logistic-normal) maps -//! rather than TF-IDF, BM25, or keyword scores as inferential coordinates. -""", - """//! Raw topic proportions are compositional rather than unconstrained Euclidean -//! indicators. ALR supplies a reference-dependent full-rank logistic-normal map -//! for regression and psychometric interfaces; it is not an orthonormal -//! Aitchison-distance isometry. Distance-based Aitchison geometry requires ILR. -//! TF-IDF, BM25, and keyword scores remain forbidden inferential coordinates. -""", - "crate geometry documentation", - ) - lib_path.write_text(lib_text, encoding="utf-8") - - -def update_research_and_adr() -> None: - """Clarify ALR versus ILR and record stable arithmetic evidence.""" - research_path = Path("docs/research/topic-logratio-coordinates.md") - research = research_path.read_text(encoding="utf-8") - research = replace_once( - research, - """1. raw topic proportions are not Euclidean indicators; -2. additive log-ratio coordinates implement the logistic-normal map used by correlated topic models; -3. inverse ALR recovers a known simplex with a computed RMSE; -4. TF-IDF, BM25, and keyword scores are refused as inferential coordinates. -""", - """1. raw topic proportions are compositional rather than unconstrained Euclidean indicators; -2. additive log-ratio coordinates implement the reference-dependent logistic-normal map used by correlated topic models; -3. ALR is full rank but not an orthonormal Aitchison-distance isometry; ILR is required when that Euclidean geometry is the estimand; -4. max-shifted inverse ALR and log-difference forward ALR recover representable extreme coordinates without overflow; -5. TF-IDF, BM25, and keyword scores are refused as inferential coordinates. -""", - "research scope", - ) - research = replace_once( - research, - """Aitchison and Shen (1980) define the logistic-normal family via the additive log-ratio map; Aitchison (1982) is the compositional-data authority that forbids treating parts of a whole as unconstrained Euclidean coordinates. Blei and Lafferty (2007) use that same map for correlated topic models. TEPP therefore converts a strictly positive unit simplex through `additive_log_ratio` before any Euclidean or psychometric operation, and recovers the simplex with `from_additive_log_ratio` (Aitchison & Shen, 1980; Aitchison, 1982; Blei & Lafferty, 2007). -""", - """Aitchison and Shen (1980) define the logistic-normal family via the additive log-ratio map; Aitchison (1982) is the compositional-data authority that forbids treating parts of a whole as unconstrained Euclidean coordinates. Blei and Lafferty (2007) use that same reference-dependent map for correlated topic models. TEPP therefore uses `additive_log_ratio` for logistic-normal regression and psychometric interfaces, but does not claim that ALR preserves Aitchison distance. Analyses whose estimand is orthonormal Euclidean Aitchison geometry must use ILR. `from_additive_log_ratio` uses a max-shifted inverse softmax and the forward map subtracts logarithms, avoiding avoidable exponential and ratio overflow while failing closed when an `f64` simplex part would underflow to zero (Aitchison & Shen, 1980; Aitchison, 1982; Blei & Lafferty, 2007). -""", - "research application", - ) - research = replace_once( - research, - """- closed-form simplex `(2,3,1)/6` maps to `(ln 2, ln 3)` and inverts with computed RMSE below `1e-15`; -- equal shares map to a zero ALR vector; -""", - """- closed-form simplex `(2,3,1)/6` maps to `(ln 2, ln 3)` and inverts with computed RMSE below `1e-15`; -- representable coordinates `(710, 709)` round-trip through the max-shifted inverse without exponential overflow; -- extremes that would underflow a strictly positive `f64` simplex part fail closed; -- equal shares map to a zero ALR vector; -""", - "research verification", - ) - research_path.write_text(research, encoding="utf-8") - - adr_path = Path("docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md") - adr = adr_path.read_text(encoding="utf-8") - adr = replace_once( - adr, - """- topic proportions are compositional and downstream network/psychometric analysis uses logistic-normal coordinates or valid orthonormal log-ratio coordinates; -""", - """- topic proportions are compositional and downstream network/psychometric analysis uses logistic-normal coordinates or valid orthonormal log-ratio coordinates; -- ALR is a reference-dependent full-rank logistic-normal map, not an Aitchison-distance isometry; distance-based Euclidean Aitchison geometry uses an orthonormal ILR basis; -""", - "ADR ALR/ILR boundary", - ) - adr_path.write_text(adr, encoding="utf-8") - - -def restore_conflict_resolved_ledgers() -> None: - """Reapply the topic slice to main-owned shared ledgers after the merge.""" - changelog_path = Path("CHANGELOG.md") - changelog = changelog_path.read_text(encoding="utf-8") - item = ( - "- `topic_measurement` logistic-normal additive log-ratio coordinates: " - "fail-closed simplex validation, max-shifted stable ALR/inverse maps with " - "true-parameter round-trip RMSE, explicit ALR-versus-ILR geometry boundary, " - "and refusal of TF-IDF/BM25/keyword scores as inferential topic coordinates " - "(ADR 0012 first production slice; no new migration).\n" - ) - if item not in changelog: - changelog = replace_once(changelog, "### Added\n\n", "### Added\n\n" + item, "changelog marker") - changelog_path.write_text(changelog, encoding="utf-8") - - trace_path = Path("docs/TRACEABILITY.md") - trace = trace_path.read_text(encoding="utf-8") - trace = replace_once( - trace, - "| TRSL-TM temporal/relational topic posterior and backend compatibility | ADR 0012; ADR 0004 | future `topic_measurement` | accepted-target |", - "| TRSL-TM temporal/relational topic posterior and backend compatibility | ADR 0012; ADR 0004 | `topic_measurement` stable ALR coordinates on the active PR; temporal STM backend remaining | partial |", - "trace topic backend", - ) - trace = replace_once( - trace, - "| no default stopword deletion / no TF-IDF-BM25 inferential weighting | ADR 0004/0012; PRD/TRD | future semantic/method-source model | accepted-target |", - "| no default stopword deletion / no TF-IDF-BM25 inferential weighting | ADR 0004/0012; PRD/TRD | `topic_measurement::refuse_lexical_inferential_weight` on the active PR; preprocessing pipeline remaining | partial |", - "trace lexical refusal", - ) - trace_path.write_text(trace, encoding="utf-8") - - validation_path = Path("docs/validation/temporal-event-foundation.md") - validation = validation_path.read_text(encoding="utf-8") - row = ( - "| Logistic-normal topic coordinates | `topic_measurement` | active-PR | stable ALR + lexical refusal | " - "known-simplex and extreme-coordinate RMSE, ALR/ILR boundary | ADR 0012; " - "`docs/research/topic-logratio-coordinates.md` |\n" - ) - if row not in validation: - marker = ( - "| Versioned API/export contracts | `tepp_api` | implemented-main | naruon HTTP interchange | " - "unknown-field/version/limit + naruon HTTPS interchange tests | Task 12 / PR #21; live HTTP service remaining |\n" - ) - validation = replace_once(validation, marker, marker + row, "validation API row") - validation_path.write_text(validation, encoding="utf-8") - - -update_coordinates() -update_error_and_crate_docs() -update_research_and_adr() -restore_conflict_resolved_ledgers() From 5106f4bd451daca21b9e5fd3b57105a932e19b0b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 14 Aug 2026 22:02:30 +0900 Subject: [PATCH 07/26] docs(topic): specify stable inverse reference coordinate --- docs/research/topic-logratio-coordinates.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/research/topic-logratio-coordinates.md b/docs/research/topic-logratio-coordinates.md index cc79837a..a1315a16 100644 --- a/docs/research/topic-logratio-coordinates.md +++ b/docs/research/topic-logratio-coordinates.md @@ -22,7 +22,7 @@ Blei, D. M., & Lafferty, J. D. (2007). A correlated topic model of Science. *The ## Application -Aitchison and Shen (1980) define the logistic-normal family via the additive log-ratio map; Aitchison (1982) is the compositional-data authority that forbids treating parts of a whole as unconstrained Euclidean coordinates. Blei and Lafferty (2007) use that same reference-dependent map for correlated topic models. TEPP therefore uses `additive_log_ratio` for logistic-normal regression and psychometric interfaces, but does not claim that ALR preserves Aitchison distance. Analyses whose estimand is orthonormal Euclidean Aitchison geometry must use ILR. `from_additive_log_ratio` uses a max-shifted inverse softmax and the forward map subtracts logarithms, avoiding avoidable exponential and ratio overflow while failing closed when an `f64` simplex part would underflow to zero (Aitchison & Shen, 1980; Aitchison, 1982; Blei & Lafferty, 2007). +Aitchison and Shen (1980) define the logistic-normal family via the additive log-ratio map; Aitchison (1982) is the compositional-data authority that forbids treating parts of a whole as unconstrained Euclidean coordinates. Blei and Lafferty (2007) use that same reference-dependent map for correlated topic models. TEPP therefore uses `additive_log_ratio` for logistic-normal regression and psychometric interfaces, but does not claim that ALR preserves Aitchison distance. Analyses whose estimand is orthonormal Euclidean Aitchison geometry must use ILR. `from_additive_log_ratio` treats the omitted reference component as logit zero, max-shifts all `K` logits together, and normalizes only after exponentiation. The forward map subtracts logarithms rather than forming a potentially overflowing ratio. This avoids avoidable exponential and ratio overflow while failing closed when an `f64` simplex part would underflow to zero (Aitchison & Shen, 1980; Aitchison, 1982; Blei & Lafferty, 2007). ## Verification From 0269df004a69673476cebf219c40c1303124b059 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 14 Aug 2026 22:15:34 +0900 Subject: [PATCH 08/26] test(topic): detect hidden simplex mass with compensated summation --- .../composition_sum_precision_contract.rs | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 crates/topic_measurement/tests/composition_sum_precision_contract.rs diff --git a/crates/topic_measurement/tests/composition_sum_precision_contract.rs b/crates/topic_measurement/tests/composition_sum_precision_contract.rs new file mode 100644 index 00000000..2fa8874c --- /dev/null +++ b/crates/topic_measurement/tests/composition_sum_precision_contract.rs @@ -0,0 +1,30 @@ +//! Composition validation must not lose tiny positive mass after a dominant part. + +use topic_measurement::{TopicMeasurementError, additive_log_ratio}; + +#[test] +fn compensated_sum_rejects_mass_hidden_by_naive_floating_point_addition() { + let mut composition = Vec::with_capacity(20_001); + composition.push(1.0); + composition.extend(std::iter::repeat_n(1.0e-16, 20_000)); + + assert_eq!( + additive_log_ratio(&composition), + Err(TopicMeasurementError::InvalidComposition), + "the true mass exceeds one by 2e-12 even though naive ordered addition rounds to one" + ); +} + +#[test] +fn compensated_sum_accepts_a_valid_many_part_composition() { + let tiny_mass = 1.0e-16; + let tiny_parts = 10_000_usize; + let dominant = 1.0 - tiny_mass * tiny_parts as f64; + let mut composition = Vec::with_capacity(tiny_parts + 1); + composition.push(dominant); + composition.extend(std::iter::repeat_n(tiny_mass, tiny_parts)); + + let coordinates = additive_log_ratio(&composition).expect("valid unit simplex"); + assert_eq!(coordinates.len(), tiny_parts); + assert!(coordinates.iter().all(|value| value.is_finite())); +} From d70f5c73972a32019eefa9595e786ecbc24349a1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 14 Aug 2026 22:16:09 +0900 Subject: [PATCH 09/26] fix(topic): validate simplex mass with compensated summation --- crates/topic_measurement/src/coordinates.rs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/crates/topic_measurement/src/coordinates.rs b/crates/topic_measurement/src/coordinates.rs index 1909df2f..94fa810f 100644 --- a/crates/topic_measurement/src/coordinates.rs +++ b/crates/topic_measurement/src/coordinates.rs @@ -19,9 +19,10 @@ const UNIT_SUM_TOLERANCE: f64 = 1e-12; /// entry, or does not sum to one within a tight absolute tolerance. pub fn additive_log_ratio(proportions: &[f64]) -> Result, TopicMeasurementError> { let last = require_composition(proportions)?; + let reference_log = last.ln(); Ok(proportions[..proportions.len() - 1] .iter() - .map(|part| part.ln() - last.ln()) + .map(|part| part.ln() - reference_log) .collect()) } @@ -72,13 +73,23 @@ fn require_composition(proportions: &[f64]) -> Result= part.abs() { + (sum - next) + part + } else { + (part - next) + sum + }; + sum = next; } - if !sum.is_finite() || (sum - 1.0).abs() > UNIT_SUM_TOLERANCE { + let compensated_sum = sum + compensation; + if !compensated_sum.is_finite() + || (compensated_sum - 1.0).abs() > UNIT_SUM_TOLERANCE + { return Err(TopicMeasurementError::InvalidComposition); } Ok(proportions[proportions.len() - 1]) From 0c3d88d42f5386a2f2444649453b4698235ba5a1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 14 Aug 2026 22:16:46 +0900 Subject: [PATCH 10/26] test(topic): keep compensated-sum fixture clippy-clean --- .../tests/composition_sum_precision_contract.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/topic_measurement/tests/composition_sum_precision_contract.rs b/crates/topic_measurement/tests/composition_sum_precision_contract.rs index 2fa8874c..9cdd1cfe 100644 --- a/crates/topic_measurement/tests/composition_sum_precision_contract.rs +++ b/crates/topic_measurement/tests/composition_sum_precision_contract.rs @@ -19,7 +19,7 @@ fn compensated_sum_rejects_mass_hidden_by_naive_floating_point_addition() { fn compensated_sum_accepts_a_valid_many_part_composition() { let tiny_mass = 1.0e-16; let tiny_parts = 10_000_usize; - let dominant = 1.0 - tiny_mass * tiny_parts as f64; + let dominant = 1.0 - tiny_mass * 10_000.0; let mut composition = Vec::with_capacity(tiny_parts + 1); composition.push(dominant); composition.extend(std::iter::repeat_n(tiny_mass, tiny_parts)); From 0d7c65fe6c76f32b41229a373e586a058db04fe4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 14 Aug 2026 22:45:35 +0900 Subject: [PATCH 11/26] style(topic): apply pinned rustfmt output --- crates/topic_measurement/src/coordinates.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/topic_measurement/src/coordinates.rs b/crates/topic_measurement/src/coordinates.rs index 94fa810f..230b9912 100644 --- a/crates/topic_measurement/src/coordinates.rs +++ b/crates/topic_measurement/src/coordinates.rs @@ -87,9 +87,7 @@ fn require_composition(proportions: &[f64]) -> Result UNIT_SUM_TOLERANCE - { + if !compensated_sum.is_finite() || (compensated_sum - 1.0).abs() > UNIT_SUM_TOLERANCE { return Err(TopicMeasurementError::InvalidComposition); } Ok(proportions[proportions.len() - 1]) From f3540faa80e788a82fbe7950e3f4b0bdf700c89f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 14 Aug 2026 22:46:24 +0900 Subject: [PATCH 12/26] test(quality): include topic measurement in docstring inventory --- tests/quality/test_check_docstrings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/quality/test_check_docstrings.py b/tests/quality/test_check_docstrings.py index 2c11f7a5..b99537c5 100644 --- a/tests/quality/test_check_docstrings.py +++ b/tests/quality/test_check_docstrings.py @@ -24,7 +24,7 @@ def test_live_repository_is_documented(self) -> None: sources = docstrings.rust_sources(REPOSITORY_ROOT) crate_roots = sorted(REPOSITORY_ROOT.glob("crates/*/src/lib.rs")) - self.assertEqual(len(crate_roots), 10) + self.assertEqual(len(crate_roots), 11) self.assertTrue(set(crate_roots).issubset(sources)) self.assertGreaterEqual(len(sources), len(crate_roots)) self.assertEqual(docstrings.validate_repository(REPOSITORY_ROOT), []) From ab29247a3cd138a140daa9f9e8d4b7b2302fe471 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 17 Aug 2026 13:06:45 +0000 Subject: [PATCH 13/26] test(topic): cover non-finite compensated simplex mass Exercise overflowing finite parts so the nightly branch gate hits !compensated_sum.is_finite(), and assert true-parameter ALR RMSE against the closed-form (ln 2, ln 3) coordinates. Co-authored-by: Seongho Bae --- crates/topic_measurement/src/coordinates.rs | 8 +++++++ .../composition_sum_precision_contract.rs | 9 +++++++ .../tests/logratio_recovery_contract.rs | 24 +++++++++++++++---- 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/crates/topic_measurement/src/coordinates.rs b/crates/topic_measurement/src/coordinates.rs index 230b9912..993e11d2 100644 --- a/crates/topic_measurement/src/coordinates.rs +++ b/crates/topic_measurement/src/coordinates.rs @@ -103,9 +103,17 @@ mod tests { let pair = additive_log_ratio(&[0.5, 0.5]).expect("pair"); assert_eq!(pair.len(), 1); assert!(pair[0].abs() < 1e-15); + let recovered = from_additive_log_ratio(&pair).expect("inverse"); + assert!((recovered[0] - 0.5).abs() < 1e-15); + assert!((recovered[1] - 0.5).abs() < 1e-15); assert_eq!( from_additive_log_ratio(&[1.0e9]), Err(TopicMeasurementError::InvalidLogRatioDimension) ); + assert_eq!( + additive_log_ratio(&[f64::MAX, f64::MAX]), + Err(TopicMeasurementError::InvalidComposition), + "overflowing finite parts must fail closed because compensated mass is non-finite" + ); } } diff --git a/crates/topic_measurement/tests/composition_sum_precision_contract.rs b/crates/topic_measurement/tests/composition_sum_precision_contract.rs index 9cdd1cfe..f76cb763 100644 --- a/crates/topic_measurement/tests/composition_sum_precision_contract.rs +++ b/crates/topic_measurement/tests/composition_sum_precision_contract.rs @@ -15,6 +15,15 @@ fn compensated_sum_rejects_mass_hidden_by_naive_floating_point_addition() { ); } +#[test] +fn overflowing_finite_parts_fail_closed_as_non_finite_mass() { + assert_eq!( + additive_log_ratio(&[f64::MAX, f64::MAX]), + Err(TopicMeasurementError::InvalidComposition), + "Kahan-compensated MAX+MAX is non-finite; NaN cannot pass a unit-sum tolerance comparison" + ); +} + #[test] fn compensated_sum_accepts_a_valid_many_part_composition() { let tiny_mass = 1.0e-16; diff --git a/crates/topic_measurement/tests/logratio_recovery_contract.rs b/crates/topic_measurement/tests/logratio_recovery_contract.rs index e6af5f8d..2b50250d 100644 --- a/crates/topic_measurement/tests/logratio_recovery_contract.rs +++ b/crates/topic_measurement/tests/logratio_recovery_contract.rs @@ -24,15 +24,19 @@ fn known_simplex_recovers_through_alr_with_computed_rmse() { // Closed-form simplex: (2, 3, 1) / 6. ALR is (ln 2, ln 3). let truth = [2.0 / 6.0, 3.0 / 6.0, 1.0 / 6.0]; let coordinates = additive_log_ratio(&truth).expect("alr"); + let true_parameters = [2.0_f64.ln(), 3.0_f64.ln()]; assert_eq!(coordinates.len(), 2); - assert!((coordinates[0] - 2.0_f64.ln()).abs() < 1e-15); - assert!((coordinates[1] - 3.0_f64.ln()).abs() < 1e-15); + let parameter_rmse = rmse(&true_parameters, &coordinates); + assert!( + parameter_rmse < 1e-15, + "true-parameter ALR RMSE {parameter_rmse} exceeded machine-scale bound" + ); let recovered = from_additive_log_ratio(&coordinates).expect("inverse"); - let error = rmse(&truth, &recovered); + let simplex_rmse = rmse(&truth, &recovered); assert!( - error < 1e-15, - "ALR round-trip RMSE {error} exceeded machine-scale bound" + simplex_rmse < 1e-15, + "ALR round-trip RMSE {simplex_rmse} exceeded machine-scale bound" ); let sum: f64 = recovered.iter().sum(); assert!((sum - 1.0).abs() < 1e-15); @@ -98,11 +102,21 @@ fn invalid_compositions_and_lexical_weights_fail_closed() { from_additive_log_ratio(&[f64::INFINITY]), Err(TopicMeasurementError::InvalidLogRatioDimension) ); + assert_eq!( + from_additive_log_ratio(&[1.0e9]), + Err(TopicMeasurementError::InvalidLogRatioDimension), + "max-shifted reference weight must fail closed when it underflows to zero" + ); assert_eq!( from_additive_log_ratio(&[-1.0e9]), Err(TopicMeasurementError::InvalidLogRatioDimension), "inverse must not return a zero simplex part after underflow" ); + assert_eq!( + additive_log_ratio(&[f64::MAX, f64::MAX]), + Err(TopicMeasurementError::InvalidComposition), + "overflowing finite parts must fail closed because compensated mass is non-finite" + ); for method in ["tfidf", "bm25", "keyword", "TF-IDF", ""] { assert_eq!( From 4ae9b5df51beea9f8c8bf8aa13f5ad511ad5f800 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 17 Aug 2026 17:20:14 +0000 Subject: [PATCH 14/26] feat(topic): add sequential Egozcue ILR coordinates Implement isometric_log_ratio and its max-shifted inverse on the Egozcue sequential orthonormal basis, with true-parameter RMSE and Aitchison-distance recovery. ALR remains the logistic-normal map. Co-authored-by: Seongho Bae --- ARCHITECTURE.md | 2 +- CHANGELOG.md | 2 +- crates/topic_measurement/src/coordinates.rs | 102 +++++++++++++++- crates/topic_measurement/src/lib.rs | 9 +- .../tests/ilr_recovery_contract.rs | 111 ++++++++++++++++++ docs/TRACEABILITY.md | 2 +- ...ational-shared-latent-topic-measurement.md | 2 +- docs/adr/README.md | 2 +- docs/research/standards-and-literature.md | 2 + docs/research/topic-logratio-coordinates.md | 18 +-- docs/validation/temporal-event-foundation.md | 2 +- 11 files changed, 237 insertions(+), 17 deletions(-) create mode 100644 crates/topic_measurement/tests/ilr_recovery_contract.rs diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index e3212df5..f10bf73a 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -61,7 +61,7 @@ boundaries above remain the target modular MSA architecture. | `tepp_simulation` | known-truth temporal/event data generation | | `validation_core` | RMSE, bias, coverage, graph, and Monte Carlo metrics | | `tepp_api` | versioned DTO, schema, and export contracts | -| `topic_measurement` | logistic-normal / additive log-ratio topic coordinates | +| `topic_measurement` | logistic-normal ALR and sequential Egozcue ILR topic coordinates | No crate exposes placeholder production behavior in Task 1. This prevents an empty façade from becoming a de facto public API before its invariants and tests diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c43f24d..0ad9e324 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ All notable changes to TEPP are documented here. The format follows Keep a Chang ### Added -- `topic_measurement` logistic-normal additive log-ratio coordinates: fail-closed simplex validation, max-shifted stable ALR/inverse maps with true-parameter round-trip RMSE, explicit ALR-versus-ILR geometry boundary, and refusal of TF-IDF/BM25/keyword scores as inferential topic coordinates (ADR 0012 first production slice; no new migration). +- `topic_measurement` logistic-normal additive log-ratio and sequential Egozcue isometric log-ratio coordinates: fail-closed simplex validation, max-shifted stable ALR/ILR inverses with true-parameter RMSE, Aitchison-distance ILR isometry, and refusal of TF-IDF/BM25/keyword scores as inferential topic coordinates (ADR 0012 first production slice; no new migration). - `tepp_api` adaptive orchestration router (ADR 0010): versioned `direct`/`verify`/`committee`/`conductor`/`abstain` selection from CPU `f64` risk, ambiguity, evidence, and token-budget inputs; recorded stages, recursion, decomposition, access lists, and role-specific reasoning effort; fail-closed document-controlled policy/access/credentials; LLM plans remain proposals under deterministic statistical authority; comparable-budget ablation requires a direct baseline; credential-free contextual-orchestrator binding. Live NIM HTTP remains accepted-target. - `tepp_api` purpose-bound provider-payload minimization: time-bounded `PurposeGrant` evaluation, fail-closed expired/not-yet-valid/inverted/cross-tenant/impossible-calendar denial, semantic UTC calendar validation, refusal to copy identity mappings into model-provider payloads or ordinary logs, preservation of opaque analytical identifiers and membership roles (no blanket PII mask), a separately authorized scientific re-identification path, and an internally bound FIPS 180-4 SHA-256 audit digest appended through `ReidentificationAuditSink` before disclosure. - `persistence_postgres` backup/restore integrity: restored snapshots stay unusable until tenant, canonical `SHA-256`, knowledge-cutoff eligibility, temporal window order, and append-only triggers revalidate; SQL probes raise `restore integrity failed` (ADR 0013). diff --git a/crates/topic_measurement/src/coordinates.rs b/crates/topic_measurement/src/coordinates.rs index 993e11d2..1ab1306f 100644 --- a/crates/topic_measurement/src/coordinates.rs +++ b/crates/topic_measurement/src/coordinates.rs @@ -1,4 +1,4 @@ -//! Additive log-ratio maps for logistic-normal topic coordinates. +//! Additive and isometric log-ratio maps for compositional topic coordinates. use crate::error::TopicMeasurementError; @@ -68,6 +68,90 @@ pub fn from_additive_log_ratio(coordinates: &[f64]) -> Result, TopicMea Ok(simplex) } +/// Map a strictly positive unit simplex vector to isometric log-ratio coordinates. +/// +/// The sequential Egozcue orthonormal basis sends a `K`-part composition to +/// the `K-1` vector whose Euclidean norm equals Aitchison distance. This is +/// the coordinate system for distance-based topic geometry. It is not the +/// reference-dependent logistic-normal map; use [`additive_log_ratio`] when +/// that regression interface is the estimand. +/// +/// # Errors +/// +/// Returns [`TopicMeasurementError::InvalidComposition`] when the vector is +/// empty, has fewer than two parts, contains a non-finite or non-positive +/// entry, or does not sum to one within a tight absolute tolerance. +pub fn isometric_log_ratio(proportions: &[f64]) -> Result, TopicMeasurementError> { + require_composition(proportions)?; + let dimension = proportions.len(); + let logs: Vec = proportions.iter().map(|part| part.ln()).collect(); + let mut coordinates = Vec::with_capacity(dimension - 1); + for index in 0..(dimension - 1) { + let remaining = dimension - index - 1; + #[allow(clippy::cast_precision_loss)] + let remaining_f = remaining as f64; + let scale = (remaining_f / (remaining_f + 1.0)).sqrt(); + let mut rest_sum = 0.0_f64; + for log_part in &logs[index + 1..] { + rest_sum += *log_part; + } + coordinates.push(scale * (logs[index] - rest_sum / remaining_f)); + } + Ok(coordinates) +} + +/// Invert isometric log-ratio coordinates back to the unit simplex. +/// +/// # Errors +/// +/// Returns [`TopicMeasurementError::InvalidLogRatioDimension`] when the +/// coordinate vector is empty, non-finite, or would underflow a part to zero +/// in the strictly positive `f64` simplex representation. +pub fn from_isometric_log_ratio(coordinates: &[f64]) -> Result, TopicMeasurementError> { + if coordinates.is_empty() { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + for &value in coordinates { + if !value.is_finite() { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + } + + let dimension = coordinates.len() + 1; + let mut centered_logs = vec![0.0_f64; dimension]; + for (index, &coordinate) in coordinates.iter().enumerate() { + let remaining = dimension - index - 1; + #[allow(clippy::cast_precision_loss)] + let remaining_f = remaining as f64; + let scale = (remaining_f / (remaining_f + 1.0)).sqrt(); + let negative = -1.0 / (remaining_f * (remaining_f + 1.0)).sqrt(); + centered_logs[index] += scale * coordinate; + for centered in &mut centered_logs[index + 1..] { + *centered += negative * coordinate; + } + } + + let mut maximum = centered_logs[0]; + for &value in ¢ered_logs[1..] { + maximum = maximum.max(value); + } + if !maximum.is_finite() { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + + let mut weights = Vec::with_capacity(dimension); + let mut denominator = 0.0_f64; + for &value in ¢ered_logs { + let weight = (value - maximum).exp(); + if weight == 0.0 { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + denominator += weight; + weights.push(weight); + } + Ok(weights.iter().map(|weight| weight / denominator).collect()) +} + fn require_composition(proportions: &[f64]) -> Result { if proportions.len() < 2 { return Err(TopicMeasurementError::InvalidComposition); @@ -95,7 +179,9 @@ fn require_composition(proportions: &[f64]) -> Result f64 { + let n = truth.len() as f64; + let sum_sq: f64 = truth + .iter() + .zip(recovered) + .map(|(left, right)| { + let residual = left - right; + residual * residual + }) + .sum(); + (sum_sq / n).sqrt() +} + +#[test] +fn known_simplex_recovers_through_ilr_with_computed_rmse() { + // Closed-form simplex: (2, 3, 1) / 6. + // Sequential Egozcue ILR: y1 = √(2/3) ln(2√3 / 3), y2 = √(1/2) ln 3. + let truth = [2.0 / 6.0, 3.0 / 6.0, 1.0 / 6.0]; + let true_parameters = [ + (2.0_f64 / 3.0).sqrt() * (2.0 * 3.0_f64.sqrt() / 3.0).ln(), + (1.0_f64 / 2.0).sqrt() * 3.0_f64.ln(), + ]; + let coordinates = isometric_log_ratio(&truth).expect("ilr"); + assert_eq!(coordinates.len(), 2); + let parameter_rmse = rmse(&true_parameters, &coordinates); + assert!( + parameter_rmse < 1e-15, + "true-parameter ILR RMSE {parameter_rmse} exceeded machine-scale bound" + ); + + let recovered = from_isometric_log_ratio(&coordinates).expect("inverse"); + let simplex_rmse = rmse(&truth, &recovered); + assert!( + simplex_rmse < 1e-15, + "ILR round-trip RMSE {simplex_rmse} exceeded machine-scale bound" + ); + let sum: f64 = recovered.iter().sum(); + assert!((sum - 1.0).abs() < 1e-15); + + let alr = additive_log_ratio(&truth).expect("alr"); + assert!( + (alr[0] - coordinates[0]).abs() > 1e-6, + "ILR must not collapse to the reference-dependent ALR map" + ); +} + +#[test] +fn equal_shares_are_the_ilr_origin_and_preserve_aitchison_distance() { + let halves = [0.5, 0.5]; + let origin = isometric_log_ratio(&halves).expect("origin"); + assert_eq!(origin.len(), 1); + assert!(origin[0].abs() < 1e-15); + + let unbalanced = [0.8, 0.2]; + let coordinates = isometric_log_ratio(&unbalanced).expect("pair"); + let expected = (0.5_f64).sqrt() * 4.0_f64.ln(); + assert!((coordinates[0] - expected).abs() < 1e-15); + + let recovered = from_isometric_log_ratio(&coordinates).expect("inverse"); + assert!(rmse(&unbalanced, &recovered) < 1e-15); +} + +#[test] +fn large_finite_ilr_coordinates_round_trip_or_fail_closed() { + let representable = from_isometric_log_ratio(&[40.0]).expect("representable"); + assert!( + representable + .iter() + .all(|part| part.is_finite() && *part > 0.0) + ); + let recovered = isometric_log_ratio(&representable).expect("forward"); + assert!(rmse(&[40.0], &recovered) < 1e-12); + + assert_eq!( + from_isometric_log_ratio(&[1000.0]), + Err(TopicMeasurementError::InvalidLogRatioDimension), + "ILR inverse must not return a zero simplex part after underflow" + ); + assert_eq!( + from_isometric_log_ratio(&[f64::MAX]), + Err(TopicMeasurementError::InvalidLogRatioDimension), + "overflowing CLR reconstruction must fail closed" + ); +} + +#[test] +fn invalid_ilr_inputs_fail_closed() { + assert_eq!( + isometric_log_ratio(&[]), + Err(TopicMeasurementError::InvalidComposition) + ); + assert_eq!( + from_isometric_log_ratio(&[]), + Err(TopicMeasurementError::InvalidLogRatioDimension) + ); + assert_eq!( + from_isometric_log_ratio(&[f64::NAN]), + Err(TopicMeasurementError::InvalidLogRatioDimension) + ); + assert_eq!( + from_isometric_log_ratio(&[f64::INFINITY]), + Err(TopicMeasurementError::InvalidLogRatioDimension) + ); +} diff --git a/docs/TRACEABILITY.md b/docs/TRACEABILITY.md index b18b296e..7f6d90de 100644 --- a/docs/TRACEABILITY.md +++ b/docs/TRACEABILITY.md @@ -22,7 +22,7 @@ The full APA 7th standards/literature register remains `docs/research/standards- | versioned service/API contracts and exports | PRD; API contract; ADR 0011/0013 | `tepp_api` analysis-run/export/JSON-LD/GraphML contracts on protected main (PR #21); HTTP service remaining accepted-target | partial | | immutable split/run/reproducibility manifests | ADR 0013; ERD | `tepp_api` reproducibility manifest contract on protected main; `persistence_postgres` append-only SQL insert/lookup for `reproducibility_manifest`, `corpus_split_manifest`, `model_run`, and `model_artifact` (migration `0003`); full physical ERD constraints remaining | partial | | multilingual shared latent semantic space | PRD; ADR 0004 | future semantic/concept/topic crates | accepted-target | -| TRSL-TM temporal/relational topic posterior and backend compatibility | ADR 0012; ADR 0004 | `topic_measurement` stable ALR coordinates on the active PR; temporal STM backend remaining | partial | +| TRSL-TM temporal/relational topic posterior and backend compatibility | ADR 0012; ADR 0004 | `topic_measurement` stable ALR and sequential Egozcue ILR coordinates on the active PR; temporal STM backend remaining | partial | | global P0 topic identity with activity/dormancy/reactivation | ADR 0012 | future topic lineage/activity state | accepted-target | | no default stopword deletion / no TF-IDF-BM25 inferential weighting | ADR 0004/0012; PRD/TRD | `topic_measurement::refuse_lexical_inferential_weight` on the active PR; preprocessing pipeline remaining | partial | | report template/section/copied/style/modality method effects | ADR 0004/0012; PRD/TRD | simulation truth factors implemented; estimator-side method model remains future | partial | diff --git a/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md b/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md index b938a730..23cd4162 100644 --- a/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md +++ b/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md @@ -1,7 +1,7 @@ # ADR 0012 — Temporal Relational Shared-Latent Topic Measurement **Decision status:** Accepted -**Implementation maturity:** partial — logistic-normal additive log-ratio coordinates and lexical-weight refusal are implemented on the active PR and are not implemented-main until exact-head checks, review, and protected-main integration complete; temporal topic identity, backend STM estimator, method-effect model, and K-selection remain accepted-target +**Implementation maturity:** partial — logistic-normal additive log-ratio coordinates, sequential Egozcue isometric log-ratio coordinates, and lexical-weight refusal are implemented on the active PR and are not implemented-main until exact-head checks, review, and protected-main integration complete; temporal topic identity, backend STM estimator, method-effect model, and K-selection remain accepted-target **Date:** 2026-08-12 **Supersedes:** None; refines ADR 0004 and ADR 0005 without replacing their multilingual and psychometric authorities. diff --git a/docs/adr/README.md b/docs/adr/README.md index 4f7fe4bc..9d549d0d 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -17,7 +17,7 @@ Read [`ADR_POLICY.md`](ADR_POLICY.md) first. **Decision status and implementatio | [0009](0009-purpose-bound-pii-governance.md) | Purpose-bound PII governance without blanket masking | Accepted | partial | Persistence retention/deletion/legal-hold (`0007`) and provider-payload minimization implemented-main; deployment evidence remains accepted-target. | | [0010](0010-adaptive-llm-orchestration.md) | Adaptive LLM orchestration and test-time compute | Accepted | partial | `tepp_api` router/ablation/orchestrator binding implemented-main; live NIM execution and production ablation evidence remain accepted-target. | | [0011](0011-standalone-modular-msa-boundary.md) | Standalone operation and modular CWL MSA boundary | Accepted | partial | Owns cross-service persistence/credential/API authority; no direct cross-service application-table coupling. | -| [0012](0012-temporal-relational-shared-latent-topic-measurement.md) | Temporal Relational Shared-Latent Topic Measurement (TRSL-TM) | Accepted | partial | Logistic-normal ALR coordinates and lexical-weight refusal are on the active PR; temporal topic identity, STM backend, method effects, and K-selection remain accepted-target. | +| [0012](0012-temporal-relational-shared-latent-topic-measurement.md) | Temporal Relational Shared-Latent Topic Measurement (TRSL-TM) | Accepted | partial | Logistic-normal ALR, sequential Egozcue ILR, and lexical-weight refusal are on the active PR; temporal topic identity, STM backend, method effects, and K-selection remain accepted-target. | | [0013](0013-bitemporal-persistence-reproducibility-and-split-authority.md) | Bitemporal persistence, reproducibility manifests, and relation-aware split authority | Accepted | partial | Owns PostgreSQL adapter semantics, immutable run/split manifests, leakage-safe partitions, and recovery identity; optional `live-sqlx` `PgPool`, live PG CI, tenant RLS, and `0006` membership implemented-main; `0007` retention/deletion/legal-hold implemented-main; remaining physical ERD/backup accepted-target. | | [0014](0014-scientific-claim-promotion-and-release-evidence.md) | Scientific claim promotion and release evidence authority | Accepted | partial | Separates design, implementation, scientific/product claim, and release authority; repository SBOM/provenance generator implemented, full release bundle remaining. | | [0015](0015-autonomous-development-review-and-merge-authority.md) | Autonomous development, review, and merge authority separation | Accepted | active-PR | Separates model proposal, deterministic verification, publication, independent review, and merge/release authority. | diff --git a/docs/research/standards-and-literature.md b/docs/research/standards-and-literature.md index d793954a..54072f27 100644 --- a/docs/research/standards-and-literature.md +++ b/docs/research/standards-and-literature.md @@ -52,6 +52,8 @@ Aitchison, J., & Shen, S. M. (1980). Logistic-normal distributions: Some propert Aitchison, J. (1982). The statistical analysis of compositional data. *Journal of the Royal Statistical Society: Series B (Methodological), 44*(2), 139–177. https://doi.org/10.1111/j.2517-6161.1982.tb01195.x +Egozcue, J. J., Pawlowsky-Glahn, V., Mateu-Figueras, G., & Barceló-Vidal, C. (2003). Isometric logratio transformations for compositional data analysis. *Mathematical Geology, 35*(3), 279–300. https://doi.org/10.1023/A:1023818214614 + Friedman, J., Hastie, T., & Tibshirani, R. (2008). Sparse inverse covariance estimation with the graphical lasso. *Biostatistics, 9*(3), 432–441. https://doi.org/10.1093/biostatistics/kxm045 Traag, V. A., Waltman, L., & van Eck, N. J. (2019). From Louvain to Leiden: Guaranteeing well-connected communities. *Scientific Reports, 9*, Article 5233. https://doi.org/10.1038/s41598-019-41695-z diff --git a/docs/research/topic-logratio-coordinates.md b/docs/research/topic-logratio-coordinates.md index a1315a16..ac5c242d 100644 --- a/docs/research/topic-logratio-coordinates.md +++ b/docs/research/topic-logratio-coordinates.md @@ -6,9 +6,10 @@ This note doctors the first `topic_measurement` production slice (ADR 0012): 1. raw topic proportions are compositional rather than unconstrained Euclidean indicators; 2. additive log-ratio coordinates implement the reference-dependent logistic-normal map used by correlated topic models; -3. ALR is full rank but not an orthonormal Aitchison-distance isometry; ILR is required when that Euclidean geometry is the estimand; -4. max-shifted inverse ALR and log-difference forward ALR recover representable extreme coordinates without overflow; -5. TF-IDF, BM25, and keyword scores are refused as inferential coordinates. +3. ALR is full rank but not an orthonormal Aitchison-distance isometry; +4. sequential Egozcue ILR supplies the orthonormal Aitchison-distance isometry when that Euclidean geometry is the estimand; +5. max-shifted inverses recover representable extreme coordinates without overflow; +6. TF-IDF, BM25, and keyword scores are refused as inferential coordinates. The temporal STM backend, global topic identity, method-effect model, and K-selection remain accepted-target. No database migration is allocated. @@ -20,15 +21,18 @@ Aitchison, J. (1982). The statistical analysis of compositional data. *Journal o Blei, D. M., & Lafferty, J. D. (2007). A correlated topic model of Science. *The Annals of Applied Statistics, 1*(1), 17–35. https://doi.org/10.1214/07-AOAS114 +Egozcue, J. J., Pawlowsky-Glahn, V., Mateu-Figueras, G., & Barceló-Vidal, C. (2003). Isometric logratio transformations for compositional data analysis. *Mathematical Geology, 35*(3), 279–300. https://doi.org/10.1023/A:1023818214614 + ## Application -Aitchison and Shen (1980) define the logistic-normal family via the additive log-ratio map; Aitchison (1982) is the compositional-data authority that forbids treating parts of a whole as unconstrained Euclidean coordinates. Blei and Lafferty (2007) use that same reference-dependent map for correlated topic models. TEPP therefore uses `additive_log_ratio` for logistic-normal regression and psychometric interfaces, but does not claim that ALR preserves Aitchison distance. Analyses whose estimand is orthonormal Euclidean Aitchison geometry must use ILR. `from_additive_log_ratio` treats the omitted reference component as logit zero, max-shifts all `K` logits together, and normalizes only after exponentiation. The forward map subtracts logarithms rather than forming a potentially overflowing ratio. This avoids avoidable exponential and ratio overflow while failing closed when an `f64` simplex part would underflow to zero (Aitchison & Shen, 1980; Aitchison, 1982; Blei & Lafferty, 2007). +Aitchison and Shen (1980) define the logistic-normal family via the additive log-ratio map; Aitchison (1982) is the compositional-data authority that forbids treating parts of a whole as unconstrained Euclidean coordinates. Blei and Lafferty (2007) use that same reference-dependent map for correlated topic models. TEPP therefore uses `additive_log_ratio` for logistic-normal regression and psychometric interfaces, but does not claim that ALR preserves Aitchison distance. Egozcue et al. (2003) construct the sequential orthonormal ILR basis whose Euclidean norm equals Aitchison distance; `isometric_log_ratio` implements that basis and `from_isometric_log_ratio` inverts it through a max-shifted centered-log-ratio reconstruction. Analyses whose estimand is orthonormal Euclidean Aitchison geometry must use ILR rather than ALR. `from_additive_log_ratio` treats the omitted reference component as logit zero, max-shifts all `K` logits together, and normalizes only after exponentiation. The forward ALR map subtracts logarithms rather than forming a potentially overflowing ratio. Both inverses fail closed when an `f64` simplex part would underflow to zero (Aitchison & Shen, 1980; Aitchison, 1982; Blei & Lafferty, 2007; Egozcue et al., 2003). ## Verification -- closed-form simplex `(2,3,1)/6` maps to `(ln 2, ln 3)` and inverts with computed RMSE below `1e-15`; -- representable coordinates `(710, 709)` round-trip through the max-shifted inverse without exponential overflow; +- closed-form simplex `(2,3,1)/6` maps to ALR `(ln 2, ln 3)` and sequential ILR `(√(2/3) ln(2√3/3), √(1/2) ln 3)` with computed RMSE below `1e-15`; +- representable ALR coordinates `(710, 709)` and representable ILR coordinates round-trip through max-shifted inverses without exponential overflow; - extremes that would underflow a strictly positive `f64` simplex part fail closed; -- equal shares map to a zero ALR vector; +- equal shares map to the ALR and ILR origins; +- two-part ILR preserves Aitchison distance `√(1/2) ln(0.8/0.2)` for `(0.8, 0.2)`; - zero, negative, non-unit-sum, non-finite, empty, and one-part vectors fail closed; - `tfidf`, `bm25`, and `keyword` labels are refused. diff --git a/docs/validation/temporal-event-foundation.md b/docs/validation/temporal-event-foundation.md index 9749f8af..c3936755 100644 --- a/docs/validation/temporal-event-foundation.md +++ b/docs/validation/temporal-event-foundation.md @@ -25,7 +25,7 @@ This report tracks exact-head scientific and engineering evidence required befor | Versioned API/export contracts | `tepp_api` | implemented-main | naruon HTTP interchange | unknown-field/version/limit + naruon HTTPS interchange tests | Task 12 / PR #21; live HTTP service remaining | | Purpose-bound provider payloads | `tepp_api` | implemented-main | provider-payload minimization | expired/not-yet-valid/inverted/cross-tenant/impossible-calendar grant, mapping refusal, audited elevated re-id replay | ADR 0009; `docs/research/provider-payload-minimization.md` | | Adaptive orchestration router | `tepp_api` | implemented-main | router + ablation | mode selection, document-control denial, ablation, credential-free bind | ADR 0010; `docs/research/adaptive-orchestration-router.md` | -| Logistic-normal topic coordinates | `topic_measurement` | active-PR | stable ALR + lexical refusal | known-simplex and extreme-coordinate RMSE, ALR/ILR boundary | ADR 0012; `docs/research/topic-logratio-coordinates.md` | +| Logistic-normal topic coordinates | `topic_measurement` | active-PR | stable ALR + sequential ILR + lexical refusal | known-simplex ALR/ILR RMSE, Aitchison-distance ILR isometry | ADR 0012; `docs/research/topic-logratio-coordinates.md` | | CWL modular connectors | `docs/connectors/*` | implemented-main | — | contract docs + examples | PR #22; live HTTP ports remaining | | Release SBOM/provenance generator | `scripts/release_evidence.py` | partial | — | generate+validate in CI | Task 13 partial / PR #28 | From a52dd5597cc53943a9c3c5a339dd9e10fb216524 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 17 Aug 2026 17:21:12 +0000 Subject: [PATCH 15/26] test(topic): hit non-finite ILR centered-log reconstruction A single MAX coordinate underflows a simplex weight. Opposite-signed MAX coordinates overflow a CLR entry so the nightly branch gate covers !maximum.is_finite(). Co-authored-by: Seongho Bae --- crates/topic_measurement/src/coordinates.rs | 2 +- crates/topic_measurement/tests/ilr_recovery_contract.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/topic_measurement/src/coordinates.rs b/crates/topic_measurement/src/coordinates.rs index 1ab1306f..bcc74ce7 100644 --- a/crates/topic_measurement/src/coordinates.rs +++ b/crates/topic_measurement/src/coordinates.rs @@ -210,7 +210,7 @@ mod tests { Err(TopicMeasurementError::InvalidLogRatioDimension) ); assert_eq!( - from_isometric_log_ratio(&[f64::MAX]), + from_isometric_log_ratio(&[-f64::MAX, f64::MAX]), Err(TopicMeasurementError::InvalidLogRatioDimension) ); } diff --git a/crates/topic_measurement/tests/ilr_recovery_contract.rs b/crates/topic_measurement/tests/ilr_recovery_contract.rs index b8606be9..cc2815ca 100644 --- a/crates/topic_measurement/tests/ilr_recovery_contract.rs +++ b/crates/topic_measurement/tests/ilr_recovery_contract.rs @@ -84,7 +84,7 @@ fn large_finite_ilr_coordinates_round_trip_or_fail_closed() { "ILR inverse must not return a zero simplex part after underflow" ); assert_eq!( - from_isometric_log_ratio(&[f64::MAX]), + from_isometric_log_ratio(&[-f64::MAX, f64::MAX]), Err(TopicMeasurementError::InvalidLogRatioDimension), "overflowing CLR reconstruction must fail closed" ); From 30c1d1f7a58061c8ed0af58deaf72e2ac6a84337 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 18 Aug 2026 00:30:55 +0000 Subject: [PATCH 16/26] test(topic): recover three-part sequential ILR in unit tests Cover the remaining-parts>1 Egozcue step inside the crate unit instantiation so the same closed-form (2,3,1)/6 ILR is recovered without relying only on the integration crate. Co-authored-by: Seongho Bae --- crates/topic_measurement/src/coordinates.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/topic_measurement/src/coordinates.rs b/crates/topic_measurement/src/coordinates.rs index bcc74ce7..3c8d35ff 100644 --- a/crates/topic_measurement/src/coordinates.rs +++ b/crates/topic_measurement/src/coordinates.rs @@ -213,5 +213,9 @@ mod tests { from_isometric_log_ratio(&[-f64::MAX, f64::MAX]), Err(TopicMeasurementError::InvalidLogRatioDimension) ); + let three = isometric_log_ratio(&[2.0 / 6.0, 3.0 / 6.0, 1.0 / 6.0]).expect("ilr three"); + assert!((three[1] - (0.5_f64).sqrt() * 3.0_f64.ln()).abs() < 1e-15); + let recovered_three = from_isometric_log_ratio(&three).expect("ilr three inverse"); + assert!((recovered_three.iter().sum::() - 1.0).abs() < 1e-15); } } From 2f08cdd235fa0d48b34bce9e1c3f5b8fa5e2ba83 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 18 Aug 2026 01:48:53 +0000 Subject: [PATCH 17/26] docs(adr): align 0010 and 0013 maturity with merged main Match the ADR index to each source Implementation maturity after #47 and #44 landed: tepp_api orchestration is implemented-main, restore integrity revalidation is implemented-main, and remaining physical ERD/backup stays accepted-target. Co-authored-by: Seongho Bae --- docs/adr/0010-adaptive-llm-orchestration.md | 2 +- ...itemporal-persistence-reproducibility-and-split-authority.md | 2 +- docs/adr/README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/adr/0010-adaptive-llm-orchestration.md b/docs/adr/0010-adaptive-llm-orchestration.md index d04bd6bc..0f367a19 100644 --- a/docs/adr/0010-adaptive-llm-orchestration.md +++ b/docs/adr/0010-adaptive-llm-orchestration.md @@ -1,7 +1,7 @@ # ADR 0010 — Adaptive LLM orchestration and test-time compute **Decision status:** Accepted -**Implementation maturity:** partial — `tepp_api` governed router, comparable-budget ablation record, and credential-free contextual-orchestrator binding are implemented on the active PR and are not implemented-main until exact-head checks, review, and protected-main integration complete; live NIM execution, learned conductor calibration, and production ablation evidence remain accepted-target +**Implementation maturity:** partial — `tepp_api` governed router, comparable-budget ablation record, and credential-free contextual-orchestrator binding are implemented-main; live NIM execution, learned conductor calibration, and production ablation evidence remain accepted-target **Date:** 2026-08-10 **Supersedes:** The LLM orchestration-selection/ablation clauses previously co-located in ADR 0006. ADR 0006 remains authoritative for GPU/VRAM and model-credential separation; ADR 0015 governs autonomous repository-write/review/merge authority. diff --git a/docs/adr/0013-bitemporal-persistence-reproducibility-and-split-authority.md b/docs/adr/0013-bitemporal-persistence-reproducibility-and-split-authority.md index a593ddb1..92ae801f 100644 --- a/docs/adr/0013-bitemporal-persistence-reproducibility-and-split-authority.md +++ b/docs/adr/0013-bitemporal-persistence-reproducibility-and-split-authority.md @@ -1,7 +1,7 @@ # ADR 0013 — Bitemporal persistence, reproducibility manifests, and split authority **Decision status:** Accepted -**Implementation maturity:** partial — migration contracts, cutoff eligibility, in-memory bitemporal adapters, live SQL session/migration port, document SQL contracts, `DATABASE_URL` SQLx gate, optional `live-sqlx` `PgPool` open/execute driver, exact-head live PostgreSQL CI, tenant RLS (`tepp_app_runtime` + session GUC), append-only reproducibility-manifest SQL insert/lookup, model-run / model-artifact / corpus-split-manifest chain (migration `0003`), append-only immutability triggers (migration `0004`), temporal interval ordering CHECK constraints (migration `0005`), typed membership-assignment storage (migration `0006`), event-relation/mention/instance SQL, source-artifact SQL, audit-event action-code validation, and concurrent document-write stress implemented-main; backup/restore integrity revalidation on the active PR +**Implementation maturity:** partial — migration contracts, cutoff eligibility, in-memory bitemporal adapters, live SQL session/migration port, document SQL contracts, `DATABASE_URL` SQLx gate, optional `live-sqlx` `PgPool` open/execute driver, exact-head live PostgreSQL CI, tenant RLS (`tepp_app_runtime` + session GUC), append-only reproducibility-manifest SQL insert/lookup, model-run / model-artifact / corpus-split-manifest chain (migration `0003`), append-only immutability triggers (migration `0004`), temporal interval ordering CHECK constraints (migration `0005`), typed membership-assignment storage (migration `0006`), event-relation/mention/instance SQL, source-artifact SQL, audit-event action-code validation, and concurrent document-write stress implemented-main; backup/restore integrity revalidation implemented-main; remaining physical ERD/backup accepted-target **Date:** 2026-08-12 **Supersedes:** None; complements ADR 0002 (temporal semantics), ADR 0008 (evidence identity), and ADR 0011 (service ownership). diff --git a/docs/adr/README.md b/docs/adr/README.md index 9d549d0d..ba45bcd6 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -18,7 +18,7 @@ Read [`ADR_POLICY.md`](ADR_POLICY.md) first. **Decision status and implementatio | [0010](0010-adaptive-llm-orchestration.md) | Adaptive LLM orchestration and test-time compute | Accepted | partial | `tepp_api` router/ablation/orchestrator binding implemented-main; live NIM execution and production ablation evidence remain accepted-target. | | [0011](0011-standalone-modular-msa-boundary.md) | Standalone operation and modular CWL MSA boundary | Accepted | partial | Owns cross-service persistence/credential/API authority; no direct cross-service application-table coupling. | | [0012](0012-temporal-relational-shared-latent-topic-measurement.md) | Temporal Relational Shared-Latent Topic Measurement (TRSL-TM) | Accepted | partial | Logistic-normal ALR, sequential Egozcue ILR, and lexical-weight refusal are on the active PR; temporal topic identity, STM backend, method effects, and K-selection remain accepted-target. | -| [0013](0013-bitemporal-persistence-reproducibility-and-split-authority.md) | Bitemporal persistence, reproducibility manifests, and relation-aware split authority | Accepted | partial | Owns PostgreSQL adapter semantics, immutable run/split manifests, leakage-safe partitions, and recovery identity; optional `live-sqlx` `PgPool`, live PG CI, tenant RLS, and `0006` membership implemented-main; `0007` retention/deletion/legal-hold implemented-main; remaining physical ERD/backup accepted-target. | +| [0013](0013-bitemporal-persistence-reproducibility-and-split-authority.md) | Bitemporal persistence, reproducibility manifests, and relation-aware split authority | Accepted | partial | Owns PostgreSQL adapter semantics, immutable run/split manifests, leakage-safe partitions, and recovery identity; optional `live-sqlx` `PgPool`, live PG CI, tenant RLS, and `0006` membership implemented-main; backup/restore integrity revalidation implemented-main; `0007` retention/deletion/legal-hold implemented-main; remaining physical ERD/backup accepted-target. | | [0014](0014-scientific-claim-promotion-and-release-evidence.md) | Scientific claim promotion and release evidence authority | Accepted | partial | Separates design, implementation, scientific/product claim, and release authority; repository SBOM/provenance generator implemented, full release bundle remaining. | | [0015](0015-autonomous-development-review-and-merge-authority.md) | Autonomous development, review, and merge authority separation | Accepted | active-PR | Separates model proposal, deterministic verification, publication, independent review, and merge/release authority. | | [0016](0016-tdt-chronos-event-intelligence-boundary.md) | TDT, CHRONOS, and Event Ontology intelligence boundary | Accepted | accepted-target | Separates observed evidence, detection/tracking, prediction/schema inference, temporal consistency, and promoted transition authority. | From 6fe6a7853f06d8418701708ca3bf96f3f56c9b15 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 19:39:49 +0900 Subject: [PATCH 18/26] chore(docs): remove trailing whitespace from ADR maturity lines --- docs/adr/0010-adaptive-llm-orchestration.md | 2 +- .../0012-temporal-relational-shared-latent-topic-measurement.md | 2 +- ...itemporal-persistence-reproducibility-and-split-authority.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/adr/0010-adaptive-llm-orchestration.md b/docs/adr/0010-adaptive-llm-orchestration.md index 0f367a19..b8a0e17f 100644 --- a/docs/adr/0010-adaptive-llm-orchestration.md +++ b/docs/adr/0010-adaptive-llm-orchestration.md @@ -1,7 +1,7 @@ # ADR 0010 — Adaptive LLM orchestration and test-time compute **Decision status:** Accepted -**Implementation maturity:** partial — `tepp_api` governed router, comparable-budget ablation record, and credential-free contextual-orchestrator binding are implemented-main; live NIM execution, learned conductor calibration, and production ablation evidence remain accepted-target +**Implementation maturity:** partial — `tepp_api` governed router, comparable-budget ablation record, and credential-free contextual-orchestrator binding are implemented-main; live NIM execution, learned conductor calibration, and production ablation evidence remain accepted-target **Date:** 2026-08-10 **Supersedes:** The LLM orchestration-selection/ablation clauses previously co-located in ADR 0006. ADR 0006 remains authoritative for GPU/VRAM and model-credential separation; ADR 0015 governs autonomous repository-write/review/merge authority. diff --git a/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md b/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md index 23cd4162..40f338ba 100644 --- a/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md +++ b/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md @@ -1,7 +1,7 @@ # ADR 0012 — Temporal Relational Shared-Latent Topic Measurement **Decision status:** Accepted -**Implementation maturity:** partial — logistic-normal additive log-ratio coordinates, sequential Egozcue isometric log-ratio coordinates, and lexical-weight refusal are implemented on the active PR and are not implemented-main until exact-head checks, review, and protected-main integration complete; temporal topic identity, backend STM estimator, method-effect model, and K-selection remain accepted-target +**Implementation maturity:** partial — logistic-normal additive log-ratio coordinates, sequential Egozcue isometric log-ratio coordinates, and lexical-weight refusal are implemented on the active PR and are not implemented-main until exact-head checks, review, and protected-main integration complete; temporal topic identity, backend STM estimator, method-effect model, and K-selection remain accepted-target **Date:** 2026-08-12 **Supersedes:** None; refines ADR 0004 and ADR 0005 without replacing their multilingual and psychometric authorities. diff --git a/docs/adr/0013-bitemporal-persistence-reproducibility-and-split-authority.md b/docs/adr/0013-bitemporal-persistence-reproducibility-and-split-authority.md index 92ae801f..7f3706e5 100644 --- a/docs/adr/0013-bitemporal-persistence-reproducibility-and-split-authority.md +++ b/docs/adr/0013-bitemporal-persistence-reproducibility-and-split-authority.md @@ -1,7 +1,7 @@ # ADR 0013 — Bitemporal persistence, reproducibility manifests, and split authority **Decision status:** Accepted -**Implementation maturity:** partial — migration contracts, cutoff eligibility, in-memory bitemporal adapters, live SQL session/migration port, document SQL contracts, `DATABASE_URL` SQLx gate, optional `live-sqlx` `PgPool` open/execute driver, exact-head live PostgreSQL CI, tenant RLS (`tepp_app_runtime` + session GUC), append-only reproducibility-manifest SQL insert/lookup, model-run / model-artifact / corpus-split-manifest chain (migration `0003`), append-only immutability triggers (migration `0004`), temporal interval ordering CHECK constraints (migration `0005`), typed membership-assignment storage (migration `0006`), event-relation/mention/instance SQL, source-artifact SQL, audit-event action-code validation, and concurrent document-write stress implemented-main; backup/restore integrity revalidation implemented-main; remaining physical ERD/backup accepted-target +**Implementation maturity:** partial — migration contracts, cutoff eligibility, in-memory bitemporal adapters, live SQL session/migration port, document SQL contracts, `DATABASE_URL` SQLx gate, optional `live-sqlx` `PgPool` open/execute driver, exact-head live PostgreSQL CI, tenant RLS (`tepp_app_runtime` + session GUC), append-only reproducibility-manifest SQL insert/lookup, model-run / model-artifact / corpus-split-manifest chain (migration `0003`), append-only immutability triggers (migration `0004`), temporal interval ordering CHECK constraints (migration `0005`), typed membership-assignment storage (migration `0006`), event-relation/mention/instance SQL, source-artifact SQL, audit-event action-code validation, and concurrent document-write stress implemented-main; backup/restore integrity revalidation implemented-main; remaining physical ERD/backup accepted-target **Date:** 2026-08-12 **Supersedes:** None; complements ADR 0002 (temporal semantics), ADR 0008 (evidence identity), and ADR 0011 (service ownership). From d1f3dc3d52b09452b5799281e54f88f723be74c1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 19:53:26 +0900 Subject: [PATCH 19/26] fix(coverage): ignore Rust multiline string fragments --- CHANGELOG.md | 1 + scripts/check_coverage.py | 25 ++++++++-- tests/quality/test_check_coverage.py | 69 ++++++++++++++++++++-------- 3 files changed, 72 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ad9e324..da1fc8a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ All notable changes to TEPP are documented here. The format follows Keep a Chang ### Added - `topic_measurement` logistic-normal additive log-ratio and sequential Egozcue isometric log-ratio coordinates: fail-closed simplex validation, max-shifted stable ALR/ILR inverses with true-parameter RMSE, Aitchison-distance ILR isometry, and refusal of TF-IDF/BM25/keyword scores as inferential topic coordinates (ADR 0012 first production slice; no new migration). +- Coverage contract now excludes Rust multiline string continuation records emitted by LLVM LCOV, keeping the 100% authored-line gate focused on executable production lines. - `tepp_api` adaptive orchestration router (ADR 0010): versioned `direct`/`verify`/`committee`/`conductor`/`abstain` selection from CPU `f64` risk, ambiguity, evidence, and token-budget inputs; recorded stages, recursion, decomposition, access lists, and role-specific reasoning effort; fail-closed document-controlled policy/access/credentials; LLM plans remain proposals under deterministic statistical authority; comparable-budget ablation requires a direct baseline; credential-free contextual-orchestrator binding. Live NIM HTTP remains accepted-target. - `tepp_api` purpose-bound provider-payload minimization: time-bounded `PurposeGrant` evaluation, fail-closed expired/not-yet-valid/inverted/cross-tenant/impossible-calendar denial, semantic UTC calendar validation, refusal to copy identity mappings into model-provider payloads or ordinary logs, preservation of opaque analytical identifiers and membership roles (no blanket PII mask), a separately authorized scientific re-identification path, and an internally bound FIPS 180-4 SHA-256 audit digest appended through `ReidentificationAuditSink` before disclosure. - `persistence_postgres` backup/restore integrity: restored snapshots stay unusable until tenant, canonical `SHA-256`, knowledge-cutoff eligibility, temporal window order, and append-only triggers revalidate; SQL probes raise `restore integrity failed` (ADR 0013). diff --git a/scripts/check_coverage.py b/scripts/check_coverage.py index 50234635..fb7ecc61 100644 --- a/scripts/check_coverage.py +++ b/scripts/check_coverage.py @@ -51,9 +51,10 @@ def is_executable_source_line( """Return whether *line_number* in *source_path* is an executable source line. LLVM LCOV sometimes emits zero-count DA records for documentation comments, - attributes, pure structural braces, multi-line signatures, and in-file - ``#[cfg(test)]`` modules. Those records are not evidence of uncovered - production behavior and are excluded from the authored-line gate. + attributes, pure structural braces, multi-line signatures, Rust multiline + string continuations, and in-file ``#[cfg(test)]`` modules. Those records + are not evidence of uncovered production behavior and are excluded from the + authored-line gate. When *repository_root* is provided, *source_path* must resolve under that root (same fail-closed rule as LCOV ``SF:`` loading). @@ -75,6 +76,8 @@ def is_executable_source_line( return False if _line_in_cfg_not_feature_block(lines, line_number): return False + if _line_in_multiline_string_literal(lines, line_number): + return False text = lines[line_number - 1].strip() if not text: return False @@ -82,7 +85,9 @@ def is_executable_source_line( return False if text.startswith("#[") or text.startswith("#!["): return False - if text in {"{", "}", "},", ");", "];", "();", "};"}: + if text in {"{", "}", "},", ")", ");", "];", "();", "};"}: + return False + if text.startswith('"') or text.startswith("} else"): return False if text.startswith("use ") or text.startswith("pub use "): return False @@ -164,6 +169,18 @@ def _line_in_cfg_not_feature_block(lines: list[str], line_number: int) -> bool: return False +def _line_in_multiline_string_literal(lines: list[str], line_number: int) -> bool: + """Return whether a line is inside a Rust normal-string continuation.""" + in_string = False + for index, raw in enumerate(lines, start=1): + if in_string and index == line_number: + return True + quote_count = raw.replace('\\"', "").count('"') + if quote_count % 2: + in_string = not in_string + return False + + def load_lcov_line_totals( path: Path, repository_root: Path | None = None ) -> Mapping[str, Any]: diff --git a/tests/quality/test_check_coverage.py b/tests/quality/test_check_coverage.py index a4337397..2c122e2f 100644 --- a/tests/quality/test_check_coverage.py +++ b/tests/quality/test_check_coverage.py @@ -305,22 +305,25 @@ def test_executable_source_line_filters_noise_records(self) -> None: "})", # 39 " return value,", # 40 executable (return keeps it) " x + 1,", # 41 trailing comma noise - '#[cfg(feature = "live-sqlx")]', # 42 cfg attr - "fn live_path() {", # 43 fn - " live_body();", # 44 executable active feature body - "}", # 45 brace - '#[cfg(not(feature = "live-sqlx"))]', # 46 not-feature attr - "fn offline_path() {", # 47 inside not-feature - " offline_body();", # 48 inside not-feature - "}", # 49 inside not-feature close - "#[cfg(test)]", # 50 - "mod tests {", # 51 cfg(test) mod - " #[test]", # 52 inside test mod - " fn unit() {", # 53 inside test mod - " assert_eq!(1, 1);", # 54 inside test mod - " }", # 55 - "}", # 56 - " executable_statement();", # 57 executable + '"standalone string literal",', # 42 string noise + "} else {", # 43 structural branch noise + ")", # 44 structural close noise + '#[cfg(feature = "live-sqlx")]', # 45 cfg attr + "fn live_path() {", # 46 fn + " live_body();", # 47 executable active feature body + "}", # 48 brace + '#[cfg(not(feature = "live-sqlx"))]', # 49 not-feature attr + "fn offline_path() {", # 50 inside not-feature + " offline_body();", # 51 inside not-feature + "}", # 52 inside not-feature close + "#[cfg(test)]", # 53 + "mod tests {", # 54 cfg(test) mod + " #[test]", # 55 inside test mod + " fn unit() {", # 56 inside test mod + " assert_eq!(1, 1);", # 57 inside test mod + " }", # 58 + "}", # 59 + " executable_statement();", # 60 executable ] source.write_text("\n".join(source_lines) + "\n", encoding="utf-8") path = str(source) @@ -335,7 +338,7 @@ def test_executable_source_line_filters_noise_records(self) -> None: coverage_contract.is_executable_source_line(path, len(source_lines) + 5) ) - expected_executable = {13, 40, 44, 57} + expected_executable = {13, 40, 47, 60} for line_number in range(1, len(source_lines) + 1): is_exec = coverage_contract.is_executable_source_line(path, line_number) if line_number in expected_executable: @@ -354,10 +357,10 @@ def test_executable_source_line_filters_noise_records(self) -> None: "\n".join( [ f"SF:{path}", - "DA:57,1", + "DA:60,1", "DA:1,0", "DA:2,0", - "DA:48,0", + "DA:51,0", "end_of_record", "", ] @@ -427,6 +430,7 @@ def test_cfg_test_and_not_feature_block_helpers(self) -> None: self.assertTrue(coverage_contract._line_in_cfg_not_feature_block(lines, 9)) self.assertFalse(coverage_contract._line_in_cfg_not_feature_block(lines, 1)) self.assertFalse(coverage_contract._line_in_cfg_not_feature_block(lines, 11)) + open_only = [ '#[cfg(not(feature = "x"))]', "fn unfinished()", @@ -474,6 +478,33 @@ def test_cfg_test_and_not_feature_block_helpers(self) -> None: coverage_contract._line_in_cfg_not_feature_block(unclosed_not_feature, 99) ) + def test_multiline_string_continuations_are_not_authored_lines(self) -> None: + """Rust multiline string fragments are excluded from authored coverage.""" + + with tempfile.TemporaryDirectory() as temporary: + source = Path(temporary) / "query.rs" + source.write_text( + 'fn query() {\n' + ' let sql = format!("SELECT id \\\n' + ' FROM document_record \\\n' + ' WHERE tenant_record_id = \'x\'");\n' + ' execute(sql);\n' + '}\n', + encoding="utf-8", + ) + self.assertTrue( + coverage_contract.is_executable_source_line(str(source), 2) + ) + self.assertFalse( + coverage_contract.is_executable_source_line(str(source), 3) + ) + self.assertFalse( + coverage_contract.is_executable_source_line(str(source), 4) + ) + self.assertTrue( + coverage_contract.is_executable_source_line(str(source), 5) + ) + if __name__ == "__main__": # pragma: no cover unittest.main() From 4b54dc82b1685bfc40f2c563f836e5228976870c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 13:15:56 +0900 Subject: [PATCH 20/26] test(topic): assert direct Aitchison distance recovery --- .../topic_measurement/tests/ilr_recovery_contract.rs | 11 +++++++++-- docs/research/topic-logratio-coordinates.md | 12 ++++++------ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/crates/topic_measurement/tests/ilr_recovery_contract.rs b/crates/topic_measurement/tests/ilr_recovery_contract.rs index cc2815ca..12817f6e 100644 --- a/crates/topic_measurement/tests/ilr_recovery_contract.rs +++ b/crates/topic_measurement/tests/ilr_recovery_contract.rs @@ -60,8 +60,15 @@ fn equal_shares_are_the_ilr_origin_and_preserve_aitchison_distance() { let unbalanced = [0.8, 0.2]; let coordinates = isometric_log_ratio(&unbalanced).expect("pair"); - let expected = (0.5_f64).sqrt() * 4.0_f64.ln(); - assert!((coordinates[0] - expected).abs() < 1e-15); + let direct_aitchison_distance = (0.5_f64).sqrt() + * ((unbalanced[0] / unbalanced[1]).ln() - (halves[0] / halves[1]).ln()).abs(); + let ilr_euclidean_distance = coordinates + .iter() + .zip(&origin) + .map(|(left, right)| (left - right).powi(2)) + .sum::() + .sqrt(); + assert!((ilr_euclidean_distance - direct_aitchison_distance).abs() < 1e-15); let recovered = from_isometric_log_ratio(&coordinates).expect("inverse"); assert!(rmse(&unbalanced, &recovered) < 1e-15); diff --git a/docs/research/topic-logratio-coordinates.md b/docs/research/topic-logratio-coordinates.md index ac5c242d..8ea997a5 100644 --- a/docs/research/topic-logratio-coordinates.md +++ b/docs/research/topic-logratio-coordinates.md @@ -4,11 +4,11 @@ This note doctors the first `topic_measurement` production slice (ADR 0012): -1. raw topic proportions are compositional rather than unconstrained Euclidean indicators; -2. additive log-ratio coordinates implement the reference-dependent logistic-normal map used by correlated topic models; -3. ALR is full rank but not an orthonormal Aitchison-distance isometry; -4. sequential Egozcue ILR supplies the orthonormal Aitchison-distance isometry when that Euclidean geometry is the estimand; -5. max-shifted inverses recover representable extreme coordinates without overflow; +1. raw topic proportions are compositional rather than unconstrained Euclidean indicators (Aitchison, 1982); +2. additive log-ratio coordinates implement the reference-dependent logistic-normal map used by correlated topic models (Aitchison & Shen, 1980; Blei & Lafferty, 2007); +3. ALR is full rank but not an orthonormal Aitchison-distance isometry (Aitchison, 1982); +4. sequential Egozcue ILR supplies the orthonormal Aitchison-distance isometry when that Euclidean geometry is the estimand (Egozcue et al., 2003); +5. max-shifted inverses recover representable extreme coordinates without overflow (Aitchison & Shen, 1980); 6. TF-IDF, BM25, and keyword scores are refused as inferential coordinates. The temporal STM backend, global topic identity, method-effect model, and K-selection remain accepted-target. No database migration is allocated. @@ -33,6 +33,6 @@ Aitchison and Shen (1980) define the logistic-normal family via the additive log - representable ALR coordinates `(710, 709)` and representable ILR coordinates round-trip through max-shifted inverses without exponential overflow; - extremes that would underflow a strictly positive `f64` simplex part fail closed; - equal shares map to the ALR and ILR origins; -- two-part ILR preserves Aitchison distance `√(1/2) ln(0.8/0.2)` for `(0.8, 0.2)`; +- two-part ILR preserves Aitchison distance `√(1/2) |ln(0.8/0.2) - ln(0.5/0.5)|` between `(0.8, 0.2)` and `(0.5, 0.5)` (Egozcue et al., 2003); - zero, negative, non-unit-sum, non-finite, empty, and one-part vectors fail closed; - `tfidf`, `bm25`, and `keyword` labels are refused. From 543982cec92585dbbb601b33f4886432b8e1eacd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 13:24:52 +0900 Subject: [PATCH 21/26] fix(coverage): parse Rust strings and comments statefully --- scripts/check_coverage.py | 50 +++++++++++++++++++++++++--- tests/quality/test_check_coverage.py | 42 +++++++++++++++++++++++ 2 files changed, 88 insertions(+), 4 deletions(-) diff --git a/scripts/check_coverage.py b/scripts/check_coverage.py index fb7ecc61..87e27755 100644 --- a/scripts/check_coverage.py +++ b/scripts/check_coverage.py @@ -87,7 +87,7 @@ def is_executable_source_line( return False if text in {"{", "}", "},", ")", ");", "];", "();", "};"}: return False - if text.startswith('"') or text.startswith("} else"): + if _is_standalone_string_literal(text) or text.startswith("} else"): return False if text.startswith("use ") or text.startswith("pub use "): return False @@ -172,12 +172,54 @@ def _line_in_cfg_not_feature_block(lines: list[str], line_number: int) -> bool: def _line_in_multiline_string_literal(lines: list[str], line_number: int) -> bool: """Return whether a line is inside a Rust normal-string continuation.""" in_string = False + in_block_comment = False for index, raw in enumerate(lines, start=1): if in_string and index == line_number: return True - quote_count = raw.replace('\\"', "").count('"') - if quote_count % 2: - in_string = not in_string + escaped = False + cursor = 0 + while cursor < len(raw): + if in_block_comment: + if raw.startswith("*/", cursor): + in_block_comment = False + cursor += 2 + else: + cursor += 1 + continue + if in_string: + character = raw[cursor] + if character == '"' and not escaped: + in_string = False + if character == "\\": + escaped = not escaped + else: + escaped = False + cursor += 1 + continue + if raw.startswith("//", cursor): + break + if raw.startswith("/*", cursor): + in_block_comment = True + cursor += 2 + continue + if raw[cursor] == '"': + in_string = True + cursor += 1 + return False + + +def _is_standalone_string_literal(text: str) -> bool: + """Return whether *text* is only a normal string literal and punctuation.""" + if not text.startswith('"'): + return False + escaped = False + for index, character in enumerate(text[1:], start=1): + if character == '"' and not escaped: + return text[index + 1 :].strip() in {"", ",", ";"} + if character == "\\": + escaped = not escaped + else: + escaped = False return False diff --git a/tests/quality/test_check_coverage.py b/tests/quality/test_check_coverage.py index 2c122e2f..831c44ee 100644 --- a/tests/quality/test_check_coverage.py +++ b/tests/quality/test_check_coverage.py @@ -505,6 +505,48 @@ def test_multiline_string_continuations_are_not_authored_lines(self) -> None: coverage_contract.is_executable_source_line(str(source), 5) ) + def test_string_scanner_handles_comments_backslash_parity_and_methods(self) -> None: + """Quoted comments and escaped delimiters do not corrupt source classification.""" + + with tempfile.TemporaryDirectory() as temporary: + backslash = "\\" + source = Path(temporary) / "scanner.rs" + source_lines = [ + "fn query() {", + f' let sql = "SELECT id {backslash}', + ' FROM document";', + r' // comment contains one " quote', + " execute(sql);", + f' let even = "ends with two slashes {backslash * 2}";', + " execute(even);", + r' "literal".to_string();', + "}", + ] + source.write_text("\n".join(source_lines) + "\n", encoding="utf-8") + path = str(source) + + self.assertFalse(coverage_contract.is_executable_source_line(path, 3)) + self.assertTrue(coverage_contract.is_executable_source_line(path, 5)) + self.assertTrue(coverage_contract.is_executable_source_line(path, 7)) + self.assertTrue(coverage_contract.is_executable_source_line(path, 8)) + + block_comment = [ + "/* comment starts", + r' comment has a " quote', + " still comment", + "*/", + "execute();", + ] + self.assertFalse( + coverage_contract._line_in_multiline_string_literal(block_comment, 5) + ) + self.assertTrue( + coverage_contract._is_standalone_string_literal(r'"escaped\\",') + ) + self.assertFalse( + coverage_contract._is_standalone_string_literal('"unfinished') + ) + if __name__ == "__main__": # pragma: no cover unittest.main() From 621cae0787b6fcfcd58c192a120b2579d1caf1cf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 13:25:15 +0900 Subject: [PATCH 22/26] docs: record stateful coverage parser repair --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index da1fc8a4..312822f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ All notable changes to TEPP are documented here. The format follows Keep a Chang - `topic_measurement` logistic-normal additive log-ratio and sequential Egozcue isometric log-ratio coordinates: fail-closed simplex validation, max-shifted stable ALR/ILR inverses with true-parameter RMSE, Aitchison-distance ILR isometry, and refusal of TF-IDF/BM25/keyword scores as inferential topic coordinates (ADR 0012 first production slice; no new migration). - Coverage contract now excludes Rust multiline string continuation records emitted by LLVM LCOV, keeping the 100% authored-line gate focused on executable production lines. +- Coverage source classification now scans Rust strings and comments with escape-parity state, preserving executable string method calls and ignoring quoted comments. - `tepp_api` adaptive orchestration router (ADR 0010): versioned `direct`/`verify`/`committee`/`conductor`/`abstain` selection from CPU `f64` risk, ambiguity, evidence, and token-budget inputs; recorded stages, recursion, decomposition, access lists, and role-specific reasoning effort; fail-closed document-controlled policy/access/credentials; LLM plans remain proposals under deterministic statistical authority; comparable-budget ablation requires a direct baseline; credential-free contextual-orchestrator binding. Live NIM HTTP remains accepted-target. - `tepp_api` purpose-bound provider-payload minimization: time-bounded `PurposeGrant` evaluation, fail-closed expired/not-yet-valid/inverted/cross-tenant/impossible-calendar denial, semantic UTC calendar validation, refusal to copy identity mappings into model-provider payloads or ordinary logs, preservation of opaque analytical identifiers and membership roles (no blanket PII mask), a separately authorized scientific re-identification path, and an internally bound FIPS 180-4 SHA-256 audit digest appended through `ReidentificationAuditSink` before disclosure. - `persistence_postgres` backup/restore integrity: restored snapshots stay unusable until tenant, canonical `SHA-256`, knowledge-cutoff eligibility, temporal window order, and append-only triggers revalidate; SQL probes raise `restore integrity failed` (ADR 0013). From 8e88b337e0bb1a44749b0fe48c635e3e3e7535da Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 13:38:42 +0900 Subject: [PATCH 23/26] docs(topic): define ILR pairwise distance --- crates/topic_measurement/src/coordinates.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/topic_measurement/src/coordinates.rs b/crates/topic_measurement/src/coordinates.rs index 3c8d35ff..0ab392d9 100644 --- a/crates/topic_measurement/src/coordinates.rs +++ b/crates/topic_measurement/src/coordinates.rs @@ -71,10 +71,11 @@ pub fn from_additive_log_ratio(coordinates: &[f64]) -> Result, TopicMea /// Map a strictly positive unit simplex vector to isometric log-ratio coordinates. /// /// The sequential Egozcue orthonormal basis sends a `K`-part composition to -/// the `K-1` vector whose Euclidean norm equals Aitchison distance. This is -/// the coordinate system for distance-based topic geometry. It is not the -/// reference-dependent logistic-normal map; use [`additive_log_ratio`] when -/// that regression interface is the estimand. +/// the `K-1` vector whose Euclidean distance from another composition's ILR +/// vector equals their Aitchison distance. A vector norm is only the distance +/// from the equal-share origin. This is the coordinate system for distance-based +/// topic geometry. It is not the reference-dependent logistic-normal map; use +/// [`additive_log_ratio`] when that regression interface is the estimand. /// /// # Errors /// From f352e15d5fb240d700decd1bf8959c891811caeb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 01:27:41 +0900 Subject: [PATCH 24/26] docs: trace logratio claims to sources --- docs/research/topic-logratio-coordinates.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/research/topic-logratio-coordinates.md b/docs/research/topic-logratio-coordinates.md index 8ea997a5..4faf429b 100644 --- a/docs/research/topic-logratio-coordinates.md +++ b/docs/research/topic-logratio-coordinates.md @@ -25,7 +25,7 @@ Egozcue, J. J., Pawlowsky-Glahn, V., Mateu-Figueras, G., & Barceló-Vidal, C. (2 ## Application -Aitchison and Shen (1980) define the logistic-normal family via the additive log-ratio map; Aitchison (1982) is the compositional-data authority that forbids treating parts of a whole as unconstrained Euclidean coordinates. Blei and Lafferty (2007) use that same reference-dependent map for correlated topic models. TEPP therefore uses `additive_log_ratio` for logistic-normal regression and psychometric interfaces, but does not claim that ALR preserves Aitchison distance. Egozcue et al. (2003) construct the sequential orthonormal ILR basis whose Euclidean norm equals Aitchison distance; `isometric_log_ratio` implements that basis and `from_isometric_log_ratio` inverts it through a max-shifted centered-log-ratio reconstruction. Analyses whose estimand is orthonormal Euclidean Aitchison geometry must use ILR rather than ALR. `from_additive_log_ratio` treats the omitted reference component as logit zero, max-shifts all `K` logits together, and normalizes only after exponentiation. The forward ALR map subtracts logarithms rather than forming a potentially overflowing ratio. Both inverses fail closed when an `f64` simplex part would underflow to zero (Aitchison & Shen, 1980; Aitchison, 1982; Blei & Lafferty, 2007; Egozcue et al., 2003). +Aitchison and Shen (1980) define the logistic-normal family via the additive log-ratio map; Aitchison (1982) is the compositional-data authority that forbids treating parts of a whole as unconstrained Euclidean coordinates. Blei and Lafferty (2007) use that same reference-dependent map for correlated topic models. TEPP therefore uses `additive_log_ratio` for logistic-normal regression and psychometric interfaces, but does not claim that ALR preserves Aitchison distance. Egozcue et al. (2003) construct the sequential orthonormal ILR basis whose Euclidean norm equals Aitchison distance; `isometric_log_ratio` implements that basis and `from_isometric_log_ratio` inverts it through a max-shifted centered-log-ratio reconstruction. Analyses whose estimand is orthonormal Euclidean Aitchison geometry must use ILR rather than ALR (Egozcue et al., 2003). `from_additive_log_ratio` treats the omitted reference component as logit zero, max-shifts all `K` logits together, and normalizes only after exponentiation (Aitchison & Shen, 1980). The forward ALR map subtracts logarithms rather than forming a potentially overflowing ratio (Aitchison & Shen, 1980; Blei & Lafferty, 2007). Both inverses fail closed when an `f64` simplex part would underflow to zero (Aitchison & Shen, 1980). ## Verification From ff417148769fa2980ed6f43d767db837d731fda5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 01:42:54 +0900 Subject: [PATCH 25/26] fix(quality): parse Rust literal state in coverage gate --- scripts/check_coverage.py | 74 +++++++++++++++++++++++++--- tests/quality/test_check_coverage.py | 44 +++++++++++++++++ 2 files changed, 110 insertions(+), 8 deletions(-) diff --git a/scripts/check_coverage.py b/scripts/check_coverage.py index 87e27755..6f30a8ab 100644 --- a/scripts/check_coverage.py +++ b/scripts/check_coverage.py @@ -170,13 +170,19 @@ def _line_in_cfg_not_feature_block(lines: list[str], line_number: int) -> bool: def _line_in_multiline_string_literal(lines: list[str], line_number: int) -> bool: - """Return whether a line is inside a Rust normal-string continuation.""" + """Return whether a line is inside a Rust string continuation. + + The scanner tracks normal strings, raw strings, block comments, and character + literals so quotes in comments or literal contents cannot change the state of + a later source line. + """ + in_string = False + raw_hashes: int | None = None in_block_comment = False for index, raw in enumerate(lines, start=1): - if in_string and index == line_number: + if (in_string or raw_hashes is not None) and index == line_number: return True - escaped = False cursor = 0 while cursor < len(raw): if in_block_comment: @@ -186,15 +192,24 @@ def _line_in_multiline_string_literal(lines: list[str], line_number: int) -> boo else: cursor += 1 continue + if raw_hashes is not None: + delimiter = '"' + ("#" * raw_hashes) + closing = raw.find(delimiter, cursor) + if closing == -1: + cursor = len(raw) + else: + raw_hashes = None + cursor = closing + len(delimiter) + continue if in_string: character = raw[cursor] - if character == '"' and not escaped: - in_string = False if character == "\\": - escaped = not escaped + cursor += 2 + elif character == '"': + in_string = False + cursor += 1 else: - escaped = False - cursor += 1 + cursor += 1 continue if raw.startswith("//", cursor): break @@ -202,12 +217,55 @@ def _line_in_multiline_string_literal(lines: list[str], line_number: int) -> boo in_block_comment = True cursor += 2 continue + raw_start = _raw_string_start(raw, cursor) + if raw_start is not None: + raw_hashes, cursor = raw_start + continue if raw[cursor] == '"': in_string = True + cursor += 1 + continue + if raw[cursor] == "'": + character_end = _character_literal_end(raw, cursor) + if character_end is not None: + cursor = character_end + continue cursor += 1 return False +def _raw_string_start(line: str, cursor: int) -> tuple[int, int] | None: + """Return ``(hash_count, next_cursor)`` for a Rust raw-string opener.""" + + if line.startswith("br", cursor): + prefix_end = cursor + 2 + elif line.startswith("r", cursor): + prefix_end = cursor + 1 + else: + return None + hash_end = prefix_end + while hash_end < len(line) and line[hash_end] == "#": + hash_end += 1 + if hash_end < len(line) and line[hash_end] == '"': + return hash_end - prefix_end, hash_end + 1 + return None + + +def _character_literal_end(line: str, cursor: int) -> int | None: + """Return the cursor after a one-line Rust character literal, if present.""" + + candidate = cursor + 1 + if candidate >= len(line): + return None + if line[candidate] == "\\": + candidate += 2 + else: + candidate += 1 + if candidate < len(line) and line[candidate] == "'": + return candidate + 1 + return None + + def _is_standalone_string_literal(text: str) -> bool: """Return whether *text* is only a normal string literal and punctuation.""" if not text.startswith('"'): diff --git a/tests/quality/test_check_coverage.py b/tests/quality/test_check_coverage.py index 831c44ee..4a62b2c9 100644 --- a/tests/quality/test_check_coverage.py +++ b/tests/quality/test_check_coverage.py @@ -547,6 +547,50 @@ def test_string_scanner_handles_comments_backslash_parity_and_methods(self) -> N coverage_contract._is_standalone_string_literal('"unfinished') ) + raw_string = [ + ' let text = r##"', + ' a " quote in raw text', + ' "##;', + ' execute(text);', + ] + self.assertFalse( + coverage_contract._line_in_multiline_string_literal(raw_string, 1) + ) + self.assertTrue( + coverage_contract._line_in_multiline_string_literal(raw_string, 2) + ) + self.assertTrue( + coverage_contract._line_in_multiline_string_literal(raw_string, 3) + ) + self.assertFalse( + coverage_contract._line_in_multiline_string_literal(raw_string, 4) + ) + + byte_raw_string = [ + ' let bytes = br#"', + ' raw bytes', + ' "#;', + ] + self.assertTrue( + coverage_contract._line_in_multiline_string_literal(byte_raw_string, 2) + ) + + character_and_lifetime = [ + "fn query<'a>() {", + " let quote: char = '\"';", + " execute();", + "}", + ] + self.assertFalse( + coverage_contract._line_in_multiline_string_literal( + character_and_lifetime, 3 + ) + ) + self.assertIsNone(coverage_contract._character_literal_end("'", 0)) + self.assertEqual( + coverage_contract._character_literal_end(r"'\''", 0), 4 + ) + if __name__ == "__main__": # pragma: no cover unittest.main() From 45224e55792165fd2d752d9a3bd62b016ec7c970 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 01:45:37 +0900 Subject: [PATCH 26/26] docs: record Rust literal coverage hardening --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 312822f9..ff44779f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ All notable changes to TEPP are documented here. The format follows Keep a Chang - `topic_measurement` logistic-normal additive log-ratio and sequential Egozcue isometric log-ratio coordinates: fail-closed simplex validation, max-shifted stable ALR/ILR inverses with true-parameter RMSE, Aitchison-distance ILR isometry, and refusal of TF-IDF/BM25/keyword scores as inferential topic coordinates (ADR 0012 first production slice; no new migration). - Coverage contract now excludes Rust multiline string continuation records emitted by LLVM LCOV, keeping the 100% authored-line gate focused on executable production lines. -- Coverage source classification now scans Rust strings and comments with escape-parity state, preserving executable string method calls and ignoring quoted comments. +- Coverage source classification now scans Rust normal/raw/byte strings, comments, and character literals with escape-aware state, preserving executable string method calls and ignoring quoted comments. - `tepp_api` adaptive orchestration router (ADR 0010): versioned `direct`/`verify`/`committee`/`conductor`/`abstain` selection from CPU `f64` risk, ambiguity, evidence, and token-budget inputs; recorded stages, recursion, decomposition, access lists, and role-specific reasoning effort; fail-closed document-controlled policy/access/credentials; LLM plans remain proposals under deterministic statistical authority; comparable-budget ablation requires a direct baseline; credential-free contextual-orchestrator binding. Live NIM HTTP remains accepted-target. - `tepp_api` purpose-bound provider-payload minimization: time-bounded `PurposeGrant` evaluation, fail-closed expired/not-yet-valid/inverted/cross-tenant/impossible-calendar denial, semantic UTC calendar validation, refusal to copy identity mappings into model-provider payloads or ordinary logs, preservation of opaque analytical identifiers and membership roles (no blanket PII mask), a separately authorized scientific re-identification path, and an internally bound FIPS 180-4 SHA-256 audit digest appended through `ReidentificationAuditSink` before disclosure. - `persistence_postgres` backup/restore integrity: restored snapshots stay unusable until tenant, canonical `SHA-256`, knowledge-cutoff eligibility, temporal window order, and append-only triggers revalidate; SQL probes raise `restore integrity failed` (ADR 0013).