diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index ffe514db..f10bf73a 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -61,6 +61,7 @@ boundaries above remain the target modular MSA architecture. | `tepp_simulation` | known-truth temporal/event data generation | | `validation_core` | RMSE, bias, coverage, graph, and Monte Carlo metrics | | `tepp_api` | versioned DTO, schema, and export contracts | +| `topic_measurement` | logistic-normal ALR and sequential Egozcue ILR topic coordinates | No crate exposes placeholder production behavior in Task 1. This prevents an empty façade from becoming a de facto public API before its invariants and tests diff --git a/CHANGELOG.md b/CHANGELOG.md index 36c2e8dd..ff79e6fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ All notable changes to TEPP are documented here. The format follows Keep a Chang ### Added +- `topic_measurement` logistic-normal additive log-ratio and sequential Egozcue isometric log-ratio coordinates: fail-closed simplex validation, max-shifted stable ALR/ILR inverses with true-parameter RMSE, Aitchison-distance ILR isometry, and refusal of TF-IDF/BM25/keyword scores as inferential topic coordinates (ADR 0012 first production slice; no new migration). +- Coverage contract now excludes Rust multiline string continuation records emitted by LLVM LCOV, keeping the 100% authored-line gate focused on executable production lines. +- Coverage source classification now scans Rust normal/raw/byte strings, comments, and character literals with escape-aware state, preserving executable string method calls and ignoring quoted comments. - `tepp_api` naruon live loopback HTTP/1.1 listener: `serve_one` installs a read/write deadline, requires a loopback `Host`, refuses `Transfer-Encoding` and NIM/proxy credential headers, parses `knowledge_cutoff` as RFC 3339 and refuses a future cutoff, keys analysis-run idempotency by tenant plus key, and proves both analysis-run and export POSTs over a real `TcpStream`. Not a production TLS/`$PORT` service (ADR 0011). - `tepp_api` adaptive orchestration router (ADR 0010): versioned `direct`/`verify`/`committee`/`conductor`/`abstain` selection from CPU `f64` risk, ambiguity, evidence, and token-budget inputs; recorded stages, recursion, decomposition, access lists, and role-specific reasoning effort; fail-closed document-controlled policy/access/credentials; LLM plans remain proposals under deterministic statistical authority; comparable-budget ablation requires a direct baseline; credential-free contextual-orchestrator binding. Live NIM HTTP remains accepted-target. - `tepp_api` purpose-bound provider-payload minimization: time-bounded `PurposeGrant` evaluation, fail-closed expired/not-yet-valid/inverted/cross-tenant/impossible-calendar denial, semantic UTC calendar validation, refusal to copy identity mappings into model-provider payloads or ordinary logs, preservation of opaque analytical identifiers and membership roles (no blanket PII mask), a separately authorized scientific re-identification path, and an internally bound FIPS 180-4 SHA-256 audit digest appended through `ReidentificationAuditSink` before disclosure. diff --git a/Cargo.lock b/Cargo.lock index fb502b9c..e3018765 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1373,6 +1373,10 @@ dependencies = [ "tokio", ] +[[package]] +name = "topic_measurement" +version = "0.1.0" + [[package]] name = "tracing" version = "0.1.44" diff --git a/Cargo.toml b/Cargo.toml index 92565940..35ba7a1e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ members = [ "crates/tepp_simulation", "crates/validation_core", "crates/tepp_api", + "crates/topic_measurement", ] default-members = [ "crates/evidence_core", @@ -23,6 +24,7 @@ default-members = [ "crates/tepp_simulation", "crates/validation_core", "crates/tepp_api", + "crates/topic_measurement", ] [workspace.package] diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 3f094947..9e93a508 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -36,6 +36,7 @@ TEPP's approved PRD v0.4 and implementation plan are the primary product baselin | Retention/deletion/legal-hold doctoring | [`docs/research/retention-deletion-legal-hold.md`](docs/research/retention-deletion-legal-hold.md) | | Provider-payload minimization doctoring | [`docs/research/provider-payload-minimization.md`](docs/research/provider-payload-minimization.md) | | Adaptive orchestration router doctoring | [`docs/research/adaptive-orchestration-router.md`](docs/research/adaptive-orchestration-router.md) | +| Topic log-ratio coordinate doctoring | [`docs/research/topic-logratio-coordinates.md`](docs/research/topic-logratio-coordinates.md) | | Hourly NIM OpenCode doctoring | [`docs/doctoring/hourly-nim-opencode-development.md`](docs/doctoring/hourly-nim-opencode-development.md) | | Change history | [`CHANGELOG.md`](CHANGELOG.md) | diff --git a/README.md b/README.md index ae74015d..ed7a2d74 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,8 @@ implemented in Rust. ## Current implementation state This branch establishes the Task 1 Rust workspace and quality-gate foundation. -The ten bounded crates compile independently but intentionally expose no -placeholder production APIs. Domain behavior begins in Task 2 with immutable -evidence identifiers and source records. +The eleven bounded crates compile independently. Domain crates expose only +validated production APIs; placeholder surfaces are prohibited. ```text crates/evidence_core @@ -22,6 +21,7 @@ crates/corpus_split crates/tepp_simulation crates/validation_core crates/tepp_api +crates/topic_measurement ``` ## Local verification diff --git a/crates/topic_measurement/Cargo.toml b/crates/topic_measurement/Cargo.toml new file mode 100644 index 00000000..299f03c2 --- /dev/null +++ b/crates/topic_measurement/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "topic_measurement" +description = "Logistic-normal and log-ratio coordinates for compositional topics." +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true +homepage.workspace = true +readme.workspace = true +keywords.workspace = true +categories.workspace = true +publish = false + +[lints] +workspace = true diff --git a/crates/topic_measurement/src/coordinates.rs b/crates/topic_measurement/src/coordinates.rs new file mode 100644 index 00000000..0ab392d9 --- /dev/null +++ b/crates/topic_measurement/src/coordinates.rs @@ -0,0 +1,222 @@ +//! Additive and isometric log-ratio maps for compositional topic coordinates. + +use crate::error::TopicMeasurementError; + +const UNIT_SUM_TOLERANCE: f64 = 1e-12; + +/// Map a strictly positive unit simplex vector to additive log-ratio coordinates. +/// +/// For a `K`-part composition `θ` the image is the `K-1` vector +/// `y_k = ln(θ_k / θ_K)`. This reference-dependent, full-rank coordinate map +/// supports logistic-normal regression and ESEM/DSEM interfaces. It is not an +/// orthonormal isometry for Aitchison distance; use ILR coordinates when that +/// Euclidean geometry is the estimand. +/// +/// # Errors +/// +/// Returns [`TopicMeasurementError::InvalidComposition`] when the vector is +/// empty, has fewer than two parts, contains a non-finite or non-positive +/// entry, or does not sum to one within a tight absolute tolerance. +pub fn additive_log_ratio(proportions: &[f64]) -> Result, TopicMeasurementError> { + let last = require_composition(proportions)?; + let reference_log = last.ln(); + Ok(proportions[..proportions.len() - 1] + .iter() + .map(|part| part.ln() - reference_log) + .collect()) +} + +/// Invert additive log-ratio coordinates back to the unit simplex. +/// +/// # Errors +/// +/// Returns [`TopicMeasurementError::InvalidLogRatioDimension`] when the +/// coordinate vector is empty, non-finite, or would underflow a part to zero +/// in the strictly positive `f64` simplex representation. +pub fn from_additive_log_ratio(coordinates: &[f64]) -> Result, TopicMeasurementError> { + if coordinates.is_empty() { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + let mut maximum = 0.0_f64; + for &value in coordinates { + if !value.is_finite() { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + maximum = maximum.max(value); + } + + let reference_weight = (-maximum).exp(); + if reference_weight == 0.0 { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + let mut shifted_weights = Vec::with_capacity(coordinates.len()); + let mut denominator = reference_weight; + for &value in coordinates { + let weight = (value - maximum).exp(); + if weight == 0.0 { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + denominator += weight; + shifted_weights.push(weight); + } + + let mut simplex = Vec::with_capacity(coordinates.len() + 1); + for weight in shifted_weights { + simplex.push(weight / denominator); + } + simplex.push(reference_weight / denominator); + Ok(simplex) +} + +/// Map a strictly positive unit simplex vector to isometric log-ratio coordinates. +/// +/// The sequential Egozcue orthonormal basis sends a `K`-part composition to +/// the `K-1` vector whose Euclidean distance from another composition's ILR +/// vector equals their Aitchison distance. A vector norm is only the distance +/// from the equal-share origin. This is the coordinate system for distance-based +/// topic geometry. It is not the reference-dependent logistic-normal map; use +/// [`additive_log_ratio`] when that regression interface is the estimand. +/// +/// # Errors +/// +/// Returns [`TopicMeasurementError::InvalidComposition`] when the vector is +/// empty, has fewer than two parts, contains a non-finite or non-positive +/// entry, or does not sum to one within a tight absolute tolerance. +pub fn isometric_log_ratio(proportions: &[f64]) -> Result, TopicMeasurementError> { + require_composition(proportions)?; + let dimension = proportions.len(); + let logs: Vec = proportions.iter().map(|part| part.ln()).collect(); + let mut coordinates = Vec::with_capacity(dimension - 1); + for index in 0..(dimension - 1) { + let remaining = dimension - index - 1; + #[allow(clippy::cast_precision_loss)] + let remaining_f = remaining as f64; + let scale = (remaining_f / (remaining_f + 1.0)).sqrt(); + let mut rest_sum = 0.0_f64; + for log_part in &logs[index + 1..] { + rest_sum += *log_part; + } + coordinates.push(scale * (logs[index] - rest_sum / remaining_f)); + } + Ok(coordinates) +} + +/// Invert isometric log-ratio coordinates back to the unit simplex. +/// +/// # Errors +/// +/// Returns [`TopicMeasurementError::InvalidLogRatioDimension`] when the +/// coordinate vector is empty, non-finite, or would underflow a part to zero +/// in the strictly positive `f64` simplex representation. +pub fn from_isometric_log_ratio(coordinates: &[f64]) -> Result, TopicMeasurementError> { + if coordinates.is_empty() { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + for &value in coordinates { + if !value.is_finite() { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + } + + let dimension = coordinates.len() + 1; + let mut centered_logs = vec![0.0_f64; dimension]; + for (index, &coordinate) in coordinates.iter().enumerate() { + let remaining = dimension - index - 1; + #[allow(clippy::cast_precision_loss)] + let remaining_f = remaining as f64; + let scale = (remaining_f / (remaining_f + 1.0)).sqrt(); + let negative = -1.0 / (remaining_f * (remaining_f + 1.0)).sqrt(); + centered_logs[index] += scale * coordinate; + for centered in &mut centered_logs[index + 1..] { + *centered += negative * coordinate; + } + } + + let mut maximum = centered_logs[0]; + for &value in ¢ered_logs[1..] { + maximum = maximum.max(value); + } + if !maximum.is_finite() { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + + let mut weights = Vec::with_capacity(dimension); + let mut denominator = 0.0_f64; + for &value in ¢ered_logs { + let weight = (value - maximum).exp(); + if weight == 0.0 { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + denominator += weight; + weights.push(weight); + } + Ok(weights.iter().map(|weight| weight / denominator).collect()) +} + +fn require_composition(proportions: &[f64]) -> Result { + if proportions.len() < 2 { + return Err(TopicMeasurementError::InvalidComposition); + } + let mut sum = 0.0_f64; + let mut compensation = 0.0_f64; + for &part in proportions { + if !part.is_finite() || part <= 0.0 { + return Err(TopicMeasurementError::InvalidComposition); + } + let next = sum + part; + compensation += if sum.abs() >= part.abs() { + (sum - next) + part + } else { + (part - next) + sum + }; + sum = next; + } + let compensated_sum = sum + compensation; + if !compensated_sum.is_finite() || (compensated_sum - 1.0).abs() > UNIT_SUM_TOLERANCE { + return Err(TopicMeasurementError::InvalidComposition); + } + Ok(proportions[proportions.len() - 1]) +} + +#[cfg(test)] +mod tests { + use super::{ + additive_log_ratio, from_additive_log_ratio, from_isometric_log_ratio, isometric_log_ratio, + }; + use crate::error::TopicMeasurementError; + + #[test] + fn two_part_equal_shares_are_zero_and_unrepresentable_extremes_fail_closed() { + let pair = additive_log_ratio(&[0.5, 0.5]).expect("pair"); + assert_eq!(pair.len(), 1); + assert!(pair[0].abs() < 1e-15); + let recovered = from_additive_log_ratio(&pair).expect("inverse"); + assert!((recovered[0] - 0.5).abs() < 1e-15); + assert!((recovered[1] - 0.5).abs() < 1e-15); + assert_eq!( + from_additive_log_ratio(&[1.0e9]), + Err(TopicMeasurementError::InvalidLogRatioDimension) + ); + assert_eq!( + additive_log_ratio(&[f64::MAX, f64::MAX]), + Err(TopicMeasurementError::InvalidComposition), + "overflowing finite parts must fail closed because compensated mass is non-finite" + ); + let origin = isometric_log_ratio(&[0.5, 0.5]).expect("ilr origin"); + assert!(origin[0].abs() < 1e-15); + let recovered_ilr = from_isometric_log_ratio(&origin).expect("ilr inverse"); + assert!((recovered_ilr[0] - 0.5).abs() < 1e-15); + assert_eq!( + from_isometric_log_ratio(&[1000.0]), + Err(TopicMeasurementError::InvalidLogRatioDimension) + ); + assert_eq!( + from_isometric_log_ratio(&[-f64::MAX, f64::MAX]), + Err(TopicMeasurementError::InvalidLogRatioDimension) + ); + let three = isometric_log_ratio(&[2.0 / 6.0, 3.0 / 6.0, 1.0 / 6.0]).expect("ilr three"); + assert!((three[1] - (0.5_f64).sqrt() * 3.0_f64.ln()).abs() < 1e-15); + let recovered_three = from_isometric_log_ratio(&three).expect("ilr three inverse"); + assert!((recovered_three.iter().sum::() - 1.0).abs() < 1e-15); + } +} diff --git a/crates/topic_measurement/src/error.rs b/crates/topic_measurement/src/error.rs new file mode 100644 index 00000000..ea934e52 --- /dev/null +++ b/crates/topic_measurement/src/error.rs @@ -0,0 +1,50 @@ +//! Fail-closed topic-coordinate errors. + +use std::fmt; + +/// A fail-closed topic-measurement error. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum TopicMeasurementError { + /// Composition is empty, has fewer than two parts, is non-positive, + /// non-finite, or does not sum to one. + InvalidComposition, + /// Log-ratio vector is empty, non-finite, or not representable as a strictly positive `f64` simplex. + InvalidLogRatioDimension, + /// TF-IDF, BM25, or keyword scores were offered as inferential coordinates. + LexicalWeightForbidden, +} + +impl fmt::Display for TopicMeasurementError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + let message = match self { + Self::InvalidComposition => "invalid compositional topic vector", + Self::InvalidLogRatioDimension => "invalid log-ratio dimension", + Self::LexicalWeightForbidden => "lexical inferential weights are forbidden", + }; + formatter.write_str(message) + } +} + +impl std::error::Error for TopicMeasurementError {} + +#[cfg(test)] +mod tests { + use super::TopicMeasurementError; + + #[test] + fn messages_are_stable() { + assert_eq!( + TopicMeasurementError::InvalidComposition.to_string(), + "invalid compositional topic vector" + ); + assert_eq!( + TopicMeasurementError::InvalidLogRatioDimension.to_string(), + "invalid log-ratio dimension" + ); + assert_eq!( + TopicMeasurementError::LexicalWeightForbidden.to_string(), + "lexical inferential weights are forbidden" + ); + } +} diff --git a/crates/topic_measurement/src/lexical.rs b/crates/topic_measurement/src/lexical.rs new file mode 100644 index 00000000..588f60f9 --- /dev/null +++ b/crates/topic_measurement/src/lexical.rs @@ -0,0 +1,36 @@ +//! Refusal of lexical heuristics as inferential topic coordinates. + +use crate::error::TopicMeasurementError; + +/// Refuse TF-IDF, BM25, and keyword scores as topic-estimator coordinates. +/// +/// ADR 0012 forbids treating lexical retrieval weights as inferential topic +/// coordinates. A recognized statistical method name is accepted so callers +/// can share one vocabulary gate. +/// +/// # Errors +/// +/// Returns [`TopicMeasurementError::LexicalWeightForbidden`] for empty labels +/// and for `tfidf`, `bm25`, and `keyword` after alphanumeric folding. +pub fn refuse_lexical_inferential_weight(method: &str) -> Result<(), TopicMeasurementError> { + let folded: String = method + .chars() + .filter(char::is_ascii_alphanumeric) + .flat_map(char::to_lowercase) + .collect(); + if folded.is_empty() || matches!(folded.as_str(), "tfidf" | "bm25" | "keyword") { + return Err(TopicMeasurementError::LexicalWeightForbidden); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::refuse_lexical_inferential_weight; + + #[test] + fn statistical_method_names_are_allowed() { + refuse_lexical_inferential_weight("tepp_topic_measurement").expect("allowed"); + refuse_lexical_inferential_weight("logistic_normal").expect("allowed"); + } +} diff --git a/crates/topic_measurement/src/lib.rs b/crates/topic_measurement/src/lib.rs new file mode 100644 index 00000000..0fff8c73 --- /dev/null +++ b/crates/topic_measurement/src/lib.rs @@ -0,0 +1,27 @@ +#![forbid(unsafe_code)] +#![deny(missing_docs)] +//! Logistic-normal and log-ratio coordinates for compositional topic proportions. +//! +//! Raw topic proportions are compositional rather than unconstrained Euclidean +//! indicators. ALR supplies a reference-dependent full-rank logistic-normal map +//! for regression and psychometric interfaces; it is not an orthonormal +//! Aitchison-distance isometry. Distance-based Aitchison geometry uses the +//! sequential Egozcue ILR basis. TF-IDF, BM25, and keyword scores remain +//! forbidden inferential coordinates. + +mod coordinates; +mod error; +mod lexical; + +/// Additive log-ratio map from a simplex vector. +pub use coordinates::additive_log_ratio; +/// Inverse additive log-ratio map back to the simplex. +pub use coordinates::from_additive_log_ratio; +/// Inverse isometric log-ratio map back to the simplex. +pub use coordinates::from_isometric_log_ratio; +/// Isometric log-ratio map from a simplex vector. +pub use coordinates::isometric_log_ratio; +/// Fail-closed topic-coordinate errors. +pub use error::TopicMeasurementError; +/// Refuse lexical retrieval weights as inferential coordinates. +pub use lexical::refuse_lexical_inferential_weight; diff --git a/crates/topic_measurement/tests/composition_sum_precision_contract.rs b/crates/topic_measurement/tests/composition_sum_precision_contract.rs new file mode 100644 index 00000000..f76cb763 --- /dev/null +++ b/crates/topic_measurement/tests/composition_sum_precision_contract.rs @@ -0,0 +1,39 @@ +//! Composition validation must not lose tiny positive mass after a dominant part. + +use topic_measurement::{TopicMeasurementError, additive_log_ratio}; + +#[test] +fn compensated_sum_rejects_mass_hidden_by_naive_floating_point_addition() { + let mut composition = Vec::with_capacity(20_001); + composition.push(1.0); + composition.extend(std::iter::repeat_n(1.0e-16, 20_000)); + + assert_eq!( + additive_log_ratio(&composition), + Err(TopicMeasurementError::InvalidComposition), + "the true mass exceeds one by 2e-12 even though naive ordered addition rounds to one" + ); +} + +#[test] +fn overflowing_finite_parts_fail_closed_as_non_finite_mass() { + assert_eq!( + additive_log_ratio(&[f64::MAX, f64::MAX]), + Err(TopicMeasurementError::InvalidComposition), + "Kahan-compensated MAX+MAX is non-finite; NaN cannot pass a unit-sum tolerance comparison" + ); +} + +#[test] +fn compensated_sum_accepts_a_valid_many_part_composition() { + let tiny_mass = 1.0e-16; + let tiny_parts = 10_000_usize; + let dominant = 1.0 - tiny_mass * 10_000.0; + let mut composition = Vec::with_capacity(tiny_parts + 1); + composition.push(dominant); + composition.extend(std::iter::repeat_n(tiny_mass, tiny_parts)); + + let coordinates = additive_log_ratio(&composition).expect("valid unit simplex"); + assert_eq!(coordinates.len(), tiny_parts); + assert!(coordinates.iter().all(|value| value.is_finite())); +} diff --git a/crates/topic_measurement/tests/crate_contract.rs b/crates/topic_measurement/tests/crate_contract.rs new file mode 100644 index 00000000..8f9cfd6a --- /dev/null +++ b/crates/topic_measurement/tests/crate_contract.rs @@ -0,0 +1,7 @@ +//! Integration contract for the `topic_measurement` package identity. + +#[test] +fn package_identity_is_stable() { + let observed = std::hint::black_box(env!("CARGO_PKG_NAME")); + assert_eq!(observed, "topic_measurement"); +} diff --git a/crates/topic_measurement/tests/ilr_recovery_contract.rs b/crates/topic_measurement/tests/ilr_recovery_contract.rs new file mode 100644 index 00000000..12817f6e --- /dev/null +++ b/crates/topic_measurement/tests/ilr_recovery_contract.rs @@ -0,0 +1,118 @@ +//! True-parameter recovery of isometric log-ratio topic coordinates. +#![allow(clippy::cast_precision_loss)] + +use topic_measurement::{ + TopicMeasurementError, additive_log_ratio, from_isometric_log_ratio, isometric_log_ratio, +}; + +fn rmse(truth: &[f64], recovered: &[f64]) -> f64 { + let n = truth.len() as f64; + let sum_sq: f64 = truth + .iter() + .zip(recovered) + .map(|(left, right)| { + let residual = left - right; + residual * residual + }) + .sum(); + (sum_sq / n).sqrt() +} + +#[test] +fn known_simplex_recovers_through_ilr_with_computed_rmse() { + // Closed-form simplex: (2, 3, 1) / 6. + // Sequential Egozcue ILR: y1 = √(2/3) ln(2√3 / 3), y2 = √(1/2) ln 3. + let truth = [2.0 / 6.0, 3.0 / 6.0, 1.0 / 6.0]; + let true_parameters = [ + (2.0_f64 / 3.0).sqrt() * (2.0 * 3.0_f64.sqrt() / 3.0).ln(), + (1.0_f64 / 2.0).sqrt() * 3.0_f64.ln(), + ]; + let coordinates = isometric_log_ratio(&truth).expect("ilr"); + assert_eq!(coordinates.len(), 2); + let parameter_rmse = rmse(&true_parameters, &coordinates); + assert!( + parameter_rmse < 1e-15, + "true-parameter ILR RMSE {parameter_rmse} exceeded machine-scale bound" + ); + + let recovered = from_isometric_log_ratio(&coordinates).expect("inverse"); + let simplex_rmse = rmse(&truth, &recovered); + assert!( + simplex_rmse < 1e-15, + "ILR round-trip RMSE {simplex_rmse} exceeded machine-scale bound" + ); + let sum: f64 = recovered.iter().sum(); + assert!((sum - 1.0).abs() < 1e-15); + + let alr = additive_log_ratio(&truth).expect("alr"); + assert!( + (alr[0] - coordinates[0]).abs() > 1e-6, + "ILR must not collapse to the reference-dependent ALR map" + ); +} + +#[test] +fn equal_shares_are_the_ilr_origin_and_preserve_aitchison_distance() { + let halves = [0.5, 0.5]; + let origin = isometric_log_ratio(&halves).expect("origin"); + assert_eq!(origin.len(), 1); + assert!(origin[0].abs() < 1e-15); + + let unbalanced = [0.8, 0.2]; + let coordinates = isometric_log_ratio(&unbalanced).expect("pair"); + let direct_aitchison_distance = (0.5_f64).sqrt() + * ((unbalanced[0] / unbalanced[1]).ln() - (halves[0] / halves[1]).ln()).abs(); + let ilr_euclidean_distance = coordinates + .iter() + .zip(&origin) + .map(|(left, right)| (left - right).powi(2)) + .sum::() + .sqrt(); + assert!((ilr_euclidean_distance - direct_aitchison_distance).abs() < 1e-15); + + let recovered = from_isometric_log_ratio(&coordinates).expect("inverse"); + assert!(rmse(&unbalanced, &recovered) < 1e-15); +} + +#[test] +fn large_finite_ilr_coordinates_round_trip_or_fail_closed() { + let representable = from_isometric_log_ratio(&[40.0]).expect("representable"); + assert!( + representable + .iter() + .all(|part| part.is_finite() && *part > 0.0) + ); + let recovered = isometric_log_ratio(&representable).expect("forward"); + assert!(rmse(&[40.0], &recovered) < 1e-12); + + assert_eq!( + from_isometric_log_ratio(&[1000.0]), + Err(TopicMeasurementError::InvalidLogRatioDimension), + "ILR inverse must not return a zero simplex part after underflow" + ); + assert_eq!( + from_isometric_log_ratio(&[-f64::MAX, f64::MAX]), + Err(TopicMeasurementError::InvalidLogRatioDimension), + "overflowing CLR reconstruction must fail closed" + ); +} + +#[test] +fn invalid_ilr_inputs_fail_closed() { + assert_eq!( + isometric_log_ratio(&[]), + Err(TopicMeasurementError::InvalidComposition) + ); + assert_eq!( + from_isometric_log_ratio(&[]), + Err(TopicMeasurementError::InvalidLogRatioDimension) + ); + assert_eq!( + from_isometric_log_ratio(&[f64::NAN]), + Err(TopicMeasurementError::InvalidLogRatioDimension) + ); + assert_eq!( + from_isometric_log_ratio(&[f64::INFINITY]), + Err(TopicMeasurementError::InvalidLogRatioDimension) + ); +} diff --git a/crates/topic_measurement/tests/logratio_recovery_contract.rs b/crates/topic_measurement/tests/logratio_recovery_contract.rs new file mode 100644 index 00000000..2b50250d --- /dev/null +++ b/crates/topic_measurement/tests/logratio_recovery_contract.rs @@ -0,0 +1,127 @@ +//! True-parameter recovery of logistic-normal topic coordinates. +#![allow(clippy::cast_precision_loss)] + +use topic_measurement::{ + TopicMeasurementError, additive_log_ratio, from_additive_log_ratio, + refuse_lexical_inferential_weight, +}; + +fn rmse(truth: &[f64], recovered: &[f64]) -> f64 { + let n = truth.len() as f64; + let sum_sq: f64 = truth + .iter() + .zip(recovered) + .map(|(left, right)| { + let residual = left - right; + residual * residual + }) + .sum(); + (sum_sq / n).sqrt() +} + +#[test] +fn known_simplex_recovers_through_alr_with_computed_rmse() { + // Closed-form simplex: (2, 3, 1) / 6. ALR is (ln 2, ln 3). + let truth = [2.0 / 6.0, 3.0 / 6.0, 1.0 / 6.0]; + let coordinates = additive_log_ratio(&truth).expect("alr"); + let true_parameters = [2.0_f64.ln(), 3.0_f64.ln()]; + assert_eq!(coordinates.len(), 2); + let parameter_rmse = rmse(&true_parameters, &coordinates); + assert!( + parameter_rmse < 1e-15, + "true-parameter ALR RMSE {parameter_rmse} exceeded machine-scale bound" + ); + + let recovered = from_additive_log_ratio(&coordinates).expect("inverse"); + let simplex_rmse = rmse(&truth, &recovered); + assert!( + simplex_rmse < 1e-15, + "ALR round-trip RMSE {simplex_rmse} exceeded machine-scale bound" + ); + let sum: f64 = recovered.iter().sum(); + assert!((sum - 1.0).abs() < 1e-15); +} + +#[test] +fn large_finite_coordinates_round_trip_without_exponential_overflow() { + let truth = [710.0, 709.0]; + let simplex = from_additive_log_ratio(&truth) + .expect("finite representable ALR coordinates must use a stable inverse"); + assert!(simplex.iter().all(|part| part.is_finite() && *part > 0.0)); + assert!((simplex.iter().sum::() - 1.0).abs() < 1e-15); + + let recovered = additive_log_ratio(&simplex) + .expect("forward ALR must subtract logs instead of overflowing the ratio"); + assert!( + rmse(&truth, &recovered) < 1e-10, + "large-coordinate round trip must retain the true parameters" + ); +} + +#[test] +fn equal_shares_map_to_zero_alr_and_refuse_raw_euclidean_use() { + let thirds = [1.0 / 3.0, 1.0 / 3.0, 1.0 / 3.0]; + let coordinates = additive_log_ratio(&thirds).expect("equal"); + assert!(coordinates.iter().all(|value| value.abs() < 1e-15)); + let recovered = from_additive_log_ratio(&[0.0, 0.0]).expect("zeros"); + assert!(rmse(&thirds, &recovered) < 1e-15); +} + +#[test] +fn invalid_compositions_and_lexical_weights_fail_closed() { + // K=2 is valid; zero/negative/non-unit-sum/non-finite/K<2 are not. + assert_eq!( + additive_log_ratio(&[0.0, 1.0]), + Err(TopicMeasurementError::InvalidComposition) + ); + assert_eq!( + additive_log_ratio(&[-0.1, 1.1]), + Err(TopicMeasurementError::InvalidComposition) + ); + assert_eq!( + additive_log_ratio(&[0.2, 0.2, 0.2]), + Err(TopicMeasurementError::InvalidComposition) + ); + assert_eq!( + additive_log_ratio(&[f64::NAN, 1.0]), + Err(TopicMeasurementError::InvalidComposition) + ); + assert_eq!( + additive_log_ratio(&[]), + Err(TopicMeasurementError::InvalidComposition) + ); + assert_eq!( + additive_log_ratio(&[1.0]), + Err(TopicMeasurementError::InvalidComposition) + ); + assert_eq!( + from_additive_log_ratio(&[]), + Err(TopicMeasurementError::InvalidLogRatioDimension) + ); + assert_eq!( + from_additive_log_ratio(&[f64::INFINITY]), + Err(TopicMeasurementError::InvalidLogRatioDimension) + ); + assert_eq!( + from_additive_log_ratio(&[1.0e9]), + Err(TopicMeasurementError::InvalidLogRatioDimension), + "max-shifted reference weight must fail closed when it underflows to zero" + ); + assert_eq!( + from_additive_log_ratio(&[-1.0e9]), + Err(TopicMeasurementError::InvalidLogRatioDimension), + "inverse must not return a zero simplex part after underflow" + ); + assert_eq!( + additive_log_ratio(&[f64::MAX, f64::MAX]), + Err(TopicMeasurementError::InvalidComposition), + "overflowing finite parts must fail closed because compensated mass is non-finite" + ); + + for method in ["tfidf", "bm25", "keyword", "TF-IDF", ""] { + assert_eq!( + refuse_lexical_inferential_weight(method), + Err(TopicMeasurementError::LexicalWeightForbidden) + ); + } +} diff --git a/docs/TRACEABILITY.md b/docs/TRACEABILITY.md index a3e674cf..c47242b1 100644 --- a/docs/TRACEABILITY.md +++ b/docs/TRACEABILITY.md @@ -22,9 +22,9 @@ The full APA 7th standards/literature register remains `docs/research/standards- | versioned service/API contracts and exports | PRD; API contract; ADR 0011/0013 | `tepp_api` analysis-run/export/JSON-LD/GraphML contracts on protected main (PR #21); HTTP service remaining accepted-target | partial | | immutable split/run/reproducibility manifests | ADR 0013; ERD | `tepp_api` reproducibility manifest contract on protected main; `persistence_postgres` append-only SQL insert/lookup for `reproducibility_manifest`, `corpus_split_manifest`, `model_run`, and `model_artifact` (migration `0003`); full physical ERD constraints remaining | partial | | multilingual shared latent semantic space | PRD; ADR 0004 | future semantic/concept/topic crates | accepted-target | -| TRSL-TM temporal/relational topic posterior and backend compatibility | ADR 0012; ADR 0004 | future `topic_measurement` | accepted-target | +| TRSL-TM temporal/relational topic posterior and backend compatibility | ADR 0012; ADR 0004 | `topic_measurement` stable ALR and sequential Egozcue ILR coordinates on the active PR; temporal STM backend remaining | partial | | global P0 topic identity with activity/dormancy/reactivation | ADR 0012 | future topic lineage/activity state | accepted-target | -| no default stopword deletion / no TF-IDF-BM25 inferential weighting | ADR 0004/0012; PRD/TRD | future semantic/method-source model | accepted-target | +| no default stopword deletion / no TF-IDF-BM25 inferential weighting | ADR 0004/0012; PRD/TRD | `topic_measurement::refuse_lexical_inferential_weight` on the active PR; preprocessing pipeline remaining | partial | | report template/section/copied/style/modality method effects | ADR 0004/0012; PRD/TRD | simulation truth factors implemented; estimator-side method model remains future | partial | | candidate K statistical/Pareto gates + blinded LLM review | ADR 0012; research | future `model_selection` | accepted-target | | compositional topic correlation / stable clustering | ADR 0005/0012; research | future `network_analysis` | accepted-target | diff --git a/docs/adr/0010-adaptive-llm-orchestration.md b/docs/adr/0010-adaptive-llm-orchestration.md index a3378983..66eb158f 100644 --- a/docs/adr/0010-adaptive-llm-orchestration.md +++ b/docs/adr/0010-adaptive-llm-orchestration.md @@ -1,7 +1,7 @@ # ADR 0010 — Adaptive LLM orchestration and test-time compute -**Decision status:** Accepted -**Implementation maturity:** partial — `tepp_api` governed router, comparable-budget ablation record, and credential-free contextual-orchestrator binding are implemented on the active PR and are not implemented-main until exact-head checks, review, and protected-main integration complete; live NIM execution, learned conductor calibration, and production ablation evidence remain accepted-target +**Decision status:** Accepted +**Implementation maturity:** partial — `tepp_api` governed router, comparable-budget ablation record, and credential-free contextual-orchestrator binding are implemented on this active PR and are not implemented-main until exact-head checks, review, and protected-main integration complete; live NIM execution, learned conductor calibration, and production ablation evidence remain accepted-target **Date:** 2026-08-10 **Supersedes:** The LLM orchestration-selection/ablation clauses previously co-located in ADR 0006. ADR 0006 remains authoritative for GPU/VRAM and model-credential separation; ADR 0015 governs autonomous repository-write/review/merge authority. diff --git a/docs/adr/0011-standalone-modular-msa-boundary.md b/docs/adr/0011-standalone-modular-msa-boundary.md index d545ee23..04181fb3 100644 --- a/docs/adr/0011-standalone-modular-msa-boundary.md +++ b/docs/adr/0011-standalone-modular-msa-boundary.md @@ -1,7 +1,7 @@ # ADR 0011 — Standalone operation and modular CWL MSA boundary **Decision status:** Accepted -**Implementation maturity:** partial — Rust crates are independently usable; naruon HTTP interchange and loopback live listener (`POST /v1/analysis-runs` and `/v1/exports`, fail-closed table-access, NIM/proxy headers, RFC 3339 cutoff, stream deadline) are on the active PR (not implemented-main); production TLS/`$PORT` and remaining persistence integrations remain accepted-target +**Implementation maturity:** partial — Rust crates are independently usable; naruon HTTP interchange and loopback live listener (`POST /v1/analysis-runs` and `/v1/exports`, fail-closed table-access, NIM/proxy headers, RFC 3339 cutoff, stream deadline) are on the active PR (not implemented-main); production TLS/`$PORT` and remaining persistence integrations remain accepted-target **Date:** 2026-08-10 **Supersedes:** The broad cross-service ownership wording in ADR 0001. ADR 0001 remains authoritative for Rust-first numerical architecture. diff --git a/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md b/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md index c3d5085f..40f338ba 100644 --- a/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md +++ b/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md @@ -1,7 +1,7 @@ # ADR 0012 — Temporal Relational Shared-Latent Topic Measurement **Decision status:** Accepted -**Implementation maturity:** accepted-target +**Implementation maturity:** partial — logistic-normal additive log-ratio coordinates, sequential Egozcue isometric log-ratio coordinates, and lexical-weight refusal are implemented on the active PR and are not implemented-main until exact-head checks, review, and protected-main integration complete; temporal topic identity, backend STM estimator, method-effect model, and K-selection remain accepted-target **Date:** 2026-08-12 **Supersedes:** None; refines ADR 0004 and ADR 0005 without replacing their multilingual and psychometric authorities. @@ -25,6 +25,7 @@ For the first production line: - stopword deletion is not the default preprocessing rule; - TF-IDF and BM25 are not inferential weights for the statistical topic estimator; - topic proportions are compositional and downstream network/psychometric analysis uses logistic-normal coordinates or valid orthonormal log-ratio coordinates; +- ALR is a reference-dependent full-rank logistic-normal map, not an Aitchison-distance isometry; distance-based Euclidean Aitchison geometry uses an orthonormal ILR basis; - model selection uses statistical/recovery/stability/alignment/fairness gates and a Pareto-style comparison before any blinded LLM review; - the LLM may recommend among statistically admissible candidates but never defines the numerical optimum or bypasses diagnostics. diff --git a/docs/adr/0013-bitemporal-persistence-reproducibility-and-split-authority.md b/docs/adr/0013-bitemporal-persistence-reproducibility-and-split-authority.md index a593ddb1..7f3706e5 100644 --- a/docs/adr/0013-bitemporal-persistence-reproducibility-and-split-authority.md +++ b/docs/adr/0013-bitemporal-persistence-reproducibility-and-split-authority.md @@ -1,7 +1,7 @@ # ADR 0013 — Bitemporal persistence, reproducibility manifests, and split authority **Decision status:** Accepted -**Implementation maturity:** partial — migration contracts, cutoff eligibility, in-memory bitemporal adapters, live SQL session/migration port, document SQL contracts, `DATABASE_URL` SQLx gate, optional `live-sqlx` `PgPool` open/execute driver, exact-head live PostgreSQL CI, tenant RLS (`tepp_app_runtime` + session GUC), append-only reproducibility-manifest SQL insert/lookup, model-run / model-artifact / corpus-split-manifest chain (migration `0003`), append-only immutability triggers (migration `0004`), temporal interval ordering CHECK constraints (migration `0005`), typed membership-assignment storage (migration `0006`), event-relation/mention/instance SQL, source-artifact SQL, audit-event action-code validation, and concurrent document-write stress implemented-main; backup/restore integrity revalidation on the active PR +**Implementation maturity:** partial — migration contracts, cutoff eligibility, in-memory bitemporal adapters, live SQL session/migration port, document SQL contracts, `DATABASE_URL` SQLx gate, optional `live-sqlx` `PgPool` open/execute driver, exact-head live PostgreSQL CI, tenant RLS (`tepp_app_runtime` + session GUC), append-only reproducibility-manifest SQL insert/lookup, model-run / model-artifact / corpus-split-manifest chain (migration `0003`), append-only immutability triggers (migration `0004`), temporal interval ordering CHECK constraints (migration `0005`), typed membership-assignment storage (migration `0006`), event-relation/mention/instance SQL, source-artifact SQL, audit-event action-code validation, and concurrent document-write stress implemented-main; backup/restore integrity revalidation implemented-main; remaining physical ERD/backup accepted-target **Date:** 2026-08-12 **Supersedes:** None; complements ADR 0002 (temporal semantics), ADR 0008 (evidence identity), and ADR 0011 (service ownership). diff --git a/docs/adr/README.md b/docs/adr/README.md index 258eb7f3..63c0d033 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -14,11 +14,11 @@ Read [`ADR_POLICY.md`](ADR_POLICY.md) first. **Decision status and implementatio | [0006](0006-vram-gpu-nvidia-orchestration.md) | VRAM-adaptive GPU compute and model-credential boundary | Accepted | accepted-target | LLM orchestration policy superseded by ADR 0010; autonomous development authority governed by ADR 0015. | | [0007](0007-rust-workspace-quality-gates.md) | Explicit Rust workspace, pinned toolchains, and exact quality gates | Accepted | implemented-main | ADR 0014 governs scientific/product claim promotion beyond repository-quality tooling. | | [0008](0008-immutable-evidence-identities-digests-and-spans.md) | Immutable evidence identities, `SHA-256` digests, exact spans, and strict wire reconstruction | Accepted | implemented-main | ADR 0013 governs future persistence/reproducibility/split authority. | -| [0009](0009-purpose-bound-pii-governance.md) | Purpose-bound PII governance without blanket masking | Accepted | partial | Persistence retention/deletion/legal-hold (`0007`) and provider-payload minimization implemented-main; deployment evidence remains accepted-target. | -| [0010](0010-adaptive-llm-orchestration.md) | Adaptive LLM orchestration and test-time compute | Accepted | partial | `tepp_api` router/ablation/orchestrator binding on the active PR; live NIM execution and production ablation evidence remain accepted-target. | +| [0009](0009-purpose-bound-pii-governance.md) | Purpose-bound PII governance without blanket masking | Accepted | partial | Persistence retention/deletion/legal-hold (`0007`) is implemented-main; provider-payload minimization remains on the active PR until exact-head checks, review, and protected-main integration; deployment evidence remains accepted-target. | +| [0010](0010-adaptive-llm-orchestration.md) | Adaptive LLM orchestration and test-time compute | Accepted | partial | `tepp_api` router/ablation/orchestrator binding remain on the active PR until exact-head checks, review, and protected-main integration; live NIM execution and production ablation evidence remain accepted-target. | | [0011](0011-standalone-modular-msa-boundary.md) | Standalone operation and modular CWL MSA boundary | Accepted | partial | Owns cross-service persistence/credential/API authority; no direct cross-service application-table coupling. | -| [0012](0012-temporal-relational-shared-latent-topic-measurement.md) | Temporal Relational Shared-Latent Topic Measurement (TRSL-TM) | Accepted | accepted-target | Owns topic backend compatibility, global topic identity, method effects, K/model-selection prerequisites, and compositional topic coordinates. | -| [0013](0013-bitemporal-persistence-reproducibility-and-split-authority.md) | Bitemporal persistence, reproducibility manifests, and relation-aware split authority | Accepted | partial | Owns PostgreSQL adapter semantics, immutable run/split manifests, leakage-safe partitions, and recovery identity; optional `live-sqlx` `PgPool`, live PG CI, tenant RLS, and `0006` membership implemented-main; `0007` retention/deletion/legal-hold on the active PR; remaining physical ERD/backup accepted-target. | +| [0012](0012-temporal-relational-shared-latent-topic-measurement.md) | Temporal Relational Shared-Latent Topic Measurement (TRSL-TM) | Accepted | partial | Logistic-normal ALR, sequential Egozcue ILR, and lexical-weight refusal are on the active PR; temporal topic identity, STM backend, method effects, and K-selection remain accepted-target. | +| [0013](0013-bitemporal-persistence-reproducibility-and-split-authority.md) | Bitemporal persistence, reproducibility manifests, and relation-aware split authority | Accepted | partial | Owns PostgreSQL adapter semantics, immutable run/split manifests, leakage-safe partitions, and recovery identity; optional `live-sqlx` `PgPool`, live PG CI, tenant RLS, and `0006` membership implemented-main; backup/restore integrity revalidation implemented-main; `0007` retention/deletion/legal-hold implemented-main; remaining physical ERD/backup accepted-target. | | [0014](0014-scientific-claim-promotion-and-release-evidence.md) | Scientific claim promotion and release evidence authority | Accepted | partial | Separates design, implementation, scientific/product claim, and release authority; repository SBOM/provenance generator implemented, full release bundle remaining. | | [0015](0015-autonomous-development-review-and-merge-authority.md) | Autonomous development, review, and merge authority separation | Accepted | active-PR | Separates model proposal, deterministic verification, publication, independent review, and merge/release authority. | | [0016](0016-tdt-chronos-event-intelligence-boundary.md) | TDT, CHRONOS, and Event Ontology intelligence boundary | Accepted | accepted-target | Separates observed evidence, detection/tracking, prediction/schema inference, temporal consistency, and promoted transition authority. | diff --git a/docs/connectors/naruon-artifact-consumer.md b/docs/connectors/naruon-artifact-consumer.md index 2e4f4d6c..5fe0424c 100644 --- a/docs/connectors/naruon-artifact-consumer.md +++ b/docs/connectors/naruon-artifact-consumer.md @@ -1,6 +1,6 @@ # naruon modular consumer contract for TEPP artifacts -**Status:** Partial — versioned DTO, HTTP interchange, and loopback live listener on the active PR; production TLS/`$PORT` remaining +**Status:** Partial — versioned DTO, HTTP interchange, and loopback live listener on the active PR; production TLS/`$PORT` remaining **Last reviewed:** 2026-08-16 ## Boundary diff --git a/docs/research/standards-and-literature.md b/docs/research/standards-and-literature.md index 28e62d5c..ccbde909 100644 --- a/docs/research/standards-and-literature.md +++ b/docs/research/standards-and-literature.md @@ -48,8 +48,12 @@ LLM evaluation complements but never replaces predictive, posterior, stability, ## Compositional data, correlation, and clusters +Aitchison, J., & Shen, S. M. (1980). Logistic-normal distributions: Some properties and uses. *Biometrika, 67*(2), 261–272. https://doi.org/10.1093/biomet/67.2.261 + Aitchison, J. (1982). The statistical analysis of compositional data. *Journal of the Royal Statistical Society: Series B (Methodological), 44*(2), 139–177. https://doi.org/10.1111/j.2517-6161.1982.tb01195.x +Egozcue, J. J., Pawlowsky-Glahn, V., Mateu-Figueras, G., & Barceló-Vidal, C. (2003). Isometric logratio transformations for compositional data analysis. *Mathematical Geology, 35*(3), 279–300. https://doi.org/10.1023/A:1023818214614 + Friedman, J., Hastie, T., & Tibshirani, R. (2008). Sparse inverse covariance estimation with the graphical lasso. *Biostatistics, 9*(3), 432–441. https://doi.org/10.1093/biostatistics/kxm045 Traag, V. A., Waltman, L., & van Eck, N. J. (2019). From Louvain to Leiden: Guaranteeing well-connected communities. *Scientific Reports, 9*, Article 5233. https://doi.org/10.1038/s41598-019-41695-z diff --git a/docs/research/topic-logratio-coordinates.md b/docs/research/topic-logratio-coordinates.md new file mode 100644 index 00000000..4faf429b --- /dev/null +++ b/docs/research/topic-logratio-coordinates.md @@ -0,0 +1,38 @@ +# Logistic-normal topic coordinates + +## Scope + +This note doctors the first `topic_measurement` production slice (ADR 0012): + +1. raw topic proportions are compositional rather than unconstrained Euclidean indicators (Aitchison, 1982); +2. additive log-ratio coordinates implement the reference-dependent logistic-normal map used by correlated topic models (Aitchison & Shen, 1980; Blei & Lafferty, 2007); +3. ALR is full rank but not an orthonormal Aitchison-distance isometry (Aitchison, 1982); +4. sequential Egozcue ILR supplies the orthonormal Aitchison-distance isometry when that Euclidean geometry is the estimand (Egozcue et al., 2003); +5. max-shifted inverses recover representable extreme coordinates without overflow (Aitchison & Shen, 1980); +6. TF-IDF, BM25, and keyword scores are refused as inferential coordinates. + +The temporal STM backend, global topic identity, method-effect model, and K-selection remain accepted-target. No database migration is allocated. + +## Authoritative sources + +Aitchison, J., & Shen, S. M. (1980). Logistic-normal distributions: Some properties and uses. *Biometrika, 67*(2), 261–272. https://doi.org/10.1093/biomet/67.2.261 + +Aitchison, J. (1982). The statistical analysis of compositional data. *Journal of the Royal Statistical Society: Series B (Methodological), 44*(2), 139–177. https://doi.org/10.1111/j.2517-6161.1982.tb01195.x + +Blei, D. M., & Lafferty, J. D. (2007). A correlated topic model of Science. *The Annals of Applied Statistics, 1*(1), 17–35. https://doi.org/10.1214/07-AOAS114 + +Egozcue, J. J., Pawlowsky-Glahn, V., Mateu-Figueras, G., & Barceló-Vidal, C. (2003). Isometric logratio transformations for compositional data analysis. *Mathematical Geology, 35*(3), 279–300. https://doi.org/10.1023/A:1023818214614 + +## Application + +Aitchison and Shen (1980) define the logistic-normal family via the additive log-ratio map; Aitchison (1982) is the compositional-data authority that forbids treating parts of a whole as unconstrained Euclidean coordinates. Blei and Lafferty (2007) use that same reference-dependent map for correlated topic models. TEPP therefore uses `additive_log_ratio` for logistic-normal regression and psychometric interfaces, but does not claim that ALR preserves Aitchison distance. Egozcue et al. (2003) construct the sequential orthonormal ILR basis whose Euclidean norm equals Aitchison distance; `isometric_log_ratio` implements that basis and `from_isometric_log_ratio` inverts it through a max-shifted centered-log-ratio reconstruction. Analyses whose estimand is orthonormal Euclidean Aitchison geometry must use ILR rather than ALR (Egozcue et al., 2003). `from_additive_log_ratio` treats the omitted reference component as logit zero, max-shifts all `K` logits together, and normalizes only after exponentiation (Aitchison & Shen, 1980). The forward ALR map subtracts logarithms rather than forming a potentially overflowing ratio (Aitchison & Shen, 1980; Blei & Lafferty, 2007). Both inverses fail closed when an `f64` simplex part would underflow to zero (Aitchison & Shen, 1980). + +## Verification + +- closed-form simplex `(2,3,1)/6` maps to ALR `(ln 2, ln 3)` and sequential ILR `(√(2/3) ln(2√3/3), √(1/2) ln 3)` with computed RMSE below `1e-15`; +- representable ALR coordinates `(710, 709)` and representable ILR coordinates round-trip through max-shifted inverses without exponential overflow; +- extremes that would underflow a strictly positive `f64` simplex part fail closed; +- equal shares map to the ALR and ILR origins; +- two-part ILR preserves Aitchison distance `√(1/2) |ln(0.8/0.2) - ln(0.5/0.5)|` between `(0.8, 0.2)` and `(0.5, 0.5)` (Egozcue et al., 2003); +- zero, negative, non-unit-sum, non-finite, empty, and one-part vectors fail closed; +- `tfidf`, `bm25`, and `keyword` labels are refused. diff --git a/docs/validation/temporal-event-foundation.md b/docs/validation/temporal-event-foundation.md index aae1a06e..c3936755 100644 --- a/docs/validation/temporal-event-foundation.md +++ b/docs/validation/temporal-event-foundation.md @@ -24,7 +24,8 @@ This report tracks exact-head scientific and engineering evidence required befor | Recovery metrics | `validation_core` | implemented-main | — | RMSE/bias/coverage/MC gates | Task 11 / PR #19 | | Versioned API/export contracts | `tepp_api` | implemented-main | naruon HTTP interchange | unknown-field/version/limit + naruon HTTPS interchange tests | Task 12 / PR #21; live HTTP service remaining | | Purpose-bound provider payloads | `tepp_api` | implemented-main | provider-payload minimization | expired/not-yet-valid/inverted/cross-tenant/impossible-calendar grant, mapping refusal, audited elevated re-id replay | ADR 0009; `docs/research/provider-payload-minimization.md` | -| Adaptive orchestration router | `tepp_api` | accepted-target | active PR | mode selection, document-control denial, ablation, credential-free bind | ADR 0010; `docs/research/adaptive-orchestration-router.md` | +| Adaptive orchestration router | `tepp_api` | implemented-main | router + ablation | mode selection, document-control denial, ablation, credential-free bind | ADR 0010; `docs/research/adaptive-orchestration-router.md` | +| Logistic-normal topic coordinates | `topic_measurement` | active-PR | stable ALR + sequential ILR + lexical refusal | known-simplex ALR/ILR RMSE, Aitchison-distance ILR isometry | ADR 0012; `docs/research/topic-logratio-coordinates.md` | | CWL modular connectors | `docs/connectors/*` | implemented-main | — | contract docs + examples | PR #22; live HTTP ports remaining | | Release SBOM/provenance generator | `scripts/release_evidence.py` | partial | — | generate+validate in CI | Task 13 partial / PR #28 | diff --git a/scripts/check_coverage.py b/scripts/check_coverage.py index 50234635..6f30a8ab 100644 --- a/scripts/check_coverage.py +++ b/scripts/check_coverage.py @@ -51,9 +51,10 @@ def is_executable_source_line( """Return whether *line_number* in *source_path* is an executable source line. LLVM LCOV sometimes emits zero-count DA records for documentation comments, - attributes, pure structural braces, multi-line signatures, and in-file - ``#[cfg(test)]`` modules. Those records are not evidence of uncovered - production behavior and are excluded from the authored-line gate. + attributes, pure structural braces, multi-line signatures, Rust multiline + string continuations, and in-file ``#[cfg(test)]`` modules. Those records + are not evidence of uncovered production behavior and are excluded from the + authored-line gate. When *repository_root* is provided, *source_path* must resolve under that root (same fail-closed rule as LCOV ``SF:`` loading). @@ -75,6 +76,8 @@ def is_executable_source_line( return False if _line_in_cfg_not_feature_block(lines, line_number): return False + if _line_in_multiline_string_literal(lines, line_number): + return False text = lines[line_number - 1].strip() if not text: return False @@ -82,7 +85,9 @@ def is_executable_source_line( return False if text.startswith("#[") or text.startswith("#!["): return False - if text in {"{", "}", "},", ");", "];", "();", "};"}: + if text in {"{", "}", "},", ")", ");", "];", "();", "};"}: + return False + if _is_standalone_string_literal(text) or text.startswith("} else"): return False if text.startswith("use ") or text.startswith("pub use "): return False @@ -164,6 +169,118 @@ def _line_in_cfg_not_feature_block(lines: list[str], line_number: int) -> bool: return False +def _line_in_multiline_string_literal(lines: list[str], line_number: int) -> bool: + """Return whether a line is inside a Rust string continuation. + + The scanner tracks normal strings, raw strings, block comments, and character + literals so quotes in comments or literal contents cannot change the state of + a later source line. + """ + + in_string = False + raw_hashes: int | None = None + in_block_comment = False + for index, raw in enumerate(lines, start=1): + if (in_string or raw_hashes is not None) and index == line_number: + return True + cursor = 0 + while cursor < len(raw): + if in_block_comment: + if raw.startswith("*/", cursor): + in_block_comment = False + cursor += 2 + else: + cursor += 1 + continue + if raw_hashes is not None: + delimiter = '"' + ("#" * raw_hashes) + closing = raw.find(delimiter, cursor) + if closing == -1: + cursor = len(raw) + else: + raw_hashes = None + cursor = closing + len(delimiter) + continue + if in_string: + character = raw[cursor] + if character == "\\": + cursor += 2 + elif character == '"': + in_string = False + cursor += 1 + else: + cursor += 1 + continue + if raw.startswith("//", cursor): + break + if raw.startswith("/*", cursor): + in_block_comment = True + cursor += 2 + continue + raw_start = _raw_string_start(raw, cursor) + if raw_start is not None: + raw_hashes, cursor = raw_start + continue + if raw[cursor] == '"': + in_string = True + cursor += 1 + continue + if raw[cursor] == "'": + character_end = _character_literal_end(raw, cursor) + if character_end is not None: + cursor = character_end + continue + cursor += 1 + return False + + +def _raw_string_start(line: str, cursor: int) -> tuple[int, int] | None: + """Return ``(hash_count, next_cursor)`` for a Rust raw-string opener.""" + + if line.startswith("br", cursor): + prefix_end = cursor + 2 + elif line.startswith("r", cursor): + prefix_end = cursor + 1 + else: + return None + hash_end = prefix_end + while hash_end < len(line) and line[hash_end] == "#": + hash_end += 1 + if hash_end < len(line) and line[hash_end] == '"': + return hash_end - prefix_end, hash_end + 1 + return None + + +def _character_literal_end(line: str, cursor: int) -> int | None: + """Return the cursor after a one-line Rust character literal, if present.""" + + candidate = cursor + 1 + if candidate >= len(line): + return None + if line[candidate] == "\\": + candidate += 2 + else: + candidate += 1 + if candidate < len(line) and line[candidate] == "'": + return candidate + 1 + return None + + +def _is_standalone_string_literal(text: str) -> bool: + """Return whether *text* is only a normal string literal and punctuation.""" + if not text.startswith('"'): + return False + escaped = False + for index, character in enumerate(text[1:], start=1): + if character == '"' and not escaped: + return text[index + 1 :].strip() in {"", ",", ";"} + if character == "\\": + escaped = not escaped + else: + escaped = False + return False + + def load_lcov_line_totals( path: Path, repository_root: Path | None = None ) -> Mapping[str, Any]: diff --git a/scripts/check_workspace_contract.py b/scripts/check_workspace_contract.py index c7b1ecf5..616f8d07 100644 --- a/scripts/check_workspace_contract.py +++ b/scripts/check_workspace_contract.py @@ -23,6 +23,7 @@ "tepp_simulation", "validation_core", "tepp_api", + "topic_measurement", ) REQUIRED_CI_SNIPPETS: tuple[str, ...] = ( diff --git a/tests/quality/test_check_coverage.py b/tests/quality/test_check_coverage.py index a4337397..4a62b2c9 100644 --- a/tests/quality/test_check_coverage.py +++ b/tests/quality/test_check_coverage.py @@ -305,22 +305,25 @@ def test_executable_source_line_filters_noise_records(self) -> None: "})", # 39 " return value,", # 40 executable (return keeps it) " x + 1,", # 41 trailing comma noise - '#[cfg(feature = "live-sqlx")]', # 42 cfg attr - "fn live_path() {", # 43 fn - " live_body();", # 44 executable active feature body - "}", # 45 brace - '#[cfg(not(feature = "live-sqlx"))]', # 46 not-feature attr - "fn offline_path() {", # 47 inside not-feature - " offline_body();", # 48 inside not-feature - "}", # 49 inside not-feature close - "#[cfg(test)]", # 50 - "mod tests {", # 51 cfg(test) mod - " #[test]", # 52 inside test mod - " fn unit() {", # 53 inside test mod - " assert_eq!(1, 1);", # 54 inside test mod - " }", # 55 - "}", # 56 - " executable_statement();", # 57 executable + '"standalone string literal",', # 42 string noise + "} else {", # 43 structural branch noise + ")", # 44 structural close noise + '#[cfg(feature = "live-sqlx")]', # 45 cfg attr + "fn live_path() {", # 46 fn + " live_body();", # 47 executable active feature body + "}", # 48 brace + '#[cfg(not(feature = "live-sqlx"))]', # 49 not-feature attr + "fn offline_path() {", # 50 inside not-feature + " offline_body();", # 51 inside not-feature + "}", # 52 inside not-feature close + "#[cfg(test)]", # 53 + "mod tests {", # 54 cfg(test) mod + " #[test]", # 55 inside test mod + " fn unit() {", # 56 inside test mod + " assert_eq!(1, 1);", # 57 inside test mod + " }", # 58 + "}", # 59 + " executable_statement();", # 60 executable ] source.write_text("\n".join(source_lines) + "\n", encoding="utf-8") path = str(source) @@ -335,7 +338,7 @@ def test_executable_source_line_filters_noise_records(self) -> None: coverage_contract.is_executable_source_line(path, len(source_lines) + 5) ) - expected_executable = {13, 40, 44, 57} + expected_executable = {13, 40, 47, 60} for line_number in range(1, len(source_lines) + 1): is_exec = coverage_contract.is_executable_source_line(path, line_number) if line_number in expected_executable: @@ -354,10 +357,10 @@ def test_executable_source_line_filters_noise_records(self) -> None: "\n".join( [ f"SF:{path}", - "DA:57,1", + "DA:60,1", "DA:1,0", "DA:2,0", - "DA:48,0", + "DA:51,0", "end_of_record", "", ] @@ -427,6 +430,7 @@ def test_cfg_test_and_not_feature_block_helpers(self) -> None: self.assertTrue(coverage_contract._line_in_cfg_not_feature_block(lines, 9)) self.assertFalse(coverage_contract._line_in_cfg_not_feature_block(lines, 1)) self.assertFalse(coverage_contract._line_in_cfg_not_feature_block(lines, 11)) + open_only = [ '#[cfg(not(feature = "x"))]', "fn unfinished()", @@ -474,6 +478,119 @@ def test_cfg_test_and_not_feature_block_helpers(self) -> None: coverage_contract._line_in_cfg_not_feature_block(unclosed_not_feature, 99) ) + def test_multiline_string_continuations_are_not_authored_lines(self) -> None: + """Rust multiline string fragments are excluded from authored coverage.""" + + with tempfile.TemporaryDirectory() as temporary: + source = Path(temporary) / "query.rs" + source.write_text( + 'fn query() {\n' + ' let sql = format!("SELECT id \\\n' + ' FROM document_record \\\n' + ' WHERE tenant_record_id = \'x\'");\n' + ' execute(sql);\n' + '}\n', + encoding="utf-8", + ) + self.assertTrue( + coverage_contract.is_executable_source_line(str(source), 2) + ) + self.assertFalse( + coverage_contract.is_executable_source_line(str(source), 3) + ) + self.assertFalse( + coverage_contract.is_executable_source_line(str(source), 4) + ) + self.assertTrue( + coverage_contract.is_executable_source_line(str(source), 5) + ) + + def test_string_scanner_handles_comments_backslash_parity_and_methods(self) -> None: + """Quoted comments and escaped delimiters do not corrupt source classification.""" + + with tempfile.TemporaryDirectory() as temporary: + backslash = "\\" + source = Path(temporary) / "scanner.rs" + source_lines = [ + "fn query() {", + f' let sql = "SELECT id {backslash}', + ' FROM document";', + r' // comment contains one " quote', + " execute(sql);", + f' let even = "ends with two slashes {backslash * 2}";', + " execute(even);", + r' "literal".to_string();', + "}", + ] + source.write_text("\n".join(source_lines) + "\n", encoding="utf-8") + path = str(source) + + self.assertFalse(coverage_contract.is_executable_source_line(path, 3)) + self.assertTrue(coverage_contract.is_executable_source_line(path, 5)) + self.assertTrue(coverage_contract.is_executable_source_line(path, 7)) + self.assertTrue(coverage_contract.is_executable_source_line(path, 8)) + + block_comment = [ + "/* comment starts", + r' comment has a " quote', + " still comment", + "*/", + "execute();", + ] + self.assertFalse( + coverage_contract._line_in_multiline_string_literal(block_comment, 5) + ) + self.assertTrue( + coverage_contract._is_standalone_string_literal(r'"escaped\\",') + ) + self.assertFalse( + coverage_contract._is_standalone_string_literal('"unfinished') + ) + + raw_string = [ + ' let text = r##"', + ' a " quote in raw text', + ' "##;', + ' execute(text);', + ] + self.assertFalse( + coverage_contract._line_in_multiline_string_literal(raw_string, 1) + ) + self.assertTrue( + coverage_contract._line_in_multiline_string_literal(raw_string, 2) + ) + self.assertTrue( + coverage_contract._line_in_multiline_string_literal(raw_string, 3) + ) + self.assertFalse( + coverage_contract._line_in_multiline_string_literal(raw_string, 4) + ) + + byte_raw_string = [ + ' let bytes = br#"', + ' raw bytes', + ' "#;', + ] + self.assertTrue( + coverage_contract._line_in_multiline_string_literal(byte_raw_string, 2) + ) + + character_and_lifetime = [ + "fn query<'a>() {", + " let quote: char = '\"';", + " execute();", + "}", + ] + self.assertFalse( + coverage_contract._line_in_multiline_string_literal( + character_and_lifetime, 3 + ) + ) + self.assertIsNone(coverage_contract._character_literal_end("'", 0)) + self.assertEqual( + coverage_contract._character_literal_end(r"'\''", 0), 4 + ) + if __name__ == "__main__": # pragma: no cover unittest.main() diff --git a/tests/quality/test_check_docstrings.py b/tests/quality/test_check_docstrings.py index 2c11f7a5..b99537c5 100644 --- a/tests/quality/test_check_docstrings.py +++ b/tests/quality/test_check_docstrings.py @@ -24,7 +24,7 @@ def test_live_repository_is_documented(self) -> None: sources = docstrings.rust_sources(REPOSITORY_ROOT) crate_roots = sorted(REPOSITORY_ROOT.glob("crates/*/src/lib.rs")) - self.assertEqual(len(crate_roots), 10) + self.assertEqual(len(crate_roots), 11) self.assertTrue(set(crate_roots).issubset(sources)) self.assertGreaterEqual(len(sources), len(crate_roots)) self.assertEqual(docstrings.validate_repository(REPOSITORY_ROOT), [])