diff --git a/lib/src/onehz/human/coaching.dart b/lib/src/onehz/human/coaching.dart index 9ad55ca..f7c906d 100644 --- a/lib/src/onehz/human/coaching.dart +++ b/lib/src/onehz/human/coaching.dart @@ -1,6 +1,7 @@ import 'dart:math' as math; import '../types.dart'; +import '../util.dart' show mean, theilSen; class SleepNeed { @@ -458,3 +459,240 @@ List journalCorrelations({ out.sort((a, b) => a.tag.compareTo(b.tag)); return out; } + +// --------------------------------------------------------------------------- +// Numeric journal fields +// +// [journalCorrelations] above answers "were the tagged days different?", which +// is the only question a tag set can answer. A field that carries a NUMBER — +// three coffees, 700 ml of water, mood 4/5, 90 minutes of screens — carries a +// dose, and collapsing it to present/absent throws that away: it cannot tell +// one coffee from five, which is usually the whole question. +// +// The statistic is Spearman's rank correlation (Spearman 1904), not Pearson. +// Self-reported dose is ordinal at best and routinely spiky (a single +// six-coffee day), and ranks are invariant to both — a monotone relationship is +// what "more of this goes with worse recovery" actually claims, and it is all +// these fields can support. +// --------------------------------------------------------------------------- + +/// One day's numeric journal fields, keyed by field name. +/// +/// A field absent from [values] means NOT RECORDED for that day, and is +/// excluded pairwise. It must never be read as a zero: "I logged no caffeine +/// today" and "I did not fill the caffeine field in" are different claims, and +/// treating the second as the first invents a data point at one end of the +/// dose range, which is exactly where a correlation is most sensitive. +class JournalNumericDay { + final String date; + final Map values; + const JournalNumericDay(this.date, this.values); +} + +/// The relationship between one numeric journal field and one outcome series. +class JournalNumericEffect { + final String outcome; + + /// Spearman's rho over the pairwise-complete days. Null when it could not be + /// computed at all (too few days, or no spread on one side). + final double? rho; + + /// Change in the outcome per one unit of the field, by Theil–Sen (median of + /// pairwise slopes, ~29% breakdown). This is the interpretable half — "about + /// 4 ms of RMSSD per extra coffee" — while [rho] carries whether the + /// relationship holds at all. Null under the same conditions as [rho], and + /// deliberately reported in the outcome's own units, unstandardized. + final double? slopePerUnit; + + /// 95% confidence interval on [rho]: Fisher z transform with the Bonett & + /// Wright (2000) rank standard error, sqrt((1 + rho²/2)/(n−3)). Null below 4 + /// pairs, where that standard error is undefined. + final double? rhoLow; + final double? rhoHigh; + + /// Days where both the field and the outcome were present. + final int n; + + /// Not enough paired days, or the field never varied — no verdict either + /// way. Distinct from a computed-but-weak relationship. + final bool insufficient; + + /// Strong enough AND separated from zero to be worth showing: |rho| clears + /// the floor and the confidence interval excludes 0. A rho alone is not + /// evidence — over ten days, |rho| ≈ 0.5 arises constantly from noise. + final bool meaningful; + + const JournalNumericEffect({ + required this.outcome, + required this.rho, + required this.slopePerUnit, + required this.rhoLow, + required this.rhoHigh, + required this.n, + required this.insufficient, + required this.meaningful, + }); +} + +class JournalNumericCorrelation { + final String field; + final List effects; + const JournalNumericCorrelation(this.field, this.effects); +} + +/// Average ranks, 1-based, ties sharing their mean rank. +/// +/// Tie handling is not a detail here: journal fields are full of ties (mood is +/// 1–5, most people log the same 2 coffees most days), and ranking ties +/// arbitrarily would invent an ordering the user never reported. +List _averageRanks(List xs) { + final idx = List.generate(xs.length, (i) => i) + ..sort((a, b) => xs[a].compareTo(xs[b])); + final ranks = List.filled(xs.length, 0); + var i = 0; + while (i < idx.length) { + var j = i; + while (j + 1 < idx.length && xs[idx[j + 1]] == xs[idx[i]]) { + j++; + } + // Ranks are 1-based, so positions i..j map to ranks i+1..j+1. + final shared = (i + 1 + j + 1) / 2.0; + for (var k = i; k <= j; k++) { + ranks[idx[k]] = shared; + } + i = j + 1; + } + return ranks; +} + +/// Pearson correlation. Null when either side has no spread. +double? _pearson(List a, List b) { + if (a.length != b.length || a.length < 2) return null; + final ma = mean(a)!; + final mb = mean(b)!; + var num = 0.0, da = 0.0, db = 0.0; + for (var i = 0; i < a.length; i++) { + final xa = a[i] - ma; + final xb = b[i] - mb; + num += xa * xb; + da += xa * xa; + db += xb * xb; + } + if (da == 0 || db == 0) return null; + return num / math.sqrt(da * db); +} + +/// Spearman's rho — Pearson on average ranks, so ties are handled correctly. +double? spearmanRho(List a, List b) => + _pearson(_averageRanks(a), _averageRanks(b)); + +/// Per-field relationship between numeric journal entries and each outcome. +/// +/// [outcomes] values must be POSITIONALLY ALIGNED to [dates], exactly as in +/// [journalCorrelations]; a series of a different length is reported as +/// insufficient rather than silently truncated. +/// +/// [minN] is the floor on paired days. It is higher than the tag path's +/// requirement because a correlation over a handful of points is close to +/// meaningless — with 5 days, |rho| > 0.8 happens by chance often enough to +/// fill a screen with confident nonsense. +List journalNumericCorrelations({ + required List journal, + required List dates, + required Map> outcomes, + int minN = 8, + double minAbsRho = 0.35, +}) { + final byDate = >{ + for (final d in journal) d.date: d.values, + }; + final fields = {for (final d in journal) ...d.values.keys}.toList() + ..sort(); + + JournalNumericEffect none(String outcome, int n) => JournalNumericEffect( + outcome: outcome, + rho: null, + slopePerUnit: null, + rhoLow: null, + rhoHigh: null, + n: n, + insufficient: true, + meaningful: false, + ); + + final out = []; + for (final field in fields) { + final effects = []; + for (final entry in outcomes.entries) { + final series = entry.value; + if (series.length != dates.length) { + effects.add(none(entry.key, 0)); + continue; + } + + // Pairwise-complete: a day counts only when the field was recorded AND + // the outcome exists for it. + final xs = []; + final ys = []; + for (var i = 0; i < dates.length; i++) { + final v = byDate[dates[i]]?[field]; + final y = series[i]; + if (v == null || y == null) continue; + xs.add(v); + ys.add(y); + } + + final n = xs.length; + final rho = n >= minN ? spearmanRho(xs, ys) : null; + if (rho == null) { + effects.add(none(entry.key, n)); + continue; + } + + // Fisher z CI. atanh diverges at |rho| = 1, which a monotone field hits + // easily — every "more coffee, worse HRV" day in order gives exactly -1. + // Abstaining there would throw away the strongest evidence there is, and + // clamping to 1 - 1e-9 would claim near-infinite confidence from twelve + // days. So the saturated value is pulled in by 1/(2n): the interval + // still excludes zero, but it widens as the sample shrinks, which is the + // honest reading of a perfect correlation over very few days. + double? lo, hi; + if (n > 3) { + final ceiling = 1.0 - 1.0 / (2.0 * n); + final r = rho.clamp(-ceiling, ceiling); + final zr = 0.5 * math.log((1 + r) / (1 - r)); + // Bonett & Wright (2000) standard error, NOT Fisher's 1/sqrt(n−3). + // That one is derived for Pearson's r under bivariate normality; ranks + // are neither, and it runs narrow for rho. Since `meaningful` is gated + // on this interval excluding zero, the narrower SE would let weaker + // relationships through — the error points the wrong way for a + // function whose job is to refuse. + final se = math.sqrt((1.0 + r * r / 2.0) / (n - 3)); + double tanh(double v) { + final e = math.exp(2 * v); + return (e - 1) / (e + 1); + } + + lo = tanh(zr - 1.96 * se); + hi = tanh(zr + 1.96 * se); + } + + final separated = lo != null && hi != null && (lo > 0) == (hi > 0); + effects.add( + JournalNumericEffect( + outcome: entry.key, + rho: rho, + slopePerUnit: theilSen(ys, xs), + rhoLow: lo, + rhoHigh: hi, + n: n, + insufficient: false, + meaningful: rho.abs() >= minAbsRho && separated, + ), + ); + } + out.add(JournalNumericCorrelation(field, effects)); + } + out.sort((a, b) => a.field.compareTo(b.field)); + return out; +} diff --git a/test/onehz/journal_numeric_correlations_test.dart b/test/onehz/journal_numeric_correlations_test.dart new file mode 100644 index 0000000..6378efa --- /dev/null +++ b/test/onehz/journal_numeric_correlations_test.dart @@ -0,0 +1,384 @@ +// Numeric journal fields — Spearman rank correlation against outcome series. +// +// The tag path can only ask "were the tagged days different?". A field that +// carries a dose (three coffees, 700 ml, mood 4/5) needs a statistic that can +// tell one coffee from five, and it has to refuse to answer far more often +// than a difference-of-means does: a correlation over a handful of days is +// close to meaningless, and that is exactly when it looks most convincing. + +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; + +/// [n] consecutive real calendar dates from 2026-01-01. Real ones, because a +/// naive day counter runs past 2026-01-31 and starts emitting dates that do +/// not exist — harmless while the code treats a date as an opaque key, and a +/// baffling failure the day it stops. +List _dates(int n) { + final start = DateTime(2026, 1, 1); + return [ + for (var i = 0; i < n; i++) + () { + final d = start.add(Duration(days: i)); + return '${d.year}-${d.month.toString().padLeft(2, '0')}' + '-${d.day.toString().padLeft(2, '0')}'; + }(), + ]; +} + +/// One field over [values], aligned to `_dates(values.length)`. +List _days(String field, List values) { + final dates = _dates(values.length); + return [ + for (var i = 0; i < values.length; i++) + JournalNumericDay( + dates[i], + values[i] == null ? const {} : {field: values[i]!}, + ), + ]; +} + +JournalNumericEffect _effect( + List out, + String field, + String outcome, +) => out.firstWhere((e) => e.field == field).effects.firstWhere( + (e) => e.outcome == outcome, +); + +void main() { + group('spearmanRho', () { + test('is 1 for any increasing relationship, however curved', () { + // The point of ranks: this is not linear, and Pearson would not say 1. + expect( + spearmanRho([1, 2, 3, 4, 5], [1, 4, 9, 16, 25]), + closeTo(1.0, 1e-12), + ); + }); + + test('is -1 for a decreasing relationship', () { + expect(spearmanRho([1, 2, 3, 4], [9, 7, 5, 1]), closeTo(-1.0, 1e-12)); + }); + + test('handles ties by sharing the mean rank', () { + // Journal fields are full of ties — mood is 1..5 and most days carry the + // same 2 coffees. Ranking ties arbitrarily would invent an order the + // user never reported. + expect(spearmanRho([1, 2, 2, 3], [1, 2, 2, 3]), closeTo(1.0, 1e-12)); + expect(spearmanRho([1, 2, 2, 3], [3, 2, 2, 1]), closeTo(-1.0, 1e-12)); + }); + + test('is null when either side never varies', () { + expect(spearmanRho([2, 2, 2, 2], [1, 2, 3, 4]), isNull); + expect(spearmanRho([1, 2, 3, 4], [5, 5, 5, 5]), isNull); + }); + + test('is null below two points', () { + expect(spearmanRho([1], [2]), isNull); + expect(spearmanRho(const [], const []), isNull); + }); + }); + + group('journalNumericCorrelations', () { + test('finds a strong monotone relationship and signs it correctly', () { + final dates = _dates(12); + final caffeine = [1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6]; + final rmssd = [for (final c in caffeine) 80.0 - 6 * c]; + final out = journalNumericCorrelations( + journal: [ + for (var i = 0; i < dates.length; i++) + JournalNumericDay(dates[i], {'caffeine': caffeine[i]}), + ], + dates: dates, + outcomes: {'rmssd': rmssd}, + ); + + final e = _effect(out, 'caffeine', 'rmssd'); + expect(e.insufficient, isFalse); + expect(e.meaningful, isTrue); + expect(e.rho, closeTo(-1.0, 1e-9)); + expect(e.n, 12); + // The interpretable half: ms of RMSSD per extra coffee, in the outcome's + // own units rather than standardized. + expect(e.slopePerUnit, closeTo(-6.0, 1e-9)); + expect(e.rhoHigh, isNotNull); + expect(e.rhoHigh!, lessThan(0), reason: 'the CI must exclude zero'); + }); + + test('refuses to answer below the paired-day floor', () { + // A perfect correlation over 5 days is a small-sample artefact. It must + // not surface as a finding just because rho happens to be 1. + final dates = _dates(5); + final out = journalNumericCorrelations( + journal: [ + for (var i = 0; i < 5; i++) + JournalNumericDay(dates[i], {'water': (i + 1).toDouble()}), + ], + dates: dates, + outcomes: {'readiness': [for (var i = 0; i < 5; i++) 50.0 + i]}, + ); + final e = _effect(out, 'water', 'readiness'); + expect(e.insufficient, isTrue); + expect(e.meaningful, isFalse); + expect(e.rho, isNull); + expect(e.n, 5); + }); + + test('a missing field is excluded pairwise, never read as zero', () { + // "I did not fill in the caffeine field" is not "I had no caffeine". + // Reading the second for the first invents a point at the bottom of the + // dose range, which is where a correlation is most sensitive. + final dates = _dates(10); + final logged = [3, null, 3, null, 3, null, 3, null, 3, null]; + final out = journalNumericCorrelations( + journal: [ + for (var i = 0; i < dates.length; i++) + JournalNumericDay( + dates[i], + logged[i] == null ? const {} : {'caffeine': logged[i]!}, + ), + ], + dates: dates, + outcomes: {'rmssd': [for (var i = 0; i < 10; i++) 60.0 + i]}, + ); + final e = _effect(out, 'caffeine', 'rmssd'); + expect(e.n, 5, reason: 'only the days the field was actually recorded'); + expect( + e.insufficient, + isTrue, + reason: 'five paired days is below the floor, and a constant field ' + 'has no spread to correlate anyway', + ); + }); + + test('a missing outcome day is excluded pairwise too', () { + final dates = _dates(12); + final out = journalNumericCorrelations( + journal: _days('mood', [ + for (var i = 0; i < 12; i++) (i % 5 + 1).toDouble(), + ]), + dates: dates, + outcomes: { + 'readiness': [ + for (var i = 0; i < 12; i++) i.isEven ? null : 50.0 + i, + ], + }, + ); + expect(_effect(out, 'mood', 'readiness').n, 6); + }); + + test('noise does not become a finding', () { + // A field that wanders independently of the outcome must come back not + // meaningful — the confidence interval straddles zero. + final dates = _dates(20); + const field = [4, 1, 3, 2, 5, 3, 1, 4, 2, 5, + 3, 2, 4, 1, 5, 2, 3, 4, 1, 5]; + // The same twenty outcome values, permuted to a rank correlation of + // exactly 0 against the field above — so this test fails if the gate + // ever starts calling an unrelated field a finding. + const outcome = [53, 47, 50, 49, 49, 51, 49, 48, 51, 50, + 52, 52, 48, 52, 47, 53, 51, 50, 48, 53]; + final out = journalNumericCorrelations( + journal: [ + for (var i = 0; i < dates.length; i++) + JournalNumericDay(dates[i], {'screens': field[i]}), + ], + dates: dates, + outcomes: {'readiness': outcome}, + ); + final e = _effect(out, 'screens', 'readiness'); + expect(e.insufficient, isFalse); + expect(e.meaningful, isFalse); + expect(e.rho!.abs(), lessThan(0.2)); + expect( + e.rhoLow!, + lessThan(0), + reason: 'the interval must straddle zero, which is what makes it ' + 'not a finding', + ); + expect(e.rhoHigh!, greaterThan(0)); + }); + + test('a misaligned outcome series is reported, not truncated or thrown', () { + final dates = _dates(10); + final out = journalNumericCorrelations( + journal: _days('water', [for (var i = 0; i < 10; i++) i.toDouble()]), + dates: dates, + outcomes: {'rhr': const [50.0, 51.0]}, + ); + final e = _effect(out, 'water', 'rhr'); + expect(e.insufficient, isTrue); + expect(e.n, 0); + }); + + test('a field that never varies yields no verdict', () { + final dates = _dates(14); + final out = journalNumericCorrelations( + journal: _days('water', [for (var i = 0; i < 14; i++) 2.0]), + dates: dates, + outcomes: {'readiness': [for (var i = 0; i < 14; i++) 50.0 + i]}, + ); + final e = _effect(out, 'water', 'readiness'); + expect(e.insufficient, isTrue); + expect(e.rho, isNull); + }); + + test('fields come back sorted, and every field sees every outcome', () { + final dates = _dates(10); + final out = journalNumericCorrelations( + journal: [ + for (var i = 0; i < dates.length; i++) + JournalNumericDay(dates[i], { + 'water': i.toDouble(), + 'caffeine': (10 - i).toDouble(), + }), + ], + dates: dates, + outcomes: { + 'rmssd': [for (var i = 0; i < 10; i++) 60.0 + i], + 'rhr': [for (var i = 0; i < 10; i++) 50.0 - i], + }, + ); + expect(out.map((e) => e.field), ['caffeine', 'water']); + for (final f in out) { + expect(f.effects.map((e) => e.outcome).toSet(), {'rmssd', 'rhr'}); + } + }); + + test('a perfect correlation still widens its interval on few days', () { + // rho saturates at exactly -1 the moment a field moves monotonically + // with an outcome, which happens easily. Abstaining there would discard + // the strongest evidence there is; claiming near-certainty from eight + // days would be the opposite mistake. The interval excludes zero either + // way, but it must be visibly wider on the smaller sample. + List run(int n) { + final dates = _dates(n); + return journalNumericCorrelations( + journal: [ + for (var i = 0; i < n; i++) + JournalNumericDay(dates[i], {'caffeine': i.toDouble()}), + ], + dates: dates, + outcomes: {'rmssd': [for (var i = 0; i < n; i++) 90.0 - 5 * i]}, + ).single.effects; + } + + final small = run(8).single; + final large = run(40).single; + expect(small.rho, closeTo(-1.0, 1e-9)); + expect(large.rho, closeTo(-1.0, 1e-9)); + expect(small.meaningful, isTrue); + expect(large.meaningful, isTrue); + expect( + small.rhoHigh!, + greaterThan(large.rhoHigh!), + reason: 'eight days must not claim the confidence of forty', + ); + expect(small.rhoHigh!, lessThan(0), reason: 'still excludes zero'); + }); + + test('the strength floor and the interval gate are separate', () { + // Both have to pass. Raising the floor above a computed rho must turn + // meaningful off WITHOUT claiming the relationship was uncomputable — + // "too weak to mention" and "not enough evidence" are different answers + // and the caller may want to phrase them differently. + final dates = _dates(30); + final journal = [ + for (var i = 0; i < 30; i++) + JournalNumericDay(dates[i], {'water': (i % 7).toDouble()}), + ]; + final outcomes = { + 'readiness': [ + for (var i = 0; i < 30; i++) 50.0 + (i % 7) * 1.5 + (i % 3), + ], + }; + + final permissive = journalNumericCorrelations( + journal: journal, + dates: dates, + outcomes: outcomes, + ).single.effects.single; + expect(permissive.insufficient, isFalse); + expect(permissive.meaningful, isTrue); + + final strict = journalNumericCorrelations( + journal: journal, + dates: dates, + outcomes: outcomes, + minAbsRho: permissive.rho!.abs() + 0.05, + ).single.effects.single; + expect(strict.rho, permissive.rho, reason: 'the statistic is unchanged'); + expect(strict.insufficient, isFalse, reason: 'it was computable'); + expect(strict.meaningful, isFalse, reason: 'just below the floor'); + }); + + test('minN gates the statistic, the interval keeps its own n > 3 rule', () { + final dates = _dates(5); + final journal = [ + for (var i = 0; i < 5; i++) + JournalNumericDay(dates[i], {'water': i.toDouble()}), + ]; + final outcomes = { + 'readiness': [for (var i = 0; i < 5; i++) 50.0 + i], + }; + + // Default floor of 8 refuses five days outright. + expect( + journalNumericCorrelations( + journal: journal, + dates: dates, + outcomes: outcomes, + ).single.effects.single.rho, + isNull, + ); + + // Lowered below the pair count, rho is computed and — because 5 > 3 — + // still carries an interval. + final e = journalNumericCorrelations( + journal: journal, + dates: dates, + outcomes: outcomes, + minN: 5, + ).single.effects.single; + expect(e.rho, closeTo(1.0, 1e-9)); + expect(e.n, 5); + expect(e.rhoLow, isNotNull); + expect(e.rhoHigh, isNotNull); + + // Four pairs is where the interval gets absurdly wide but still exists + // — and being unable to exclude zero is exactly the right answer there. + expect(e.rhoLow!, lessThan(0), reason: 'five days cannot clear zero'); + expect(e.meaningful, isFalse); + + // At three the standard error is undefined outright, so there is no + // interval at all and therefore no verdict. + final three = _dates(3); + final e3 = journalNumericCorrelations( + journal: [ + for (var i = 0; i < 3; i++) + JournalNumericDay(three[i], {'water': i.toDouble()}), + ], + dates: three, + outcomes: {'readiness': [for (var i = 0; i < 3; i++) 50.0 + i]}, + minN: 3, + ).single.effects.single; + expect(e3.rho, closeTo(1.0, 1e-9)); + expect(e3.rhoLow, isNull, reason: 'n > 3 is required for the SE'); + expect( + e3.meaningful, + isFalse, + reason: 'no interval means no evidence it clears zero', + ); + }); + + test('empty input is empty output, not a crash', () { + expect( + journalNumericCorrelations( + journal: const [], + dates: const [], + outcomes: const {}, + ), + isEmpty, + ); + }); + }); +}