research(conversion): test which pairing window to train on

research/conversion-model/REPORT.md

@@ -511,6 +511,19 @@ The slowest decile converts too fast in both holdouts, and in the regional holdo
 
 MAE rises with the window for every method, consistent with more change in level between swims further apart. The ordering of methods is the same in all three windows.
 
+That table scores the 30-day model on wider test pairs; it does not show which window to train on. [`window_experiment.py`](./window_experiment.py) fits one model per training window, with the same partitions, noise estimates and shrinkage, and scores every model on the same 14-day test pairs, where A3 is most plausible (receipt [`window-experiment-01.json`](./window-experiment-01.json); differences are swimmer-clustered bootstrap estimates with 95% intervals, positive meaning worse than `30d`):
+
+<!-- receipt:window-training -->
+| Training window | Training swimmers | Low-evidence cells | MAE, 14-day test | vs `30d`, 14-day test | vs `30d`, SCM pairs | vs `30d`, 30-day test |
+| :--- | ---: | ---: | ---: | ---: | ---: | ---: |
+| `14d` | 14,470 | 43 (40 SCM) | `2.4992` | `−0.010` `[−0.020, 0.000]` | `+0.011` `[−0.015, +0.039]` | `+0.008` `[0.000, +0.016]` |
+| `30d` (main) | 22,425 | 30 (28 SCM) | `2.5095` | — | — | — |
+| `60d` | 35,334 | 28 (28 SCM) | `2.5692` | `+0.060` `[+0.046, +0.073]` | `+0.036` `[+0.001, +0.071]` | `+0.023` `[+0.013, +0.032]` |
+| `90d` | 45,507 | 25 (25 SCM) | `2.5134` | `+0.004` `[−0.005, +0.013]` | `+0.067` `[+0.014, +0.121]` | `+0.005` `[−0.002, +0.012]` |
+<!-- /receipt:window-training -->
+
+Sixty days is worse on both test sets. Ninety days ties overall but is worse on pairs involving short-course meters, and wider windows clear few low-evidence SCM cells, so widening only the thin SCM cells would not help. Fourteen days ties `30d` (the sign flips between test sets) with a third fewer training swimmers and more low-evidence cells. Thirty days stays. Beyond 30 days most SCY–LCM pairs have the LCM swim later (about 62% at 60 and 90 days, against 48% at 30), but the SCY→LCM ratio rises to 60 days and falls back at 90, so swim order alone does not explain the drift. The intervals hold each fitted model fixed and are not adjusted for the number of comparisons.
+
 ### 7.9 Venue Cohorts (descriptive)
 
 `same_matched_facility` and `different_matched_facility` compare the facility record each meet's locality resolved to (Appendix B), a proxy rather than a verified building. `altitude_proxy_meet_involved` flags a pair when either meet is in a mountain state, a listed high-elevation city or postal prefix, declares a `B1` altitude of at least 3,000 ft, or comes from the Utah or Montana archive. The cohorts differ in swimmers, events and pace, so these differences are descriptive, not measured effects of travel or altitude.

research/conversion-model/render_report_tables.py

@@ -23,6 +23,8 @@
 EVALUATION = HERE / "evaluation-checkpoint-08.json"
 ACQUISITION = HERE / "acquisition-checkpoint-06.json"
 SNAPSHOT = HERE.parents[1] / "data" / "empirical.json"
+# A separate experiment, read by its own renderer rather than passed to every table.
+WINDOW_EXPERIMENT = HERE / "window-experiment-01.json"
 
 BLOCK = re.compile(r"(<!-- receipt:([a-z0-9-]+) -->\n)(.*?)(<!-- /receipt:\2 -->)", re.DOTALL)
 MINUS = "−"
@@ -511,6 +513,36 @@ def window(ev: dict, acq: dict, snap: dict) -> str:
     )
 
 
+def window_training(*_docs: dict) -> str:
+    """One model fitted per training window, every model scored on the same 14-day test pairs."""
+    exp = json.loads(WINDOW_EXPERIMENT.read_text(encoding="utf-8"))
+    fixed_test, wider_test = exp["evaluations"]["test_14d"], exp["evaluations"]["test_30d"]
+
+    def delta(evaluation: dict, scope: str, name: str) -> str:
+        if name == "equivalence_30d":
+            return "—"
+        entry = evaluation["bootstrap_mae_swimmer_clustered"][scope]["mae_minus_equivalence_30d"][name]
+        return f"{code(signed(entry['estimate'], 3))} {ci_only(entry, 3)}"
+
+    rows = []
+    for window, train in exp["training_sets"].items():
+        name = f"equivalence_{window}"
+        rows.append([
+            code(window) + (" (main)" if window == "30d" else ""),
+            count(train["clean_swimmers"]),
+            f"{train['low_evidence_course_pair_cells']} ({train['scm_course_pair_cells_low_evidence']} SCM)",
+            code(fixed(fixed_test["methods"][name]["overall"]["mae_seconds"], 4)),
+            delta(fixed_test, "overall", name),
+            delta(fixed_test, "scm_involving", name),
+            delta(wider_test, "overall", name),
+        ])
+    return table(
+        ["Training window", "Training swimmers", "Low-evidence cells", "MAE, 14-day test", "vs `30d`, 14-day test", "vs `30d`, SCM pairs", "vs `30d`, 30-day test"],
+        [":---"] + ["---:"] * 6,
+        rows,
+    )
+
+
 def caveats(ev: dict, acq: dict, snap: dict) -> str:
     shares = ev["swimmer_disjoint_holdout"]["caveat_shares_on_test"]
     meta = ev["model_metadata"]
@@ -617,6 +649,7 @@ def lakehouse(ev: dict, acq: dict, snap: dict) -> str:
     "holdouts": holdouts,
     "holdout-diagnostics": holdout_diagnostics,
     "window": window,
+    "window-training": window_training,
     "caveats": caveats,
     "venue-cohorts": venue_cohorts,
     "fast-swimmer": fast_swimmer,

research/conversion-model/test_window_experiment.py

@@ -0,0 +1,54 @@
+"""Unit tests for the pairing-window experiment's own logic, on invented rows."""
+
+from __future__ import annotations
+
+import math
+import unittest
+from dataclasses import replace
+
+from window_experiment import abs_errors_s, probe_pair, seasonal_order
+
+
+def row(src: str, dst: str, signed_gap: int, outlier: bool = False, target_cs: int = 6672):
+    return replace(
+        probe_pair(5832, 100, "freestyle", "women", "100-FR", src, dst),
+        signed_gap_days=signed_gap,
+        gap_days=abs(signed_gap),
+        target_time_cs=target_cs,
+        is_ratio_outlier=outlier,
+    )
+
+
+class SeasonalOrderTest(unittest.TestCase):
+    def test_counts_each_pair_once_in_scy_to_lcm_orientation(self):
+        pairs = [
+            row("SCY", "LCM", 10),
+            row("LCM", "SCY", -10),  # reverse of the first: ignored
+            row("SCY", "LCM", -4),
+            row("SCY", "LCM", 0),
+            row("SCY", "LCM", 30),
+            row("SCY", "LCM", 50, outlier=True),  # screened out
+            row("SCY", "SCM", 20),  # other course pair
+        ]
+        out = seasonal_order(pairs)
+        self.assertEqual(out["pairs"], 4)
+        self.assertEqual(out["share_LCM_later"], 0.5)
+        self.assertEqual(out["share_LCM_earlier"], 0.25)
+        self.assertEqual(out["share_same_day"], 0.25)
+        self.assertEqual(out["mean_signed_gap_days_LCM_minus_SCY"], 9.0)
+        self.assertEqual(out["mean_abs_gap_days"], 11.0)
+
+    def test_empty(self):
+        self.assertEqual(seasonal_order([]), {"pairs": 0})
+
+
+class AbsErrorsTest(unittest.TestCase):
+    def test_missing_prediction_is_nan_not_scored(self):
+        pairs = [row("SCY", "LCM", 1, target_cs=6672), row("SCY", "LCM", 1, target_cs=6700)]
+        errs = abs_errors_s(pairs, lambda p: 6600 if p.target_time_cs == 6672 else None)
+        self.assertAlmostEqual(errs[0], 0.72)
+        self.assertTrue(math.isnan(errs[1]))
+
+
+if __name__ == "__main__":
+    unittest.main()