fix(types): refuse NaN and out-of-range probabilities; stop the Brier floor indexing past a distribution

CHANGELOG.md

@@ -83,6 +83,12 @@ different event from one that moved because it was wrong.
 
 ### Fixed
 
+- A prediction whose distribution held NaN, a negative, or a value above 1 was
+  accepted as long as the entries summed to about 1, and a NaN latency was
+  accepted too (#32); each is now refused with the option it concerns. The
+  multiclass Brier floor raised `IndexError` on a distribution summing to a
+  little under 1, which `Prediction` allows (#31); it now draws from the
+  distribution as reported, and one that sums to 1 draws exactly as before.
 - The pricing template priced every call at $0, so a copy used unedited let
   any run past `--max-cost-usd` (#28). Its prices are now null and its `as_of`
   a `YYYY-MM-DD` placeholder that the loader refuses, and any entry pricing

src/plumbline/metrics/calibration.py

@@ -561,8 +561,15 @@ def multiclass_brier_floor(
     totals = np.zeros(n_boot, dtype=np.float64)
     for distribution in distributions:
         values = np.asarray(list(distribution.values()), dtype=np.float64)
+        # A reported distribution may sum to slightly less than 1 (Prediction
+        # allows 1e-3), and a uniform draw above that sum would index past the
+        # last option. Normalizing the cumulative sum, and clamping for the
+        # float residue, draws from the distribution as reported; one that
+        # already sums to 1 draws exactly as before.
+        cumulative = np.cumsum(values)
+        index = np.searchsorted(cumulative / cumulative[-1], rng.random(n_boot))
+        drawn = values[np.minimum(index, len(values) - 1)]
         # sum_k (p_k - y_k)^2 collapses to sum_k p_k^2 + 1 - 2 p_gold.
-        drawn = values[np.searchsorted(np.cumsum(values), rng.random(n_boot))]
         totals += float((values**2).sum()) + 1.0 - 2.0 * drawn
 
     draws = totals / len(distributions)

src/plumbline/types.py

@@ -232,9 +232,17 @@ def __post_init__(self) -> None:
             value = getattr(self, name)
             if value is not None and not 0.0 <= value <= 1.0:
                 raise ValueError(f"{name} must lie in [0, 1], got {value!r}")
-        if self.latency_ms < 0:
-            raise ValueError(f"latency_ms must be non-negative, got {self.latency_ms!r}")
+        if not math.isfinite(self.latency_ms) or self.latency_ms < 0:
+            raise ValueError(f"latency_ms must be finite and non-negative, got {self.latency_ms!r}")
         if self.distribution is not None:
+            # Checked one by one: NaN compares false with everything, and a
+            # negative entry can hide behind a sum that still comes to 1.
+            for option, probability in self.distribution.items():
+                if not (math.isfinite(probability) and 0.0 <= probability <= 1.0):
+                    raise ValueError(
+                        f"distribution gives {option!r} a probability of {probability!r}, "
+                        "which is not a finite number in [0, 1]"
+                    )
             if self.label not in self.distribution:
                 raise ValueError(
                     f"label {self.label!r} is missing from distribution "

tests/test_calibration.py

@@ -387,3 +387,9 @@ def test_a_multiclass_brier_figure_is_read_against_its_own_null() -> None:
     )
     assert "300 rows" in figure.statement()
     assert not figure.is_distinguishable
+
+
+def test_the_brier_floor_accepts_a_distribution_a_little_under_one() -> None:
+    """Prediction allows a sum within 1e-3 of 1; the floor must not index past it (#31)."""
+    floor = calibration.multiclass_brier_floor([{"a": 0.4995, "b": 0.5}] * 50, n_boot=500)
+    assert 0.0 <= floor.mean <= 2.0

tests/test_types.py

@@ -115,3 +115,26 @@ def test_confidence_normalizes_for_option_count_and_prob_selected_does_not() ->
 def test_confidence_needs_at_least_two_options() -> None:
     with pytest.raises(ValueError, match="at least 2 options"):
         docs_confidence({"a": 1.0})
+
+
+@pytest.mark.parametrize(
+    "distribution",
+    [
+        {"a": float("nan"), "b": 1.0},
+        {"a": 1.5, "b": -0.5},
+        {"a": float("inf"), "b": 0.0},
+    ],
+    ids=["nan", "negative", "inf"],
+)
+def test_every_probability_in_a_distribution_is_finite_and_in_unit_range(
+    distribution: dict[str, float],
+) -> None:
+    """A sum near 1 is not enough: NaN and a compensating negative both pass it (#32)."""
+    with pytest.raises(ValueError, match="distribution"):
+        a_prediction(label="a", prob_selected=0.5, confidence=None, distribution=distribution)
+
+
+@pytest.mark.parametrize("latency", [float("nan"), float("inf")])
+def test_a_latency_must_be_finite(latency: float) -> None:
+    with pytest.raises(ValueError, match="latency_ms"):
+        a_prediction(latency_ms=latency)