From 8df07cefd793bf97dc75dd539dbb3f8950fbd559 Mon Sep 17 00:00:00 2001 From: TMHSDigital <154358121+TMHSDigital@users.noreply.github.com> Date: Fri, 25 Sep 2026 07:13:10 -0400 Subject: [PATCH] feat(report): state the grid probabilities arrived on, and show it costs the floor nothing Hosted Jev rounds to two decimals. The report now says so at the point of use: when every probability an arm returned sits on a grid of 0.001 or coarser, over at least 20 values, a Resolution line names it, and warns when the bins are narrower than the grid. scripts/quantization_floor.py answers whether the rounding raises the ECE floor. Across 40 to 10,000 rows, 10 or 20 bins, and grids of 0.01 or 0.05, a calibrated model reported on the grid clears the floor's 95th percentile at the nominal 5 percent, so the floor needs no correction. METHODOLOGY carries the table. Part of #9: whether the grid varies by model or option count needs live runs. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 8 +++++ METHODOLOGY.md | 37 +++++++++++++++++++ scripts/quantization_floor.py | 44 +++++++++++++++++++++++ src/plumbline/metrics/calibration.py | 23 ++++++++++++ src/plumbline/report/markdown.py | 28 +++++++++++++++ tests/test_probability_grid.py | 54 ++++++++++++++++++++++++++++ 6 files changed, 194 insertions(+) create mode 100644 scripts/quantization_floor.py create mode 100644 tests/test_probability_grid.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e74b854..f3814de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,14 @@ different event from one that moved because it was wrong. ### Added +- The report states the grid an arm's probabilities arrived on, when every one + sits on a grid of 0.001 or coarser, and warns when the bins are narrower than + it (#9). Hosted Jev rounds to two decimals, and a reader needs that to know + what a small difference is worth. `scripts/quantization_floor.py` measures + whether the rounding raises the ECE floor: across 40 to 10,000 rows, 10 or 20 + bins, and grids of 0.01 or 0.05, a calibrated model reported on the grid + clears the floor's 95th percentile at the nominal 5 percent, so it does not. + METHODOLOGY carries the table. - A row whose top probability is shared by two or more options is now recorded as one (#10). Each artifact record carries `tied_for_top`, and when any row tied the report says how many beside the accuracy figure, and on how many the diff --git a/METHODOLOGY.md b/METHODOLOGY.md index 2597c6e..f02fe49 100644 --- a/METHODOLOGY.md +++ b/METHODOLOGY.md @@ -330,6 +330,43 @@ for the purpose they exist for. It means a report should not claim resolution its inputs do not have, and it is a fact about the measurement that a reader of these figures needs in order to know what a small difference is worth. +So the report says it at the point of use. When every probability an arm +returned sits on a grid of 0.001 or coarser, over at least 20 values, the arm's +section carries a **Resolution** line naming the grid, and adds a warning when +the bins are narrower than it. + +### The grid does not raise the floor + +The open question was whether rounding costs calibration the floor does not +model, which would read every figure through this transport against a floor +slightly too low. It does not, at any size this tool reports on. + +`scripts/quantization_floor.py` simulates a perfectly calibrated model whose +true probabilities are continuous, sends them rounded to the grid, and reads +ECE on the rounded values against `calibration_floor` of those same values, as +the report does. If rounding cost calibration, the calibrated model would clear +the floor's 95th percentile more often than 5 percent of the time. Over 200 +trials per row, so a rate within about three points of 5 percent is noise: + +| rows | bins | grid | mean ECE | floor mean | above p95 | +|---|---|---|---|---|---| +| 40 | 10 | 0.01 | 0.1141 | 0.1149 | 5.5% | +| 105 | 10 | 0.01 | 0.0743 | 0.0728 | 6.0% | +| 500 | 10 | 0.01 | 0.0345 | 0.0339 | 5.5% | +| 2,000 | 10 | 0.01 | 0.0164 | 0.0171 | 5.5% | +| 10,000 | 10 | 0.01 | 0.0074 | 0.0076 | 2.5% | +| 105 | 10 | 0.05 | 0.0727 | 0.0714 | 6.0% | +| 10,000 | 10 | 0.05 | 0.0080 | 0.0075 | 6.5% | +| 105 | 20 | 0.01 | 0.1002 | 0.0994 | 7.0% | +| 10,000 | 20 | 0.01 | 0.0106 | 0.0107 | 4.5% | + +The measured ECE and the floor agree to the third decimal throughout, and the +rate stays at the nominal 5 percent, even on a grid five times coarser than the +vendor's. The reason is that the floor is computed from the rounded values +themselves: rounding moves each probability by at most half a step, in both +directions, and within a bin those errors cancel rather than accumulate. What +the grid bounds is resolution, above; it does not bias the verdict. + ## The three probability_semantics classes Every adapter declares what kind of number it reports, and the report groups on diff --git a/scripts/quantization_floor.py b/scripts/quantization_floor.py new file mode 100644 index 0000000..e51b59b --- /dev/null +++ b/scripts/quantization_floor.py @@ -0,0 +1,44 @@ +"""Does a two-decimal grid raise the ECE floor? (issue #9) + +A perfectly calibrated model has true probabilities p; the vendor sends +q = round(p, 2). plumbline reads ECE(q, outcomes) against calibration_floor(q), +the floor for a model calibrated at q. If rounding costs calibration the floor +does not model, a calibrated model reported on the grid would clear the floor's +95th percentile more often than the nominal 5 percent. + + uv run python scripts/quantization_floor.py [trials] + +METHODOLOGY quotes the table this prints at 200 trials. It takes several +minutes, since every trial bootstraps its own floor. +""" + +import sys + +import numpy as np + +from plumbline.metrics.calibration import calibration_floor, ece +from plumbline.types import ProbabilitySeries + +TRIALS = int(sys.argv[1]) if len(sys.argv) > 1 else 200 +rng = np.random.default_rng(20260925) +print(f"{'n':>6} {'bins':>4} {'grid':>5} {'mean ECE':>9} {'floor mean':>10} {'over p95':>9}") +for n_bins, grid in [(10, 0.01), (10, 0.05), (20, 0.01)]: + for n in (40, 105, 500, 2000, 10000): + measured, means, over = [], [], 0 + for trial in range(TRIALS): + p = np.clip(rng.beta(0.8 * 6, 0.2 * 6, size=n), 0.0, 1.0) + outcomes = rng.random(n) < p + q = np.clip(np.round(p / grid) * grid, 0.0, 1.0) + series = ProbabilitySeries( + values=tuple(float(v) for v in q), semantics="calibrated_claim" + ) + value = ece(series, list(outcomes), n_bins=n_bins) + band = calibration_floor(series, n_bins=n_bins, n_boot=400, seed=trial)["ece"] + measured.append(value) + means.append(band.mean) + over += value > band.p95 + print( + f"{n:>6} {n_bins:>4} {grid:>5} {np.mean(measured):>9.4f} {np.mean(means):>10.4f} " + f"{over / TRIALS:>9.1%}", + flush=True, + ) diff --git a/src/plumbline/metrics/calibration.py b/src/plumbline/metrics/calibration.py index 89bfa7f..5774fef 100644 --- a/src/plumbline/metrics/calibration.py +++ b/src/plumbline/metrics/calibration.py @@ -303,6 +303,29 @@ def multiclass_brier( return total / len(distributions) +#: Grids a vendor might round to, coarsest first. +_GRIDS = (0.1, 0.05, 0.01, 0.005, 0.001) + +#: Below this many values, landing on a grid says little about the vendor. +MIN_GRID_VALUES = 20 + + +def probability_grid(values: Sequence[float]) -> float | None: + """The coarsest grid every value sits on, or None when there is none to see. + + Hosted Jev returns probabilities as multiples of 0.01. That bounds what any + figure computed from them can resolve: no bin or threshold finer than the + grid means anything, and ties for the top option become ordinary. None when + the values are finer than 0.001, or too few to say. + """ + if len(values) < MIN_GRID_VALUES: + return None + for step in _GRIDS: + if all(abs(value / step - round(value / step)) < 1e-6 for value in values): + return step + return None + + def calibration_floor( series: ProbabilitySeries, n_bins: int = DEFAULT_N_BINS, diff --git a/src/plumbline/report/markdown.py b/src/plumbline/report/markdown.py index 50edba1..a801160 100644 --- a/src/plumbline/report/markdown.py +++ b/src/plumbline/report/markdown.py @@ -246,6 +246,7 @@ def _arm(result: RunResult, options: ReportOptions, heading: str) -> list[str]: f"- **Excluded**: {excluded} rows of an unsupported question type were not scored." ) lines.extend(_asked_as(scoreable)) + lines.extend(_resolution_lines(scoreable, options)) if not successes: # One shared reason is almost always an install or setup step (a missing @@ -690,6 +691,33 @@ def _confidence_lines( return [f"- **Confidence**: {figure.statement()}"] +def _resolution_lines(records: Sequence[CaseRecord], options: ReportOptions) -> list[str]: + """The grid the probabilities arrived on, when they arrived on one.""" + values: list[float] = [] + for record in records: + prediction = record.prediction + if prediction is None: + continue + if prediction.distribution: + values.extend(prediction.distribution.values()) + elif prediction.prob_selected is not None: + values.append(prediction.prob_selected) + step = calibration.probability_grid(values) + if step is None: + return [] + line = ( + f"- **Resolution**: all {len(values)} probabilities this arm returned are multiples " + f"of {step:g}, so no bin or threshold finer than {step:g} can mean anything, and a " + "tie for the top option is ordinary rather than rare." + ) + if options.n_bins * step > 1: + line += ( + f" At {options.n_bins} bins each bin is narrower than the grid, so the binning " + "measures the rounding rather than the model." + ) + return [line] + + def _tie_lines(successes: Sequence[CaseRecord]) -> list[str]: """How many answers the vendor's tie-break chose, and how many went against gold.""" tied = [record for record in successes if record.prediction and record.prediction.tied_for_top] diff --git a/tests/test_probability_grid.py b/tests/test_probability_grid.py new file mode 100644 index 0000000..82863b7 --- /dev/null +++ b/tests/test_probability_grid.py @@ -0,0 +1,54 @@ +"""The report states the grid probabilities arrived on (issue #9). + +Hosted Jev rounds to two decimals. A reader has to know that to know what a +small difference is worth, and should not have to find it in METHODOLOGY. +""" + +from __future__ import annotations + +from plumbline.metrics.calibration import probability_grid +from plumbline.report import markdown +from plumbline.runner import execute +from plumbline.types import Case, Prediction +from tests.test_ties import a_prediction + + +def test_the_coarsest_grid_every_value_sits_on_is_found() -> None: + assert probability_grid([0.25, 0.5, 0.75] * 10) == 0.05 + assert probability_grid([0.23, 0.41, 0.07] * 10) == 0.01 + assert probability_grid([0.231, 0.4, 0.07] * 10) == 0.001 + assert probability_grid([0.2317, 0.4] * 20) is None + + +def test_too_few_values_say_nothing_about_a_grid() -> None: + assert probability_grid([0.5, 0.25]) is None + + +class OnTheGrid: + """Answers on a two-decimal grid, as the hosted vendor does.""" + + name = "grid" + model_requested = "grid-1" + revision = None + probability_semantics = "calibrated_claim" + reports_tokens = False + + @property + def call_params(self) -> dict[str, object]: + return {} + + def classify(self, text: str, labels: list[str], **_kwargs: object) -> Prediction: + index = int(text[1:]) + top = 0.5 + (index % 40) / 100 + return a_prediction({"a": round(top, 2), "b": round(1 - top, 2)}, "a") + + +def test_the_report_names_the_grid_it_observed() -> None: + cases = [Case(id=f"c{i}", text=f"t{i}", labels=("a", "b"), gold_label="a") for i in range(40)] + result = execute.run(OnTheGrid(), cases) # type: ignore[arg-type] + + document = markdown.render([result], options=markdown.ReportOptions(n_boot=50)) + line = next(line for line in document.splitlines() if "**Resolution**" in line) + + assert "multiples of 0.01" in line + assert "80 probabilities" in line