From 0bf84ca6fae250b57ce6f93612616e513ae65bcb Mon Sep 17 00:00:00 2001 From: TMHSDigital <154358121+TMHSDigital@users.noreply.github.com> Date: Fri, 25 Sep 2026 18:33:17 -0400 Subject: [PATCH] feat(recalibration): a temperature per predicted label, when one temperature is the wrong shape One temperature cannot reach a model that is overconfident on one label and honest on the rest, and the global verdict already diagnosed that and stopped. Now, after exactly the two wrong-shape verdicts, the report fits one temperature per predicted label on the same split and judges it by the same rule; a label with under 100 fit rows is left as it came and named. Its block says the temperatures are not comparable with the global one and names which correction to apply. scripts/per_label_study.py measures it on the mock: from about 100 fit rows per label it lands a per-label bias inside the floor where one temperature leaves 1.2 to 3.3 times it, and where one temperature is the right shape the extra parameters cost 5 to 15 percent more held-out ECE. The global verdict's properties move to a shared _Gate, so both fits earn their verdict by one rule. Fixes #4. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 9 + METHODOLOGY.md | 67 +++++ README.md | 7 +- docs/PLAN.md | 13 +- scripts/per_label_study.py | 110 +++++++++ src/plumbline/metrics/recalibration.py | 324 +++++++++++++++++++++++-- src/plumbline/report/markdown.py | 137 ++++++++++- tests/test_per_label_recalibration.py | 137 +++++++++++ tests/test_recalibration.py | 5 +- 9 files changed, 780 insertions(+), 29 deletions(-) create mode 100644 scripts/per_label_study.py create mode 100644 tests/test_per_label_recalibration.py diff --git a/CHANGELOG.md b/CHANGELOG.md index cce4c8b..8adab8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,15 @@ different event from one that moved because it was wrong. ### Added +- A temperature per predicted label, tried only when the global fit is refused + as the wrong shape or stops short of the floor (#4). It uses the global fit's + split and verdict rule, leaves a label with under 100 fit rows as it came and + names it, prints its temperatures in a block of their own that says they are + not comparable with the global one, and names which correction to apply. On + the mock, from about 100 fit rows per label, it lands a per-label bias inside + the floor where one temperature leaves 1.2 to 3.3 times it; where one + temperature is the right shape its extra parameters cost 5 to 15 percent more + held-out ECE. `scripts/per_label_study.py` and METHODOLOGY carry the tables. - `--option-style letter` asks a local checkpoint its options as A, B, C and reads the letter tokens, so options of any length can be scored (#3). The default still reads each option's own token and refuses one that is several diff --git a/METHODOLOGY.md b/METHODOLOGY.md index 3b27b31..2367c6b 100644 --- a/METHODOLOGY.md +++ b/METHODOLOGY.md @@ -483,6 +483,73 @@ fitted on fewer rows carries uncertainty larger than the correction it claims to make, and it arrives looking authoritative. The report says the rule it failed and gives no temperature. +## A temperature per predicted label, when one temperature is the wrong shape + +A model can be well calibrated on most labels and overconfident on one. One +temperature cannot reach that: flattening enough for the skewed label +over-flattens the honest ones, and the global fit's verdict says so, either as a +refusal because the change was no larger than chance or as a partial fit whose +residual stays above the floor. After exactly those two verdicts, and never +otherwise, the report tries the smallest correction that can reach it: one +temperature per predicted label. + +It is keyed on the label the model predicted, because that is all a caller knows +when the correction is applied. Each label's temperature is fitted on the fit +rows the model predicted that label for, in the same form as the global fit, and +every held-out row is scaled by its own label's temperature. Scaling never +changes which label is on top, so a row stays in its label's group. The split is +the global fit's, so both are judged on the same held-out rows, and the verdict +is the same rule: inside the floor is recommended, a material improvement that +stops short is partial, and anything else is refused with no temperature +printed. The report prints the per-label temperatures in a block of their own, +says they are a different correction and not comparable with the global +temperature, and names which of the two to apply. + +A label with fewer than 100 fit rows is not fitted. It keeps its probabilities as +they came, and the report names it with its counts. A temperature fitted on a +handful of rows is the noise of those rows, and a per-label fit has one of those +per label. + +### When it helps, and what it costs + +`scripts/per_label_study.py` measures both on the seeded mock (accuracy 0.75, +four options, five seeds). Each cell is post-scaling ECE on the held-out half +over the floor's 95th percentile, so 1.0 or below is inside the floor, with how +often each verdict would ship a correction: + +| rows | fit rows per label | one temperature | per label | per label ships | +|---|---|---|---|---| +| 500 | 49 | 1.05x | not fitted | 0% | +| 1,000 | 103 | 1.22x | 0.77x | 60% | +| 2,000 | 216 | 1.93x | 0.83x | 100% | +| 4,000 | 461 | 2.09x | 0.86x | 100% | +| 8,000 | 952 | 3.30x | 0.67x | 100% | + +That is a per-label bias: the model sharpened by a temperature of 0.45 whenever +it says billing, and honest otherwise. From about 100 fit rows per label the +per-label fit lands inside the floor, while the global fit falls further behind +as rows are added, the same pattern as the top-line penalty above: a fixed bias +that a shrinking floor exposes. + +The cost is measured where one temperature is the right shape: every answer +sharpened by 0.5. There the four extra parameters can only add noise, and they +leave 5 to 15 percent more ECE on the held-out rows than one temperature does +(0.77x against 0.72x at 1,000 rows, 0.67x against 0.58x at 8,000), all inside the +floor. The report never pays that cost, since it tries per-label only after one +temperature was diagnosed as the wrong shape. + +The row gate is set more strictly than the mock requires. At 500 rows with the +gate lowered to 40, the per-label fit still reached 0.69x, and the verdict +shipped it on only two of five seeds. But the mock is per-label scaling's best +case, since its distortion is exactly a per-label temperature and a real model's +is not. So the default waits for 100 rows a label, where the result above is not +in doubt. + +What it does not do: vector or matrix scaling, which fit more parameters than +most datasets here can support; a correction keyed on the gold label, which no +caller knows at inference time; or feeding the per-label temperatures into the +cascade, which still scores on the global temperature when one was emitted. + ## The binary Brier formulation The Brier score plumbline reports by default is diff --git a/README.md b/README.md index 88913ef..14d8635 100644 --- a/README.md +++ b/README.md @@ -350,9 +350,10 @@ Specific, and none of them are going to surprise you later. - **One request per case, no batching.** Cost and latency figures are therefore conservative relative to batched use, where a single call carrying many questions against one shared state is materially cheaper and faster. -- **Temperature scaling only.** Per-label and vector scaling are not fitted. When - the residual says temperature is the wrong correction, the tool refuses and - emits no temperature rather than returning one that does not fit. +- **Temperature scaling, globally or per predicted label.** When one + temperature is the wrong shape, the report tries one per predicted label, and + refuses both rather than return a correction that does not fit. Vector and + matrix scaling are not fitted. - **Recalibration needs 200 held-out rows.** Below that it refuses. Most datasets people try first will not reach it. - **Cost requires a pricing table you supply.** plumbline ships no figures for diff --git a/docs/PLAN.md b/docs/PLAN.md index b180dd5..60699d9 100644 --- a/docs/PLAN.md +++ b/docs/PLAN.md @@ -68,12 +68,13 @@ All three items are done. Kept here because the answers matter, not the list. packing many questions against one shared state in a single call, which is a materially different cost and latency profile and is the single largest measurement gap in v0.1. -- **Per-label and vector scaling.** Only temperature is fitted. When the - residual says temperature is the wrong correction the tool refuses, which is - right but leaves the user with nothing to apply. -- **Adapters do not receive `label_descriptions`**, so a dataset's per-option - criteria never reach the wire. A Choice takes them directly and a Noul takes - true/false descriptions. +- **Per-label scaling.** Landed (#4): after a global fit refused as the wrong + shape or stopped short of the floor, a temperature per predicted label, on the + same split and verdict rule, with a 100-row gate per label. Vector and matrix + scaling stay out until per-label proves insufficient on real data. +- **Option descriptions on the wire.** Landed in v0.1.1 (#39): `typesafe_wire` + sends a Choice's descriptions as its criteria, they join the cache key and the + dataset hash, and the report says when an adapter does not send them. ## Decisions diff --git a/scripts/per_label_study.py b/scripts/per_label_study.py new file mode 100644 index 0000000..500a9c0 --- /dev/null +++ b/scripts/per_label_study.py @@ -0,0 +1,110 @@ +"""When a temperature per predicted label helps, and what it costs (issue #4). + +Two cases on the seeded mock (accuracy 0.75, four options), at several sizes, +over seeds. Every figure is on the held-out half, as the report's are. + +- A per-label bias: the model is sharpened (temperature 0.45) whenever it says + "billing" and honest otherwise. One temperature cannot reach this; the + question is how many rows a label needs before its own temperature can. +- A global skew: every answer sharpened by the same temperature (0.5). Here one + temperature is the right shape, so any held-out loss from fitting four is the + price of the extra parameters, which is what overfitting means here. + +Each cell is post-scaling ECE over the floor's 95th percentile, averaged over +seeds (at or below 1.0 is inside the floor), with how often each fit's verdict +would ship a correction. + + uv run python scripts/per_label_study.py [seeds] + +METHODOLOGY quotes the tables this prints at 5 seeds. +""" + +from __future__ import annotations + +import dataclasses +import sys + +import numpy as np + +from plumbline.adapters.mock import MockAdapter +from plumbline.metrics.recalibration import recalibrate, recalibrate_per_label +from plumbline.runner import execute +from plumbline.types import Case, Prediction, ProbabilitySeries, apply_temperature + +LABELS = ("billing", "returns", "shipping", "other") +SEEDS = int(sys.argv[1]) if len(sys.argv) > 1 else 5 +SIZES = (500, 1000, 2000, 4000, 8000) +BOOT = {"n_boot_ci": 100, "n_boot_floor": 300} + + +def rows(n: int, seed: int) -> list[Case]: + rng = np.random.default_rng(seed) + return [ + Case(id=f"r{i}", text=f"row {seed} {i}", labels=LABELS, gold_label=LABELS[g]) + for i, g in enumerate(rng.integers(0, len(LABELS), size=n)) + ] + + +def sharpen(prediction: Prediction, temperature: float) -> Prediction: + assert prediction.distribution is not None + scaled = apply_temperature(prediction.distribution, temperature) + return dataclasses.replace( + prediction, distribution=scaled, prob_selected=scaled[prediction.label] + ) + + +def measured(n: int, seed: int, case: str) -> tuple[list[Prediction], list[bool], list[str]]: + cases = rows(n, seed) + adapter = MockAdapter( + {c.text: c.gold_label for c in cases}, + accuracy=0.75, + calibration_temperature=0.5 if case == "global" else 1.0, + seed=seed, + ) + result = execute.run(adapter, cases, workers=1) + predictions = [r.prediction for r in result.records if r.prediction is not None] + if case == "per-label": + predictions = [sharpen(p, 0.45) if p.label == "billing" else p for p in predictions] + gold = [r.gold_label for r in result.records if r.prediction is not None] + return predictions, [p.label == g for p, g in zip(predictions, gold, strict=True)], gold + + +def study(n: int, seed: int, case: str) -> tuple[float, float, bool, bool, int]: + predictions, correct, gold = measured(n, seed, case) + series = ProbabilitySeries( + values=tuple(p.prob_selected or 0.0 for p in predictions), semantics="calibrated_claim" + ) + common = {"distributions": [p.distribution for p in predictions], "gold_labels": gold} + one = recalibrate(series, correct, seed=seed, **common, **BOOT) + many = recalibrate_per_label( + series, + correct, + predicted_labels=[p.label for p in predictions], + seed=seed, + **common, + **BOOT, + ) + smallest = min(entry.n_fit for entry in many.labels) + return ( + one.after.ece / one.floor["ece"].p95, + many.after.ece / many.floor["ece"].p95, + one.recommendation != "refused", + many.recommendation != "refused", + smallest, + ) + + +for case in ("per-label", "global"): + print(f"\n{case} skew") + header = ("rows", 6), ("label rows", 10), ("one T", 7), ("per label", 10) + print(" ".join(f"{name:>{width}}" for name, width in header), f"{'ships 1':>8} {'ships K':>8}") + for n in SIZES: + results = [study(n, seed, case) for seed in range(SEEDS)] + one, many, ship_one, ship_many, smallest = ( + np.array(column) for column in zip(*results, strict=True) + ) + print( + f"{n:>6} {int(smallest.min()):>10} {one.mean():>6.2f}x {many.mean():>9.2f}x " + f"{ship_one.mean():>8.0%} {ship_many.mean():>8.0%}", + flush=True, + ) diff --git a/src/plumbline/metrics/recalibration.py b/src/plumbline/metrics/recalibration.py index 0f3420f..5f688d2 100644 --- a/src/plumbline/metrics/recalibration.py +++ b/src/plumbline/metrics/recalibration.py @@ -90,6 +90,11 @@ #: noise of the split itself. DEFAULT_MIN_EVAL_ROWS = 200 +#: Below this many fit rows predicted as a label, that label keeps its +#: probabilities as they came. A temperature fitted on a handful of rows is the +#: noise of those rows, and a per-label fit has one of those per label. +DEFAULT_MIN_LABEL_ROWS = 100 + #: Search bounds for the fit. Wide enough to cover any real miscalibration and #: narrow enough that hitting an edge is a signal that something is wrong. TEMPERATURE_BOUNDS = (0.05, 20.0) @@ -150,24 +155,16 @@ class MetricSet: multiclass_brier: float | None -@dataclass(frozen=True) -class RecalibrationResult: - """Everything the report needs to state what the fit did and did not achieve. +class _Gate: + """The verdict a fitted correction earns, whatever its shape. - The headline is :attr:`recommendation`, not :attr:`temperature`. A fitted - temperature with a tight interval clear of 1.0 can still make calibration - worse, which is exactly what a per-label bias produces, so a clear interval - is not permission to ship a number. + Shared by the global and the per-label fit, so both are judged by one rule: + on the held-out rows, against the floor, and refused unless the change is + larger than this sample size produces by chance. """ - method: Method - semantics: ProbabilitySemantics - had_distribution: bool - temperature: float - temperature_ci: tuple[float, float] - justified: bool split: Split - seed: int + justified: bool before: MetricSet after: MetricSet floor: dict[str, FloorBand] @@ -251,6 +248,30 @@ def recommendation(self) -> Recommendation: return "partial" return "refused" + +@dataclass(frozen=True) +class RecalibrationResult(_Gate): + """Everything the report needs to state what the fit did and did not achieve. + + The headline is :attr:`recommendation`, not :attr:`temperature`. A fitted + temperature with a tight interval clear of 1.0 can still make calibration + worse, which is exactly what a per-label bias produces, so a clear interval + is not permission to ship a number. + """ + + method: Method + semantics: ProbabilitySemantics + had_distribution: bool + temperature: float + temperature_ci: tuple[float, float] + justified: bool + split: Split + seed: int + before: MetricSet + after: MetricSet + floor: dict[str, FloorBand] + floor_before: dict[str, FloorBand] + @property def temperature_to_use(self) -> float | None: """The temperature to put into production code, or None when refused.""" @@ -274,9 +295,9 @@ def _verdict(self) -> str: "at least as likely to hurt as to help. The signature is consistent with " "a per-label bias, where some labels are systematically overconfident and " "others are not. One global parameter cannot reach that, since flattening " - "enough for the skewed labels over-flattens the honest ones. Per-label or " - "vector scaling would be the next thing to try, and plumbline fits " - "neither." + "enough for the skewed labels over-flattens the honest ones. A temperature " + "per predicted label is the next thing to try (recalibrate_per_label); " + "plumbline does not fit vector scaling." ) if reason == "interval_spans_one": return ( @@ -615,3 +636,272 @@ def recalibrate( seed=seed + 3, ), ) + + +@dataclass(frozen=True) +class LabelFit: + """One label's share of a per-label fit.""" + + label: str + n_fit: int + """Fit rows the model predicted this label for.""" + n_eval: int + """Held-out rows the model predicted this label for.""" + temperature: float | None + """None when the label had too few fit rows; its probabilities are left alone.""" + temperature_ci: tuple[float, float] | None + + +@dataclass(frozen=True) +class PerLabelResult(_Gate): + """A temperature per predicted label, judged exactly as the global fit is. + + Each held-out row is scaled by the temperature of the label the model + predicted for it, since that is all a caller knows at inference time. + Scaling never changes which label is on top, so every row stays in its + label's group. A label below the row gate is left as it came, and named. + """ + + method: Method + semantics: ProbabilitySemantics + had_distribution: bool + labels: tuple[LabelFit, ...] + min_label_rows: int + justified: bool + split: Split + seed: int + before: MetricSet + after: MetricSet + floor: dict[str, FloorBand] + floor_before: dict[str, FloorBand] + + @property + def fitted(self) -> tuple[LabelFit, ...]: + return tuple(fit for fit in self.labels if fit.temperature is not None) + + @property + def temperatures_to_use(self) -> dict[str, float] | None: + """Label to temperature for production code, or None when refused. + + A label absent from the mapping keeps its probabilities, temperature 1. + """ + if self.recommendation == "refused": + return None + return {fit.label: fit.temperature for fit in self.fitted if fit.temperature is not None} + + def _verdict(self) -> str: + reason = self.reason + if not self.fitted: + return ( + f"Refused: no label had {self.min_label_rows} fit rows, so no per-label " + "temperature was fitted. A temperature fitted on fewer rows is the noise " + "of those rows. Collect more rows per label." + ) + if reason == "already_calibrated": + return ( + "Refused: calibration is already inside the floor, so there is nothing " + "for a per-label correction to do." + ) + if reason == "no_material_improvement": + return ( + f"Refused: per-label scaling moved ECE from {self.before.ece:.4f} to " + f"{self.after.ece:.4f}, no more than this sample size moves it by chance " + f"(noise scale {self.noise_scale:.4f}). The miscalibration is not a " + "per-label temperature either. No temperatures are emitted." + ) + if reason == "interval_spans_one": + return ( + "Refused: every fitted label's interval spans 1.0, so this sample does " + "not establish that any label needs correcting." + ) + if reason == "residual_above_floor": + return ( + f"Partial: per-label scaling removed {self.improvement:.4f} of ECE and " + f"{self.after.ece:.4f} remains, {self.residual_ratio:.1f} times the floor. " + "Use the temperatures, and know that some miscalibration survives them." + ) + return ( + "Recommended: post-scaling ECE is inside the floor, so a temperature per " + "predicted label accounts for the miscalibration present." + ) + + def summary(self) -> str: + fitted = ", ".join( + f"{fit.label} T = {fit.temperature:.3f}" + for fit in self.fitted + if fit.temperature is not None + ) + left = [fit.label for fit in self.labels if fit.temperature is None] + lines = [ + f"Per-label temperature scaling, {self.method} form, on the same split as the " + f"global fit: {self.split_sizes}.", + f"Fitted {fitted or 'no label'}" + + ( + f"; left as they came, under {self.min_label_rows} fit rows: {', '.join(left)}." + if left + else "." + ), + f"ECE {self.before.ece:.4f} before, {self.after.ece:.4f} after, against a " + f"calibrated-model floor of {self.floor['ece'].mean:.4f} " + f"(95th percentile {self.floor['ece'].p95:.4f}).", + self._verdict(), + ] + return " ".join(lines) + + @property + def split_sizes(self) -> str: + return f"{self.n_fit} fit rows and {self.n_eval} held-out rows (seed {self.seed})" + + +def recalibrate_per_label( + series: ProbabilitySeries, + correct: Sequence[bool], + *, + predicted_labels: Sequence[str], + distributions: Sequence[Mapping[str, float] | None] | None = None, + gold_labels: Sequence[str] | None = None, + split: Split | None = None, + fit_fraction: float = 0.5, + seed: int = 0, + min_eval_rows: int = DEFAULT_MIN_EVAL_ROWS, + min_label_rows: int = DEFAULT_MIN_LABEL_ROWS, + n_bins: int = DEFAULT_N_BINS, + binning: Binning = DEFAULT_BINNING, + min_bin_count: int = DEFAULT_MIN_BIN_COUNT, + n_boot_ci: int = DEFAULT_N_BOOT_CI, + n_boot_floor: int = 800, +) -> PerLabelResult: + """Fit one temperature per predicted label on the global fit's split. + + The fallback for a global fit refused as the wrong shape, not a default. + The split is built exactly as :func:`recalibrate` builds it from the same + seed, so the two are judged on the same held-out rows. ``predicted_labels`` + is the label each row's answer chose, the vendor's pick rather than an + argmax, since on a tie the two differ. + """ + probabilities = series.require_reportable() + if not (len(probabilities) == len(correct) == len(predicted_labels)): + raise ValueError("probabilities, outcomes, and predicted labels must align") + use_multiclass = ( + distributions is not None + and gold_labels is not None + and all(distribution is not None for distribution in distributions) + ) + split = split if split is not None else make_split(len(probabilities), fit_fraction, seed) + if len(split.eval_indices) < min_eval_rows: + raise InsufficientDataError( + f"recalibration needs at least {min_eval_rows} held-out evaluation rows and " + f"this split has {len(split.eval_indices)}." + ) + + complete: list[Mapping[str, float]] = ( + [distribution for distribution in distributions if distribution is not None] + if use_multiclass and distributions is not None + else [] + ) + fits: list[LabelFit] = [] + temperatures: dict[str, float] = {} + for offset, label in enumerate(sorted(set(predicted_labels))): + fit_rows = [index for index in split.fit_indices if predicted_labels[index] == label] + eval_count = sum(1 for index in split.eval_indices if predicted_labels[index] == label) + if len(fit_rows) < min_label_rows: + fits.append(LabelFit(label, len(fit_rows), eval_count, None, None)) + continue + if use_multiclass: + assert gold_labels is not None + label_distributions = [complete[index] for index in fit_rows] + label_gold = [gold_labels[index] for index in fit_rows] + temperature = fit_temperature_multiclass(label_distributions, label_gold) + interval = bootstrap_temperature_ci( + True, label_distributions, label_gold, None, None, n_boot_ci, seed + 11 + offset + ) + else: + label_probabilities = [probabilities[index] for index in fit_rows] + label_correct = [correct[index] for index in fit_rows] + temperature = fit_temperature_binary(label_probabilities, label_correct) + interval = bootstrap_temperature_ci( + False, None, None, label_probabilities, label_correct, n_boot_ci, seed + 11 + offset + ) + temperatures[label] = temperature + fits.append(LabelFit(label, len(fit_rows), eval_count, temperature, interval)) + + eval_rows = split.eval_indices + if use_multiclass: + assert gold_labels is not None + before_distributions: list[Mapping[str, float]] | None = [ + complete[index] for index in eval_rows + ] + scaled = [ + apply_temperature(complete[index], temperatures.get(predicted_labels[index], 1.0)) + for index in eval_rows + ] + after_probabilities = tuple( + distribution[predicted_labels[index]] + for distribution, index in zip(scaled, eval_rows, strict=True) + ) + after_distributions: list[Mapping[str, float]] | None = list(scaled) + eval_gold: list[str] | None = [gold_labels[index] for index in eval_rows] + method: Method = "multiclass" + else: + after_probabilities = tuple( + apply_temperature_binary( + probabilities[index], temperatures.get(predicted_labels[index], 1.0) + ) + for index in eval_rows + ) + before_distributions = after_distributions = eval_gold = None + method = "binary" + + eval_correct = [correct[index] for index in eval_rows] + before_series = ProbabilitySeries( + values=tuple(probabilities[index] for index in eval_rows), semantics=series.semantics + ) + after_series = ProbabilitySeries(values=after_probabilities, semantics=series.semantics) + return PerLabelResult( + method=method, + semantics=series.semantics, + had_distribution=use_multiclass, + labels=tuple(fits), + min_label_rows=min_label_rows, + justified=any( + fit.temperature_ci is not None + and not (fit.temperature_ci[0] <= 1.0 <= fit.temperature_ci[1]) + for fit in fits + ), + split=split, + seed=seed, + before=_metrics( + before_series, + eval_correct, + before_distributions, + eval_gold, + n_bins, + binning, + min_bin_count, + ), + after=_metrics( + after_series, + eval_correct, + after_distributions, + eval_gold, + n_bins, + binning, + min_bin_count, + ), + floor=calibration_floor( + after_series, + n_bins=n_bins, + binning=binning, + min_bin_count=min_bin_count, + n_boot=n_boot_floor, + seed=seed + 2, + ), + floor_before=calibration_floor( + before_series, + n_bins=n_bins, + binning=binning, + min_bin_count=min_bin_count, + n_boot=n_boot_floor, + seed=seed + 3, + ), + ) diff --git a/src/plumbline/report/markdown.py b/src/plumbline/report/markdown.py index be90001..4671598 100644 --- a/src/plumbline/report/markdown.py +++ b/src/plumbline/report/markdown.py @@ -294,6 +294,7 @@ def _arm(result: RunResult, options: ReportOptions, heading: str) -> list[str]: fit = _fit(probabilities, successes, outcomes, options) lines.extend(_recalibration_section(fit, options)) + lines.extend(_per_label_section(fit)) lines.extend(_cascade_section(fit, successes, outcomes, options, result.probability_semantics)) lines.extend(_diagnostics(result, probabilities, successes, outcomes, options)) return lines @@ -305,6 +306,8 @@ class _Fit: result: recalibration.RecalibrationResult | None unavailable: str | None # the reason, when there is no result at all + per_label: recalibration.PerLabelResult | None = None + """The fallback, tried only when the global fit was the wrong shape.""" def _fit( @@ -336,7 +339,30 @@ def _fit( ) except (InsufficientDataError, ValueError) as refused: return _Fit(None, str(refused)) - return _Fit(fitted, None) + if fitted.reason not in _WRONG_SHAPE: + return _Fit(fitted, None) + try: + per_label = recalibration.recalibrate_per_label( + probabilities, + outcomes, + predicted_labels=[record.prediction.label for record in successes if record.prediction], + distributions=distributions, + gold_labels=gold, + fit_fraction=options.fit_fraction, + seed=options.seed, + min_eval_rows=options.min_eval_rows, + n_bins=options.n_bins, + binning=options.binning, + n_boot_floor=options.n_boot, + ) + except (InsufficientDataError, ValueError): + per_label = None + return _Fit(fitted, None, per_label) + + +#: The global verdicts that say one temperature was the wrong shape, and so +#: the ones after which a temperature per predicted label is worth trying. +_WRONG_SHAPE = frozenset({"no_material_improvement", "residual_above_floor"}) #: What a refusal says, with no number in it. A refused fit emits no @@ -405,6 +431,115 @@ def _recalibration_section(fit: _Fit, options: ReportOptions) -> list[str]: return lines +def _per_label_section(fit: _Fit) -> list[str]: + """The fallback's block: which labels were fitted, the verdict, and what to apply.""" + result, per_label = fit.result, fit.per_label + if result is None or per_label is None: + return [] + lines = [ + "", + "#### Per-label fallback", + "", + "- One temperature did not fit, so one temperature per predicted label was fitted " + f"on the same split: {per_label.split_sizes}, {per_label.method} form. These are " + "a different correction from the global temperature, one parameter per label, and " + "are not comparable with it.", + "", + ] + refused = per_label.recommendation == "refused" + if refused: + lines += ["| label | fit rows | held-out rows |", "|---|---|---|"] + lines += [ + f"| {_cell(entry.label)} | {entry.n_fit} | {entry.n_eval} |" + for entry in per_label.labels + ] + else: + lines += [ + "| label | fit rows | held-out rows | T | 95 percent interval |", + "|---|---|---|---|---|", + ] + for entry in per_label.labels: + if entry.temperature is None or entry.temperature_ci is None: + lines.append( + f"| {_cell(entry.label)} | {entry.n_fit} | {entry.n_eval} | not fitted, under " + f"{per_label.min_label_rows} fit rows | |" + ) + else: + low, high = entry.temperature_ci + lines.append( + f"| {_cell(entry.label)} | {entry.n_fit} | {entry.n_eval} | " + f"{entry.temperature:.3f} | [{low:.3f}, {high:.3f}] |" + ) + lines.append("") + + if refused: + # As with the global fit, a refusal carries no number anybody can lift. + lines.append(f"- {_PER_LABEL_REFUSALS.get(per_label.reason, _PER_LABEL_REFUSALS['none'])}") + if not per_label.fitted: + lines[-1] = ( + f"- Refused: no label had {per_label.min_label_rows} fit rows, so no per-label " + "temperature was fitted. Collect more rows per label." + ) + else: + floor = per_label.floor["ece"] + lines.append( + f"- ECE {per_label.before.ece:.4f} before, {per_label.after.ece:.4f} after, " + f"against a calibrated-model floor of {floor.mean:.4f} (95th percentile " + f"{floor.p95:.4f}) on the held-out rows." + ) + lines.append( + "- Recommended: post-scaling ECE is inside the floor." + if per_label.recommendation == "recommended" + else f"- Partial: {per_label.after.ece:.4f} remains, " + f"{per_label.residual_ratio:.1f} times the floor's 95th percentile." + ) + + if not refused and ( + result.recommendation == "refused" or per_label.after.ece < result.after.ece + ): + apply = ( + "- Apply: the per-label temperatures, each to the rows the model predicts that " + "label for, leaving any label not fitted as it came. Not the global temperature." + ) + elif result.recommendation != "refused": + apply = ( + "- Apply: the global temperature above. The per-label fit did not leave less " + "miscalibration behind it." + ) + else: + apply = "- Apply: neither. No correction here recovers this arm's calibration." + lines.append(apply) + lines.append( + "- The cascade below scores on the global temperature when one was emitted, " + "otherwise on the probabilities as they came; it does not use these." + ) + return lines + + +#: A per-label refusal, with no number in it. +_PER_LABEL_REFUSALS = { + "already_calibrated": ( + "Refused: calibration is already inside the floor, so there is nothing for a " + "per-label correction to do." + ), + "no_material_improvement": ( + "Refused: per-label scaling changed ECE by no more than this sample size changes it " + "by chance, so no temperatures are emitted. The miscalibration is not a per-label " + "temperature either." + ), + "interval_spans_one": ( + "Refused: every fitted label's interval spans 1.0, so this sample does not " + "establish that any label needs correcting." + ), + "none": "Refused: no per-label temperature recovers this arm's calibration.", +} + + +def _cell(value: str) -> str: + """A label as a table cell, whatever it contains.""" + return value.replace("|", "\\|") + + def _cascade_section( fit: _Fit, successes: Sequence[CaseRecord], diff --git a/tests/test_per_label_recalibration.py b/tests/test_per_label_recalibration.py new file mode 100644 index 0000000..976bf87 --- /dev/null +++ b/tests/test_per_label_recalibration.py @@ -0,0 +1,137 @@ +"""Per-label temperature scaling, the fallback for a diagnosed per-label bias (#4). + +A model can be well calibrated on most labels and overconfident on one. One +global temperature cannot reach that: flattening enough for the skewed label +over-flattens the honest ones. A temperature per predicted label can, and it +has to earn its place by the same gate as the global fit: fitted on one half, +judged on the other, refused when it does not recover calibration. +""" + +from __future__ import annotations + +import dataclasses + +from plumbline.adapters.mock import MockAdapter +from plumbline.metrics import recalibration +from plumbline.report import markdown +from plumbline.runner import execute +from plumbline.types import Prediction, apply_temperature +from tests.helpers import Measured, per_label_skew, redistort +from tests.test_recalibration import LABELS, N_BOOT_CI, N_BOOT_FLOOR, recalibrate, run_for + + +def per_label(run: Measured, **overrides: object) -> recalibration.PerLabelResult: + options: dict = { + "n_boot_ci": N_BOOT_CI, + "n_boot_floor": N_BOOT_FLOOR, + "distributions": run.distributions, + "gold_labels": run.gold_labels, + } + options.update(overrides) + return recalibration.recalibrate_per_label( + run.probabilities, + run.outcomes, + predicted_labels=[prediction.label for prediction in run.predictions], + **options, + ) + + +def billing_biased(seed: int) -> Measured: + return redistort(run_for(4000, 1.0, seed=seed), per_label_skew(("billing",))) + + +def test_a_per_label_bias_that_defeats_the_global_fit_is_mostly_recovered_per_label() -> None: + """Most of the way, not all of it, and the test says which. + + Conditioning on the predicted label is a selection: the rows a model calls + billing are not a calibrated sample of anything, so the fitted temperature + for billing (about 2.0) falls short of the exact inverse of the injected + skew (2.2). What per-label buys is measured against what the global fit + leaves, on the same held-out rows. + """ + for seed in (1, 2): + run = billing_biased(seed) + global_fit = recalibrate(run) + assert not global_fit.fit_is_complete + + result = per_label(run) + + assert result.recommendation != "refused", result.summary() + assert result.improvement_is_material + assert result.residual_ratio < 1.25, result.summary() + assert result.after.ece < 0.8 * global_fit.after.ece + + +def test_the_skewed_label_gets_the_correction_and_the_honest_ones_do_not() -> None: + result = per_label(billing_biased(1)) + by_label = {fit.label: fit for fit in result.labels} + + billing = by_label["billing"] + assert billing.temperature is not None and billing.temperature > 1.5 + for label in ("returns", "shipping", "other"): + honest = by_label[label] + assert honest.temperature is not None and abs(honest.temperature - 1.0) < 0.25 + + +def test_it_judges_on_the_same_held_out_rows_as_the_global_fit() -> None: + run = billing_biased(1) + + assert per_label(run, seed=3).split == recalibrate(run, seed=3).split + + +def test_a_label_below_the_row_gate_keeps_its_probabilities_and_says_why() -> None: + result = per_label(billing_biased(1), min_label_rows=5000) + + assert all(fit.temperature is None for fit in result.labels) + assert all(fit.n_fit < 5000 for fit in result.labels) + assert result.recommendation == "refused" + assert "5000" in result.summary() + + +def test_an_honest_model_is_not_given_a_per_label_correction() -> None: + # The overfitting check: with nothing per-label to find, several free + # parameters must not manufacture a recommendation. + result = per_label(run_for(4000, 1.0, seed=1)) + + assert result.recommendation == "refused" + + +class BillingOverconfident(MockAdapter): + """The mock, with its answers sharpened whenever it says billing.""" + + def classify(self, text: str, labels: list[str], **kwargs: object) -> Prediction: + answer = super().classify(text, labels, **kwargs) # type: ignore[arg-type] + if answer.label != "billing" or answer.distribution is None: + return answer + sharpened = apply_temperature(answer.distribution, 0.45) + return dataclasses.replace( + answer, distribution=sharpened, prob_selected=sharpened[answer.label] + ) + + +def a_report(adapter_type: type[MockAdapter], temperature: float = 1.0) -> str: + from tests.helpers import gold_by_text, make_cases + + cases = make_cases(4000, labels=LABELS) + adapter = adapter_type( + gold_by_text(cases), seed=1, accuracy=0.75, calibration_temperature=temperature + ) + result = execute.run(adapter, cases, workers=1) + return markdown.render([result], options=markdown.ReportOptions(n_boot=N_BOOT_FLOOR)) + + +def test_the_report_tries_per_label_after_the_global_fit_is_refused() -> None: + document = a_report(BillingOverconfident) + section = document.split("#### Per-label fallback", 1) + + assert len(section) == 2, "no per-label block after a per-label bias" + block = section[1].split("\n#### ", 1)[0] + assert "| billing |" in block + assert "Apply" in block + assert "not comparable" in block + + +def test_the_report_does_not_try_per_label_when_one_temperature_is_the_right_shape() -> None: + document = a_report(MockAdapter, temperature=0.5) + + assert "#### Per-label fallback" not in document diff --git a/tests/test_recalibration.py b/tests/test_recalibration.py index 03de60a..f0a26f2 100644 --- a/tests/test_recalibration.py +++ b/tests/test_recalibration.py @@ -369,8 +369,9 @@ def test_a_per_label_bias_is_refused_on_the_seeds_where_scaling_degrades() -> No def test_the_refusal_names_a_next_step_rather_than_stopping() -> None: run = redistort(run_for(4000, 1.0, seed=2), per_label_skew(("billing",))) summary = recalibrate(run, seed=0).summary() - assert "vector scaling" in summary - assert "plumbline fits neither" in summary + assert "per predicted label" in summary + assert "recalibrate_per_label" in summary + assert "does not fit vector scaling" in summary def test_an_interval_spanning_one_is_refused_even_when_ece_improves() -> None: