From b8eec0c3bd92c148a02f63baa6d0582e9a8ddc4b Mon Sep 17 00:00:00 2001 From: TMHSDigital <154358121+TMHSDigital@users.noreply.github.com> Date: Fri, 25 Sep 2026 07:08:37 -0400 Subject: [PATCH] feat: record a tie for the top option and count ties in the report On a two-decimal grid two options sharing the top probability is ordinary, and a row decided by the vendor's tie-break looked like any other. Prediction.tied_for_top names the tied options, each artifact record carries them, and the report says beside accuracy how many rows tied and on how many the gold label was a tied option the vendor did not choose. Part of #10: whether the vendor's tie-break is deterministic needs live calls, and the flag is what lets that be checked. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 5 +++ METHODOLOGY.md | 14 +++++++ src/plumbline/report/markdown.py | 25 +++++++++++ src/plumbline/runner/execute.py | 7 ++++ src/plumbline/types.py | 15 +++++++ tests/test_ties.py | 72 ++++++++++++++++++++++++++++++++ 6 files changed, 138 insertions(+) create mode 100644 tests/test_ties.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8090123..e74b854 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,11 @@ different event from one that moved because it was wrong. ### Added +- A row whose top probability is shared by two or more options is now recorded + as one (#10). Each artifact record carries `tied_for_top`, and when any row + tied the report says how many beside the accuracy figure, and on how many the + gold label was a tied option the vendor did not choose. On a two-decimal grid + such ties are ordinary, and a row decided by a tie-break looked like any other. - `plumbline run --dry-run` loads and checks the dataset, builds the adapter, applies the cost guard, and prints the case count, the endpoint, the timeout, the semantics, and the estimated cost with the pricing entry behind it, then diff --git a/METHODOLOGY.md b/METHODOLOGY.md index 6342695..2597c6e 100644 --- a/METHODOLOGY.md +++ b/METHODOLOGY.md @@ -287,6 +287,20 @@ label is one of the labels that were asked about, and raises rather than scoring the row if it is not. A service answering a different question than the one posed is not a prediction to score. +A tie is recorded, not only survived. Each record in the artifact carries +`tied_for_top`, the options that shared the highest probability when two or more +did, so a reader auditing a close result can see which rows were decided by the +vendor's tie-break rather than by a margin. When any row tied, the report says +how many beside the accuracy figure, and on how many of them the gold label was +a tied option the vendor did not choose: those rows count as wrong by a +tie-break, which is a fact about the measurement's precision rather than about +the model's preference. + +Whether a vendor's tie-break is deterministic is not known. If it is not, the +same tied row can resolve differently between calls, and a cached answer and a +live one can disagree on it for a reason that is nobody's fault. The flag is +what lets that be checked. + ## Probabilities arrive quantized, which bounds the resolution of any figure here Hosted Jev returns probabilities on a two-decimal grid. All 165 probability diff --git a/src/plumbline/report/markdown.py b/src/plumbline/report/markdown.py index 0a4032e..50edba1 100644 --- a/src/plumbline/report/markdown.py +++ b/src/plumbline/report/markdown.py @@ -282,6 +282,7 @@ def _arm(result: RunResult, options: ReportOptions, heading: str) -> list[str]: f"- **Failures**: {len(failures)} of {len(scoreable)} cases produced no " "prediction and are excluded from accuracy rather than scored wrong." ) + lines.extend(_tie_lines(successes)) probabilities = _probabilities(result, successes) lines.extend(_calibration_lines(probabilities, outcomes, options)) @@ -689,6 +690,30 @@ def _confidence_lines( return [f"- **Confidence**: {figure.statement()}"] +def _tie_lines(successes: Sequence[CaseRecord]) -> list[str]: + """How many answers the vendor's tie-break chose, and how many went against gold.""" + tied = [record for record in successes if record.prediction and record.prediction.tied_for_top] + if not tied: + return [] + against = sum( + 1 + for record in tied + if record.prediction + and record.gold_label in record.prediction.tied_for_top + and not record.correct + ) + line = ( + f"- **Ties**: {len(tied)} of {len(successes)} scored rows had two or more options tied " + "for the highest probability, so the vendor's tie-break chose the answer, not a margin." + ) + if against: + line += ( + f" On {against} of them the gold label was one of the tied options and was not the " + "one chosen, so those rows count as wrong by a tie-break." + ) + return [line] + + def _distribution_caveat(result: RunResult, successes: Sequence[CaseRecord]) -> list[str]: """The one line the methodology requires wherever a top line arrives alone.""" if result.probability_semantics == "none": diff --git a/src/plumbline/runner/execute.py b/src/plumbline/runner/execute.py index e7cb1be..b5d9ac5 100644 --- a/src/plumbline/runner/execute.py +++ b/src/plumbline/runner/execute.py @@ -315,6 +315,13 @@ def to_jsonable(self) -> dict[str, Any]: "prediction": ( to_jsonable(record.prediction) if record.prediction is not None else None ), + # Written for whoever audits a close result; derived from + # the distribution, so it is not read back. + "tied_for_top": ( + list(record.prediction.tied_for_top) + if record.prediction is not None + else [] + ), } for record in self.records ], diff --git a/src/plumbline/types.py b/src/plumbline/types.py index 9aae823..4a59221 100644 --- a/src/plumbline/types.py +++ b/src/plumbline/types.py @@ -252,6 +252,21 @@ def __post_init__(self) -> None: if abs(total - 1.0) > DISTRIBUTION_SUM_TOLERANCE: raise ValueError(f"distribution sums to {total!r}, expected approximately 1") + @property + def tied_for_top(self) -> tuple[str, ...]: + """The options sharing the highest probability, when two or more do. + + On such a row the answer was chosen by the vendor's tie-break rather + than by a margin. Probabilities on a two-decimal grid make that + ordinary, and ``label`` is always the vendor's pick, never a recomputed + argmax. Empty when nothing tied or there is no distribution. + """ + if not self.distribution: + return () + peak = max(self.distribution.values()) + tied = sorted(option for option, value in self.distribution.items() if value == peak) + return tuple(tied) if len(tied) > 1 else () + def docs_confidence(distribution: Mapping[str, float]) -> float: """Confidence as the TypeSafe docs describe it, for mocks and for comparison. diff --git a/tests/test_ties.py b/tests/test_ties.py new file mode 100644 index 0000000..1df36e6 --- /dev/null +++ b/tests/test_ties.py @@ -0,0 +1,72 @@ +"""A tie for the top option, recorded and counted (issue #10). + +Probabilities arrive on a two-decimal grid, so two options sharing the maximum +is ordinary. On such a row the vendor's tie-break chose the answer, not a +margin, and an auditor has to be able to see that. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from plumbline.adapters.mock import MockAdapter +from plumbline.report import markdown +from plumbline.runner import execute +from plumbline.types import Case, Prediction + + +def a_prediction(distribution: dict[str, float], label: str) -> Prediction: + return Prediction( + label=label, + prob_selected=distribution[label], + distribution=distribution, + confidence=None, + latency_ms=1.0, + cost_usd=None, + input_tokens=None, + output_tokens=None, + model_reported=None, + ) + + +def test_the_options_tied_for_the_top_are_named() -> None: + assert a_prediction({"a": 0.4, "b": 0.4, "c": 0.2}, "a").tied_for_top == ("a", "b") + assert a_prediction({"a": 0.5, "b": 0.3, "c": 0.2}, "a").tied_for_top == () + no_distribution = a_prediction({"a": 1.0}, "a") + assert no_distribution.tied_for_top == () + + +class TiesOnOddRows(MockAdapter): + """Picks "a" everywhere; on odd rows "a" and "b" tie at the top.""" + + def classify(self, text: str, labels: list[str], **_kwargs: object) -> Prediction: + tied = int(text[-1]) % 2 == 1 + spread = {"a": 0.4, "b": 0.4, "c": 0.2} if tied else {"a": 0.6, "b": 0.3, "c": 0.1} + return a_prediction(spread, "a") + + +def test_a_tie_is_in_the_artifact_and_counted_in_the_report(tmp_path: Path) -> None: + # Gold is "b" on every row, so each tied row went to the other tied option. + cases = [ + Case(id=f"c{i}", text=f"t{i}", labels=("a", "b", "c"), gold_label="b") for i in range(10) + ] + result = execute.run(TiesOnOddRows(gold_by_text={}), cases) + + stored = json.loads(result.write(tmp_path).read_text(encoding="utf-8")) + flagged = {row["case_id"]: row["tied_for_top"] for row in stored["records"]} + assert flagged["c1"] == ["a", "b"] and flagged["c0"] == [] + + document = markdown.render([result], options=markdown.ReportOptions(n_boot=50)) + ties = next(line for line in document.splitlines() if "**Ties**" in line) + assert "5 of 10 scored rows" in ties + assert "5 of them" in ties + + +def test_no_tie_line_when_nothing_tied() -> None: + cases = [Case(id=f"c{i}", text=f"t{i}", labels=("a", "b"), gold_label="a") for i in range(6)] + result = execute.run(MockAdapter(gold_by_text={case.text: "a" for case in cases}), cases) + + document = markdown.render([result], options=markdown.ReportOptions(n_boot=50)) + + assert "**Ties**" not in document