feat: record a tie for the top option and count ties in the report

CHANGELOG.md

@@ -13,6 +13,11 @@ different event from one that moved because it was wrong.
 
 ### Added
 
+- A row whose top probability is shared by two or more options is now recorded
+  as one (#10). Each artifact record carries `tied_for_top`, and when any row
+  tied the report says how many beside the accuracy figure, and on how many the
+  gold label was a tied option the vendor did not choose. On a two-decimal grid
+  such ties are ordinary, and a row decided by a tie-break looked like any other.
 - `plumbline run --dry-run` loads and checks the dataset, builds the adapter,
   applies the cost guard, and prints the case count, the endpoint, the timeout,
   the semantics, and the estimated cost with the pricing entry behind it, then

METHODOLOGY.md

@@ -287,6 +287,20 @@ label is one of the labels that were asked about, and raises rather than scoring
 the row if it is not. A service answering a different question than the one
 posed is not a prediction to score.
 
+A tie is recorded, not only survived. Each record in the artifact carries
+`tied_for_top`, the options that shared the highest probability when two or more
+did, so a reader auditing a close result can see which rows were decided by the
+vendor's tie-break rather than by a margin. When any row tied, the report says
+how many beside the accuracy figure, and on how many of them the gold label was
+a tied option the vendor did not choose: those rows count as wrong by a
+tie-break, which is a fact about the measurement's precision rather than about
+the model's preference.
+
+Whether a vendor's tie-break is deterministic is not known. If it is not, the
+same tied row can resolve differently between calls, and a cached answer and a
+live one can disagree on it for a reason that is nobody's fault. The flag is
+what lets that be checked.
+
 ## Probabilities arrive quantized, which bounds the resolution of any figure here
 
 Hosted Jev returns probabilities on a two-decimal grid. All 165 probability

src/plumbline/report/markdown.py

@@ -282,6 +282,7 @@ def _arm(result: RunResult, options: ReportOptions, heading: str) -> list[str]:
             f"- **Failures**: {len(failures)} of {len(scoreable)} cases produced no "
             "prediction and are excluded from accuracy rather than scored wrong."
         )
+    lines.extend(_tie_lines(successes))
 
     probabilities = _probabilities(result, successes)
     lines.extend(_calibration_lines(probabilities, outcomes, options))
@@ -689,6 +690,30 @@ def _confidence_lines(
     return [f"- **Confidence**: {figure.statement()}"]
 
 
+def _tie_lines(successes: Sequence[CaseRecord]) -> list[str]:
+    """How many answers the vendor's tie-break chose, and how many went against gold."""
+    tied = [record for record in successes if record.prediction and record.prediction.tied_for_top]
+    if not tied:
+        return []
+    against = sum(
+        1
+        for record in tied
+        if record.prediction
+        and record.gold_label in record.prediction.tied_for_top
+        and not record.correct
+    )
+    line = (
+        f"- **Ties**: {len(tied)} of {len(successes)} scored rows had two or more options tied "
+        "for the highest probability, so the vendor's tie-break chose the answer, not a margin."
+    )
+    if against:
+        line += (
+            f" On {against} of them the gold label was one of the tied options and was not the "
+            "one chosen, so those rows count as wrong by a tie-break."
+        )
+    return [line]
+
+
 def _distribution_caveat(result: RunResult, successes: Sequence[CaseRecord]) -> list[str]:
     """The one line the methodology requires wherever a top line arrives alone."""
     if result.probability_semantics == "none":

src/plumbline/runner/execute.py

@@ -315,6 +315,13 @@ def to_jsonable(self) -> dict[str, Any]:
                     "prediction": (
                         to_jsonable(record.prediction) if record.prediction is not None else None
                     ),
+                    # Written for whoever audits a close result; derived from
+                    # the distribution, so it is not read back.
+                    "tied_for_top": (
+                        list(record.prediction.tied_for_top)
+                        if record.prediction is not None
+                        else []
+                    ),
                 }
                 for record in self.records
             ],

src/plumbline/types.py

@@ -252,6 +252,21 @@ def __post_init__(self) -> None:
             if abs(total - 1.0) > DISTRIBUTION_SUM_TOLERANCE:
                 raise ValueError(f"distribution sums to {total!r}, expected approximately 1")
 
+    @property
+    def tied_for_top(self) -> tuple[str, ...]:
+        """The options sharing the highest probability, when two or more do.
+
+        On such a row the answer was chosen by the vendor's tie-break rather
+        than by a margin. Probabilities on a two-decimal grid make that
+        ordinary, and ``label`` is always the vendor's pick, never a recomputed
+        argmax. Empty when nothing tied or there is no distribution.
+        """
+        if not self.distribution:
+            return ()
+        peak = max(self.distribution.values())
+        tied = sorted(option for option, value in self.distribution.items() if value == peak)
+        return tuple(tied) if len(tied) > 1 else ()
+
 
 def docs_confidence(distribution: Mapping[str, float]) -> float:
     """Confidence as the TypeSafe docs describe it, for mocks and for comparison.

tests/test_ties.py

@@ -0,0 +1,72 @@
+"""A tie for the top option, recorded and counted (issue #10).
+
+Probabilities arrive on a two-decimal grid, so two options sharing the maximum
+is ordinary. On such a row the vendor's tie-break chose the answer, not a
+margin, and an auditor has to be able to see that.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from plumbline.adapters.mock import MockAdapter
+from plumbline.report import markdown
+from plumbline.runner import execute
+from plumbline.types import Case, Prediction
+
+
+def a_prediction(distribution: dict[str, float], label: str) -> Prediction:
+    return Prediction(
+        label=label,
+        prob_selected=distribution[label],
+        distribution=distribution,
+        confidence=None,
+        latency_ms=1.0,
+        cost_usd=None,
+        input_tokens=None,
+        output_tokens=None,
+        model_reported=None,
+    )
+
+
+def test_the_options_tied_for_the_top_are_named() -> None:
+    assert a_prediction({"a": 0.4, "b": 0.4, "c": 0.2}, "a").tied_for_top == ("a", "b")
+    assert a_prediction({"a": 0.5, "b": 0.3, "c": 0.2}, "a").tied_for_top == ()
+    no_distribution = a_prediction({"a": 1.0}, "a")
+    assert no_distribution.tied_for_top == ()
+
+
+class TiesOnOddRows(MockAdapter):
+    """Picks "a" everywhere; on odd rows "a" and "b" tie at the top."""
+
+    def classify(self, text: str, labels: list[str], **_kwargs: object) -> Prediction:
+        tied = int(text[-1]) % 2 == 1
+        spread = {"a": 0.4, "b": 0.4, "c": 0.2} if tied else {"a": 0.6, "b": 0.3, "c": 0.1}
+        return a_prediction(spread, "a")
+
+
+def test_a_tie_is_in_the_artifact_and_counted_in_the_report(tmp_path: Path) -> None:
+    # Gold is "b" on every row, so each tied row went to the other tied option.
+    cases = [
+        Case(id=f"c{i}", text=f"t{i}", labels=("a", "b", "c"), gold_label="b") for i in range(10)
+    ]
+    result = execute.run(TiesOnOddRows(gold_by_text={}), cases)
+
+    stored = json.loads(result.write(tmp_path).read_text(encoding="utf-8"))
+    flagged = {row["case_id"]: row["tied_for_top"] for row in stored["records"]}
+    assert flagged["c1"] == ["a", "b"] and flagged["c0"] == []
+
+    document = markdown.render([result], options=markdown.ReportOptions(n_boot=50))
+    ties = next(line for line in document.splitlines() if "**Ties**" in line)
+    assert "5 of 10 scored rows" in ties
+    assert "5 of them" in ties
+
+
+def test_no_tie_line_when_nothing_tied() -> None:
+    cases = [Case(id=f"c{i}", text=f"t{i}", labels=("a", "b"), gold_label="a") for i in range(6)]
+    result = execute.run(MockAdapter(gold_by_text={case.text: "a" for case in cases}), cases)
+
+    document = markdown.render([result], options=markdown.ReportOptions(n_boot=50))
+
+    assert "**Ties**" not in document