From 586583ccdda7790efe60b9615bf9acb470b4da8e Mon Sep 17 00:00:00 2001 From: TMHSDigital <154358121+TMHSDigital@users.noreply.github.com> Date: Fri, 25 Sep 2026 17:41:50 -0400 Subject: [PATCH] feat(local): ask a local checkpoint its options by letter The local arm read each option's own token and refused options that are several tokens, so on the public fixture it scored 39 of 105 rows. --option-style letter lists the options as A, B, C and reads the letter tokens: every row scores, the distribution still maps back to the labels, and the artifact and the report say the question was lettered. The style joins the cache key only when it is letter, so existing entries keep theirs. Run for real, the pinned Qwen2.5-1.5B landed at chance accuracy with an ECE of 0.349 against a floor of 0.087. Sequence probability was the alternative and was not taken: longer options lose probability for being long, and any length correction is a choice the result would silently depend on. METHODOLOGY says so. Part of #3. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 12 +++++ METHODOLOGY.md | 20 ++++++++ README.md | 16 ++++-- src/plumbline/adapters/local_logits.py | 66 +++++++++++++++++++++--- src/plumbline/cli.py | 19 +++++++ src/plumbline/report/markdown.py | 7 +++ src/plumbline/runner/execute.py | 4 ++ tests/test_cli_run_options.py | 23 +++++++++ tests/test_local_logits.py | 70 ++++++++++++++++++++++++++ 9 files changed, 225 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 56992de..cce4c8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,18 @@ different event from one that moved because it was wrong. ## Unreleased +### Added + +- `--option-style letter` asks a local checkpoint its options as A, B, C and + reads the letter tokens, so options of any length can be scored (#3). The + default still reads each option's own token and refuses one that is several + tokens. On the public fixture the default scores 39 of 105 rows; by letter, + all 105, at chance accuracy with an ECE of 0.349 against a floor of 0.087. + The style is part of the cache key only when it is `letter`, so existing + entries keep their keys, and the artifact and the report record it. + METHODOLOGY says why sequence probability was not used instead: longer + options would lose probability for being long. + ## v0.1.1 (2026-09-25) The review release: every issue the repository review filed is fixed, and the diff --git a/METHODOLOGY.md b/METHODOLOGY.md index b98e41e..3b27b31 100644 --- a/METHODOLOGY.md +++ b/METHODOLOGY.md @@ -425,6 +425,26 @@ moves, without anything about the case having changed. It can behave like a calibrated probability on a given workload, and whether it does is measurable, which is why plumbline measures it rather than assuming either way. +The local arm reads that softmax in one of two ways, and the artifact and the +report say which. By default it reads each option's own token, which means every +option must be a single token for the checkpoint: an option that tokenizes into +several pieces is refused by name, because a softmax over first tokens answers a +different question than the dataset asks. Options written as identifiers, such +as `pay_subject_to_10000_sublimit`, are almost never one token, so on a dataset +of them this reading scores little beyond the yes/no rows. + +`--option-style letter` asks the same options by letter instead: the prompt lists +them as A, B, C, and the softmax is over the letter tokens, each of which is one +token. That scores options of any length and is the usual way multiple-choice +questions are put to an open model. It is a different question, though, and is +labelled as one. The letters bring position into the answer, which is why the +cache keys the option order for this arm, and the numbers are conditional on the +lettered option set exactly as the plain reading is on the words. Scoring each +option by the probability of its whole token sequence was the other candidate, +and was not taken: longer options would lose probability for being long, and +every way of correcting for length is a choice the result would silently depend +on. + Citations, kept separate on purpose. SemIf is an independent project and says so: "not affiliated with or endorsed by TypeSafe". fastjev is an independently maintained fork of SemIf that preserves its history and MIT license, follows its diff --git a/README.md b/README.md index 0b1796d..88913ef 100644 --- a/README.md +++ b/README.md @@ -201,10 +201,16 @@ uv run plumbline run datasets/public/jevbench-hard.jsonl --format jevbench \ `--device` defaults to `cpu`, which works anywhere and is slow. On Windows the torch that PyPI serves is CPU only; for `--device cuda`, install a CUDA build -of torch from the PyTorch index into the same environment. The arm scores only -options that are a single token for the checkpoint and refuses the rest by -name, so on the public fixture, whose choice options are mostly multi-word -identifiers, most choice rows are refused and the yes/no rows carry the run. +of torch from the PyTorch index into the same environment. + +By default the arm reads each option's own token, so it scores only options +that are a single token for the checkpoint and refuses the rest by name. The +public fixture's choice options are mostly multi-word identifiers, so that way +66 of its 67 choice rows are refused. Add `--option-style letter` to ask the +options as A, B, C and read the letters instead: every row is scored, and the +report says the question was lettered. On the fixture that run landed at chance +accuracy with an ECE of 0.349 against a floor of 0.087: a small model that is +confidently wrong, which is the case a calibration figure exists to catch. ### Running a hosted vendor @@ -313,7 +319,7 @@ and self-hosted servers speak it, which is why one adapter covers all of them. | Adapter | Transport | Semantics | Run for real | Adding one | |---|---|---|---|---| | `typesafe_wire` | Jev wire format over HTTP | `calibrated_claim` | Yes, 40 rows against a hosted vendor | `--base-url`. Anything serving the same wire format is one flag, recorded in the artifact, including self-hosted endpoints and open models behind a compatible server. | -| `local_logits` | Option-token logits from a local checkpoint | `restricted_softmax` | Yes, 105 rows against a pinned Qwen2.5-1.5B-Instruct on a GPU; 39 scored, 66 refused as multi-token | A HuggingFace model id and a pinned revision. Needs the optional `local` extra. | +| `local_logits` | Option-token logits from a local checkpoint | `restricted_softmax` | Yes, 105 rows against a pinned Qwen2.5-1.5B-Instruct on a GPU; 39 scored by label, all 105 by letter | A HuggingFace model id and a pinned revision. Needs the optional `local` extra. | | `generative` | Chat completion, parsed | `none` | **No, tests only** | A model string. | | `mock` | None, seeded | configurable | Yes, it is the example report | Built in. A deterministic stand-in, not a system under test. | diff --git a/src/plumbline/adapters/local_logits.py b/src/plumbline/adapters/local_logits.py index d877286..3745a44 100644 --- a/src/plumbline/adapters/local_logits.py +++ b/src/plumbline/adapters/local_logits.py @@ -34,13 +34,26 @@ import threading import time from collections.abc import Mapping, Sequence -from typing import Protocol +from typing import Literal, Protocol, get_args from plumbline.adapters.base import Adapter from plumbline.types import CaseRefusedError, PlumblineError, Prediction, QuestionType DEFAULT_INSTRUCTIONS = "Which label best describes this text?" +#: How the options are put to the checkpoint. ``label`` reads each option's own +#: token, so every option must be a single token. ``letter`` lists the options +#: as A, B, C... and reads the letters, so an option can be any text. +OptionStyle = Literal["label", "letter"] +OPTION_STYLES: tuple[OptionStyle, ...] = get_args(OptionStyle) + +#: The default question when options are asked by letter. +DEFAULT_LETTER_INSTRUCTIONS = ( + "Which option best describes this text? Answer with the letter of the option." +) + +_LETTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + #: The one prompt shape this adapter sends. It is in ``call_params``, so editing #: it invalidates the cache rather than mixing two prompt shapes in one table. DEFAULT_PROMPT_TEMPLATE = "{instructions}\n\nText: {text}\n\nOptions: {options}\n\nAnswer:" @@ -98,6 +111,11 @@ class LocalLogitsAdapter(Adapter): cannot be reproduced from the artifact alone. device: Passed to the default readout. Not part of ``call_params``: it moves arithmetic, not the question. + option_style: ``label`` reads each option's own token and refuses a + case whose options are not single tokens. ``letter`` lists the + options as A, B, C... and reads the letter tokens, so options of + any length can be asked, at the price of asking a lettered + question; the style is part of ``call_params``. ``probability_semantics`` is fixed at ``"restricted_softmax"`` and is not a constructor argument. There is no configuration under which this arm becomes @@ -112,12 +130,17 @@ def __init__( model_requested: str, revision: str, readout: LogitReadout | None = None, - instructions: str = DEFAULT_INSTRUCTIONS, + instructions: str | None = None, prompt_template: str = DEFAULT_PROMPT_TEMPLATE, option_prefix: str = DEFAULT_OPTION_PREFIX, allow_unpinned_revision: bool = False, device: str = "cpu", + option_style: OptionStyle = "label", ) -> None: + if option_style not in OPTION_STYLES: + raise ValueError( + f"option_style must be one of {list(OPTION_STYLES)!r}, got {option_style!r}" + ) if not revision or not revision.strip(): raise ValueError( "local_logits requires a revision. An unpinned checkpoint makes a result " @@ -140,7 +163,10 @@ def __init__( self.pinned_revision = revision.strip() self.revision = self.pinned_revision self.probability_semantics = "restricted_softmax" - self.instructions = instructions + self.option_style: OptionStyle = option_style + self.instructions = instructions or ( + DEFAULT_LETTER_INSTRUCTIONS if option_style == "letter" else DEFAULT_INSTRUCTIONS + ) self.prompt_template = prompt_template self.option_prefix = option_prefix self.allow_unpinned_revision = allow_unpinned_revision @@ -173,11 +199,16 @@ def call_params(self) -> Mapping[str, object]: The base key already covers the adapter name, the model, the revision, the case text, and the sorted labels. The prompt shape is added here. """ - return { + params: dict[str, object] = { "instructions": self.instructions, "prompt_template": self.prompt_template, "option_prefix": self.option_prefix, } + # Added only when it differs from the original question, so every + # cache entry written before letters existed keeps its key. + if self.option_style != "label": + params["option_style"] = self.option_style + return params #: The prompt lists the options in the order given. label_order_matters = True @@ -200,12 +231,18 @@ def classify( if len(labels) < 2: raise ValueError(f"need at least 2 labels, got {len(labels)}") + letters = self._letters(labels) readout = self.readout - token_ids = self._single_token_ids(readout, labels) + token_ids = self._single_token_ids(readout, letters or labels) self._check_revision(readout) + options = ( + ", ".join(f"{letter}. {label}" for letter, label in zip(letters, labels, strict=True)) + if letters + else ", ".join(labels) + ) prompt = self.prompt_template.format( - instructions=self.instructions, text=text, options=", ".join(labels) + instructions=self.instructions, text=text, options=options ) with self._forward_lock: @@ -244,9 +281,22 @@ def classify( "option_token_ids": dict(zip(labels, token_ids, strict=True)), "option_logits": dict(zip(labels, logits, strict=True)), "probability_semantics": self.probability_semantics, + "option_style": self.option_style, + **({"option_letters": dict(zip(labels, letters, strict=True))} if letters else {}), }, ) + def _letters(self, labels: list[str]) -> list[str]: + """The letter standing for each option, or none when options are read as themselves.""" + if self.option_style != "letter": + return [] + if len(labels) > len(_LETTERS): + raise CaseRefusedError( + f"this case has {len(labels)} options and there are {len(_LETTERS)} letters, " + "so it cannot be asked by letter. Ask it by label, or run it on another arm." + ) + return list(_LETTERS[: len(labels)]) + def _single_token_ids(self, readout: LogitReadout, labels: list[str]) -> list[int]: """One token id per option, or a refusal naming what went wrong. @@ -270,7 +320,9 @@ def _single_token_ids(self, readout: LogitReadout, labels: list[str]) -> list[in "plumbline refuses the case rather than truncating an option to its " "first token, because a softmax over truncated options answers a " "different question than the one the dataset asks. Use options that are " - "single tokens for this checkpoint, or run this case on another arm." + "single tokens for this checkpoint, ask the options by letter " + "(option_style letter, --option-style letter), or run this case on another " + "arm." ) duplicates = [ diff --git a/src/plumbline/cli.py b/src/plumbline/cli.py index 467ad25..51e51fb 100644 --- a/src/plumbline/cli.py +++ b/src/plumbline/cli.py @@ -98,6 +98,15 @@ def run( "takes it. Recorded in the artifact.", ), ] = None, + option_style: Annotated[ + str | None, + typer.Option( + "--option-style", + help="How a local checkpoint is asked its options: label reads each option's own " + "token and refuses options that are not one token; letter lists them as A, B, " + "C... and reads the letters.", + ), + ] = None, semantics: Annotated[ str | None, typer.Option( @@ -159,6 +168,7 @@ def run( base_url=base_url, timeout=timeout, device=device, + option_style=option_style, semantics=semantics, seed=seed, accuracy=accuracy, @@ -317,6 +327,7 @@ def _build( base_url: str | None = None, timeout: float | None = None, device: str | None = None, + option_style: str | None = None, semantics: str | None, seed: int, accuracy: float, @@ -333,6 +344,8 @@ def _build( config["timeout"] = timeout if device is not None: config["device"] = device + if option_style is not None: + config["option_style"] = option_style if semantics is not None: config["probability_semantics"] = _semantics(semantics) @@ -374,6 +387,7 @@ def _build( "base_url": "--base-url", "timeout": "--timeout", "device": "--device", + "option_style": "--option-style", "probability_semantics": "--semantics", } @@ -413,6 +427,11 @@ def _plan_text( if timeout else "- Timeout: the adapter's default.", *([f"- Device: {device}."] if device else []), + *( + ["- Options: asked as letters, A for the first, and read from the letter tokens."] + if getattr(adapter, "option_style", None) == "letter" + else [] + ), f"- Probability semantics: {semantics}.", f"- Cost: {cost}", f"- Guard: {cost_limit}, {case_limit}; the run would start.", diff --git a/src/plumbline/report/markdown.py b/src/plumbline/report/markdown.py index a801160..be90001 100644 --- a/src/plumbline/report/markdown.py +++ b/src/plumbline/report/markdown.py @@ -566,6 +566,13 @@ def _provenance(result: RunResult, options: ReportOptions) -> list[str]: if hits: line += f" {hits} of {len(result.records)} rows came from cache and cost nothing." lines = [line] + if result.config.get("option_style") == "letter": + lines.append( + "- **Options**: asked as letters, A for the first option and so on, and read from " + "the letter tokens. The softmax is over the letters, so it is still conditional on " + "the options supplied, and a lettered question is not the question an arm that " + "reads the option words is asked." + ) if result.config.get("semantics_set_by") == "operator": lines.append( f"- **Probability semantics**: {_code(result.probability_semantics)}, set by the " diff --git a/src/plumbline/runner/execute.py b/src/plumbline/runner/execute.py index b5d9ac5..8d41d27 100644 --- a/src/plumbline/runner/execute.py +++ b/src/plumbline/runner/execute.py @@ -641,6 +641,10 @@ def handle(index: int) -> None: # Where a local checkpoint ran. It does not change the question, but a # latency figure means nothing without it. "device": getattr(adapter, "device", None), + # How a local checkpoint was asked its options: by their own tokens, or + # by letter. The report says so, since a lettered question is a + # different question. + "option_style": getattr(adapter, "option_style", None), # Whether rows carried option descriptions and whether they were sent, # so the report can say when the dataset's descriptions went nowhere. "label_descriptions": { diff --git a/tests/test_cli_run_options.py b/tests/test_cli_run_options.py index b3121bd..1e72209 100644 --- a/tests/test_cli_run_options.py +++ b/tests/test_cli_run_options.py @@ -282,3 +282,26 @@ def test_refused_cases_are_not_called_cache_hits_and_no_confidence_is_not_blamed assert "over 5 live calls" in latency_line assert "cache hits" not in latency_line assert "yes/no" not in confidence_line + + +def test_an_option_style_goes_to_the_local_arm_and_is_refused_elsewhere(tmp_path: Path) -> None: + dataset = a_dataset(tmp_path / "d.jsonl") + + refused = invoke("run", str(dataset), "--option-style", "letter", "--dry-run") + planned = invoke( + "run", + str(dataset), + "--adapter", + "local_logits", + "--model", + "some/checkpoint", + "--revision", + "c" * 40, + "--option-style", + "letter", + "--dry-run", + ) + + assert refused.exit_code == 1 and "does not take --option-style" in refused.stderr + assert planned.exit_code == 0, planned.stderr + assert "Options: asked as letters" in planned.stdout diff --git a/tests/test_local_logits.py b/tests/test_local_logits.py index eb7b9a8..fcb5de3 100644 --- a/tests/test_local_logits.py +++ b/tests/test_local_logits.py @@ -308,3 +308,73 @@ def __init__(self, **kwargs: object) -> None: assert seen["device"] == "cuda" assert result.config["device"] == "cuda" + + +# Letter labels, for options that are not single tokens + + +class LetterReadout: + """Scores the letters A, B, C...; any other text is several tokens.""" + + def __init__(self, letter_scores: dict[str, float]) -> None: + self.letter_scores = letter_scores + self.resolved_revision = PINNED + self.prompts: list[str] = [] + self.asked: list[str] = [] + + def token_ids(self, text: str) -> list[int]: + self.asked.append(text) + letter = text.strip() + if len(letter) == 1 and letter.isupper(): + return [ord(letter)] + return [1, 2, 3] + + def option_logits(self, prompt: str, token_ids: list[int]) -> list[float]: + self.prompts.append(prompt) + return [self.letter_scores[chr(token_id)] for token_id in token_ids] + + +MULTI = ["pay_full_estimate", "deny_vacancy_exclusion", "not_covered"] + + +def test_letters_ask_multi_token_options_by_their_letter() -> None: + readout = LetterReadout({"A": 0.0, "B": 2.0, "C": 1.0}) + adapter = an_adapter(readout, option_style="letter") # type: ignore[arg-type] + + prediction = adapter.classify("the claim", MULTI) + + assert prediction.label == "deny_vacancy_exclusion" + assert set(prediction.distribution or {}) == set(MULTI) + assert readout.asked == [" A", " B", " C"] + assert "A. pay_full_estimate, B. deny_vacancy_exclusion, C. not_covered" in readout.prompts[0] + assert prediction.raw["option_style"] == "letter" + assert prediction.raw["option_letters"] == dict(zip(MULTI, "ABC", strict=True)) + + +def test_letters_default_to_asking_for_the_letter() -> None: + readout = LetterReadout({"A": 1.0, "B": 0.0, "C": 0.0}) + an_adapter(readout, option_style="letter").classify("t", MULTI) # type: ignore[arg-type] + + assert "letter" in readout.prompts[0].lower() + + +def test_more_options_than_letters_is_refused() -> None: + readout = LetterReadout({chr(65 + i): 0.0 for i in range(26)}) + labels = [f"option_{i}" for i in range(27)] + + with pytest.raises(CaseRefusedError, match="27 options"): + an_adapter(readout, option_style="letter").classify("t", labels) # type: ignore[arg-type] + + +def test_the_style_changes_the_cache_key_and_label_mode_keys_do_not_move() -> None: + labelled = an_adapter() + lettered = an_adapter(option_style="letter") + + assert "option_style" not in labelled.call_params + assert lettered.call_params["option_style"] == "letter" + assert lettered.call_params != labelled.call_params + + +def test_an_unknown_style_is_refused() -> None: + with pytest.raises(ValueError, match="option_style"): + an_adapter(option_style="numbers")