From e4d101e4cf3156df3d8ddb706fcedc88a1e50ac8 Mon Sep 17 00:00:00 2001 From: TMHSDigital <154358121+TMHSDigital@users.noreply.github.com> Date: Thu, 24 Sep 2026 18:59:08 -0400 Subject: [PATCH] feat(cli): --dry-run, --base-url, --timeout, and the load report in the artifact A dry run loads and checks the dataset, builds the adapter, and applies the cost guard through the same plan the run starts from, then prints the case count, endpoint, timeout, semantics, and estimate, and sends nothing. --base-url and --timeout reach an adapter that takes them, and the timeout is recorded beside the endpoint. --semantics now overrides any adapter that takes it, and the report says the operator set it. An option an adapter does not take is refused by name, read from its signature. An artifact stores a LoadSummary, so plumbline report rebuilds the Dataset section it used to drop. Fixes #58. Co-Authored-By: Claude Opus 5.5 (1M context) --- .env.example | 3 +- CHANGELOG.md | 17 +++ CONTRIBUTING.md | 3 +- README.md | 13 +- src/plumbline/adapters/registry.py | 16 ++ src/plumbline/cli.py | 140 ++++++++++++++++-- src/plumbline/datasets/loader.py | 68 ++++++++- src/plumbline/report/markdown.py | 16 +- src/plumbline/runner/execute.py | 53 ++++++- tests/test_cli_run_options.py | 230 +++++++++++++++++++++++++++++ 10 files changed, 528 insertions(+), 31 deletions(-) create mode 100644 tests/test_cli_run_options.py diff --git a/.env.example b/.env.example index c4cdcd5..0fe43b3 100644 --- a/.env.example +++ b/.env.example @@ -18,7 +18,8 @@ TYPESAFE_API_KEY= # endpoint instead of the hosted service. The endpoint is part of the cache key, # so a self-hosted run is never served a hosted run's answers, and it is # recorded in the artifact and named in the report. ANTHROPIC_BASE_URL does the -# same for the generative arm. +# same for the generative arm. `plumbline run --base-url` sets it for one run +# and takes precedence over either variable. TYPESAFE_BASE_URL= # Generative control arm, which speaks Anthropic's Messages API. Only needed if diff --git a/CHANGELOG.md b/CHANGELOG.md index 06f6aa9..4c4fb30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,23 @@ different event from one that moved because it was wrong. ### Added +- `plumbline run --dry-run` loads and checks the dataset, builds the adapter, + applies the cost guard, and prints the case count, the endpoint, the timeout, + the semantics, and the estimated cost with the pricing entry behind it, then + exits without sending a request or writing a file (#58). It refuses exactly + what the run would refuse, because the run starts from the same plan. +- `--base-url` and `--timeout` on `plumbline run`, for an adapter that takes + them (#58). The endpoint could be changed only through an environment + variable before; it is recorded in the artifact and the cache key either + way, and the timeout is now recorded too. `--semantics` now overrides the + declared semantics for any adapter that takes it, not only the mock, and the + report says when the operator set it. An option an adapter does not take is + refused by name before anything is built. +- An artifact now stores what loading the dataset found, so a report rebuilt + with `plumbline report` prints the same Dataset section, with the read, + loaded, and refused counts, as the one written at run time (#58). Artifacts + written before this still report, without the section. + - A browser calculator for the ECE floor (`site/`), a JavaScript port of `synthetic_floor` that reproduces numpy's seeded random stream draw for draw. `scripts/floor_golden.py` exports golden values from the Python and diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 650bd8e..b82cc53 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -9,7 +9,8 @@ one of them. Adding a vendor should not add a file. - **Anything serving the Jev wire format is a `base_url` for `typesafe_wire`.** A self-hosted endpoint, a compatible server in front of an open model, a - provider that implemented the same shape. It is a config entry. That includes + provider that implemented the same shape. It is `plumbline run --base-url`, + recorded in the artifact and part of the cache key. That includes the open decision models that serve this wire format: the unchanged adapter runs them. - **Any open-weights checkpoint is a config entry for `local_logits`**, as a diff --git a/README.md b/README.md index 611dd95..c824694 100644 --- a/README.md +++ b/README.md @@ -237,6 +237,13 @@ Without a pricing table the run still works; cost reports as unpriced, and `--max-cost-usd` refuses rather than bounding a run it cannot cost. See [Limitations](#limitations). +Add `--dry-run` to either command first to see what the run would do without +spending anything. It loads and checks the dataset, builds the adapter, applies +the guard, and prints the case count, the endpoint, and the estimated cost, then +exits without sending a request or writing a file. Pass `--base-url` to point +`typesafe_wire` at a self-hosted endpoint, and `--timeout` to bound one request; +both are recorded in the artifact. + Your own data goes in `datasets/private/`, which is gitignored, and that is the only path on which the recalibration numbers mean anything. [Your own data](docs/datasets.md) gives the row format, what is refused and why, @@ -290,7 +297,7 @@ and self-hosted servers speak it, which is why one adapter covers all of them. | Adapter | Transport | Semantics | Run for real | Adding one | |---|---|---|---|---| -| `typesafe_wire` | Jev wire format over HTTP | `calibrated_claim` | Yes, 40 rows against a hosted vendor | A `base_url`. Anything serving the same wire format is a config entry, including self-hosted endpoints and open models behind a compatible server. | +| `typesafe_wire` | Jev wire format over HTTP | `calibrated_claim` | Yes, 40 rows against a hosted vendor | `--base-url`. Anything serving the same wire format is one flag, recorded in the artifact, including self-hosted endpoints and open models behind a compatible server. | | `local_logits` | Option-token logits from a local checkpoint | `restricted_softmax` | **No, tests only** | A HuggingFace model id and a pinned revision. Needs the optional `local` extra. | | `generative` | Chat completion, parsed | `none` | **No, tests only** | A model string. | | `mock` | None, seeded | configurable | Yes, it is the example report | Built in. A deterministic stand-in, not a system under test. | @@ -369,8 +376,8 @@ Specific, and none of them are going to surprise you later. - **[Mapika/decider](https://github.com/Mapika/decider)**. One-pass typed decisions with calibrated probabilities, fine-tuned from Qwen3.5-2B, in several sizes. Its own guidance is to check calibration on your own labels before - routing on confidence. It serves the same typed question shape, so it is a - `base_url` config entry here rather than new code. + routing on confidence. It serves the same typed question shape, so it is + `--base-url` on `typesafe_wire` here rather than new code. - **[Bespoke Nimble](https://github.com/bespokelabsai/nimble)**. An open recipe for typed decision models, a LoRA fine-tune on Qwen3.5-9B, trained with contrastive data curation. It serves the Jev wire format, so the unchanged diff --git a/src/plumbline/adapters/registry.py b/src/plumbline/adapters/registry.py index 25ddbff..6e28cdd 100644 --- a/src/plumbline/adapters/registry.py +++ b/src/plumbline/adapters/registry.py @@ -110,6 +110,22 @@ def _missing_arguments(factory: AdapterFactory, config: dict[str, Any]) -> list[ ] +def accepts(name: str, setting: str) -> bool: + """Whether the adapter registered as ``name`` takes ``setting`` when it is built. + + Imports a built-in to read its signature, as ``create`` would, so a caller can + name the one setting an adapter refuses before trying to build it. + """ + try: + parameters = inspect.signature(_factory(name)).parameters.values() + except (TypeError, ValueError): # no signature to read, so let create decide + return True + return any( + parameter.name == setting or parameter.kind is inspect.Parameter.VAR_KEYWORD + for parameter in parameters + ) + + def available() -> tuple[str, ...]: """Registered adapter names, sorted, without importing any of them.""" return tuple(sorted({*_REGISTRY, *_BUILTINS})) diff --git a/src/plumbline/cli.py b/src/plumbline/cli.py index dca08e7..9e7eb1e 100644 --- a/src/plumbline/cli.py +++ b/src/plumbline/cli.py @@ -76,13 +76,34 @@ def run( "truncation: use --limit to run fewer.", ), ] = None, + base_url: Annotated[ + str | None, + typer.Option( + "--base-url", + help="Send requests here instead of the vendor's endpoint, for an adapter that " + "takes one. Recorded in the artifact and part of the cache key.", + ), + ] = None, + timeout: Annotated[ + float | None, + typer.Option( + "--timeout", help="Seconds one request may take, for an adapter that takes it." + ), + ] = None, semantics: Annotated[ str | None, typer.Option( - help="Override probability_semantics, mock only: one of " - f"{', '.join(PROBABILITY_SEMANTICS)}." + help="Override the probability_semantics the adapter declares, for one that " + f"takes it: one of {', '.join(PROBABILITY_SEMANTICS)}. The report says so." ), ] = None, + dry_run: Annotated[ + bool, + typer.Option( + "--dry-run", + help="Load, check, and price the run, print what it would do, and send nothing.", + ), + ] = False, seed: Annotated[int, typer.Option(help="Mock seed.")] = 7, accuracy: Annotated[float, typer.Option(help="Mock target accuracy.")] = 0.8, escalation_cost: Annotated[ @@ -102,12 +123,15 @@ def run( Status lines and errors go to stderr, so stdout carries only the report when no --report is given. The exit code is 1 when no case produced a prediction, - after the artifact and the report are written. + after the artifact and the report are written. With --dry-run, stdout carries + the plan instead, and nothing is sent or written. """ # Everything that can be checked before a call goes out is checked here, # so a mistake costs nothing. if report is not None and report.is_dir(): _fail(f"--report {report} is a directory; give it a file path, such as {report}/report.md") + if timeout is not None and not timeout > 0: + _fail(f"--timeout must be a number of seconds above 0, got {timeout}") load = dataclasses.replace(_load(dataset, data_format), source=_shown(dataset)) _status(load.statement()) for refusal in load.refusals: @@ -124,19 +148,32 @@ def run( cases, model=model, revision=revision, + base_url=base_url, + timeout=timeout, semantics=semantics, seed=seed, accuracy=accuracy, ) + guard = execute.CostGuard(max_cost_usd=max_cost_usd, max_cases=max_cases) + table = _pricing(pricing) + if dry_run: + planned = _guard(lambda: execute.plan(built, cases, guard=guard, pricing_table=table)) + typer.echo(_plan_text(built, planned, guard, semantics_set=semantics is not None)) + return + + extra: dict[str, object] = {"dataset": _shown(dataset), "format": data_format} + if semantics is not None: + extra["semantics_set_by"] = "operator" result = _guard( lambda: execute.run( built, cases, cache=Cache(cache_dir) if cache_dir else None, - guard=execute.CostGuard(max_cost_usd=max_cost_usd, max_cases=max_cases), - pricing_table=_pricing(pricing), + guard=guard, + pricing_table=table, workers=workers, - extra_config={"dataset": _shown(dataset), "format": data_format}, + extra_config=extra, + load=load.summary(), ) ) @@ -192,6 +229,7 @@ def report( document = _guard( lambda: markdown.render( results, + load=_stored_load(results), options=markdown.ReportOptions( n_boot=n_boot, cost_escalation_usd=escalation_cost, @@ -208,6 +246,20 @@ def report( typer.echo(document) +def _stored_load(results: list[execute.RunResult]) -> loader.LoadSummary | None: + """The Dataset section a rebuilt report can print, from what the runs stored. + + Only when every run is over one dataset and they all say the same thing + about loading it. Otherwise there is no single section that is true of the + document, and printing one run's counts over another's figures would be + worse than printing none. + """ + if len({result.dataset_hash for result in results}) != 1: + return None + first = results[0].load + return first if all(result.load == first for result in results) else None + + @app.command() def adapters() -> None: """List the transports this install can run.""" @@ -253,6 +305,8 @@ def _build( *, model: str | None, revision: str | None, + base_url: str | None = None, + timeout: float | None = None, semantics: str | None, seed: int, accuracy: float, @@ -263,6 +317,12 @@ def _build( config["model_requested"] = model if revision is not None: config["revision"] = revision + if base_url is not None: + config["base_url"] = base_url + if timeout is not None: + config["timeout"] = timeout + if semantics is not None: + config["probability_semantics"] = _semantics(semantics) if adapter == "mock": # The mock is told the answer key up front, because classify() is never @@ -270,17 +330,23 @@ def _build( config["gold_by_text"] = {case.text: case.gold_label for case in cases} config["seed"] = seed config["accuracy"] = accuracy - if semantics is not None: - config["probability_semantics"] = _semantics(semantics) - elif semantics is not None: - _fail("--semantics applies to the mock only; a real adapter declares its own.") + + # Name the exact option an adapter will not take, before building it: the + # generative arm reports no probability, so it has no semantics to override. + refused_options = [ + flag + for setting, flag in _OPTIONS.items() + if setting in config and not _guard(partial(registry.accepts, adapter, setting)) + ] + if refused_options: + _fail(f"the {adapter} adapter does not take {', '.join(refused_options)}.") try: return registry.create(adapter, **config) except (PlumblineError, ValueError) as refused: _fail(str(refused)) except TypeError: - given = [f"--{name.replace('_requested', '')}" for name in config if name in _OPTIONS] + given = [flag for setting, flag in _OPTIONS.items() if setting in config] _fail(f"the {adapter} adapter does not take {', '.join(given) or 'these settings'}.") except Exception as unavailable: # an SDK that cannot start, such as a missing key variable = _KEY_VARIABLES.get(adapter) @@ -288,8 +354,56 @@ def _build( _fail(f"could not set up the {adapter} adapter: {unavailable}.{hint}") -#: Settings the CLI passes to an adapter from its own options. -_OPTIONS = ("model_requested", "revision") +#: Settings the CLI passes to an adapter from its own options, and the option +#: each comes from. +_OPTIONS = { + "model_requested": "--model", + "revision": "--revision", + "base_url": "--base-url", + "timeout": "--timeout", + "probability_semantics": "--semantics", +} + + +def _plan_text( + adapter: Adapter, planned: execute.Plan, guard: execute.CostGuard, *, semantics_set: bool +) -> str: + """What a dry run prints: what would be sent, to whom, and what it would cost.""" + endpoint = getattr(adapter, "base_url", None) + timeout = getattr(adapter, "timeout", None) + semantics = adapter.probability_semantics + (", set by --semantics" if semantics_set else "") + if planned.estimated_cost_usd is not None and planned.pricing is not None: + cost = ( + f"about {planned.estimated_cost_usd:.4f} USD, priced by `{planned.pricing_key}` " + f"as read on {planned.pricing.as_of.isoformat()}. The estimate is rough: it counts " + "the case text, the option names, and a fixed overhead." + ) + else: + cost = ( + f"not estimated: `{adapter.model_requested}` is not in the pricing table. Pass " + "--pricing with an entry for it to cost the run in advance." + ) + cost_limit = ( + f"max cost {guard.max_cost_usd} USD" if guard.max_cost_usd is not None else "no cost limit" + ) + case_limit = f"max cases {guard.max_cases}" if guard.max_cases is not None else "no case limit" + revision = f", revision `{adapter.revision}`" if adapter.revision else "" + return "\n".join( + [ + "dry run: nothing was sent and nothing was written.", + "", + f"- {planned.cases} cases for the {adapter.name} adapter, model " + f"`{adapter.model_requested}`{revision}.", + f"- Endpoint: `{endpoint}`." if endpoint else "- Endpoint: the adapter's default.", + f"- Timeout: {timeout:g} s per request." + if timeout + else "- Timeout: the adapter's default.", + f"- Probability semantics: {semantics}.", + f"- Cost: {cost}", + f"- Guard: {cost_limit}, {case_limit}; the run would start.", + ] + ) + #: Where each hosted adapter reads its key, for the message when it is missing. _KEY_VARIABLES = {"typesafe_wire": "TYPESAFE_API_KEY", "generative": "ANTHROPIC_API_KEY"} diff --git a/src/plumbline/datasets/loader.py b/src/plumbline/datasets/loader.py index a629928..5f32c84 100644 --- a/src/plumbline/datasets/loader.py +++ b/src/plumbline/datasets/loader.py @@ -50,6 +50,56 @@ def __str__(self) -> str: return f"line {self.row}{named}: {self.reason}" +@dataclass(frozen=True) +class LoadSummary: + """What a load found, without its cases: the part a report prints. + + An artifact keeps this, so a report rendered from artifacts later still says + how many rows were read, loaded, and refused, as the one written at run time + did. The cases themselves are already in the artifact's records. + """ + + source: str + rows_read: int + loaded: int + refusals: tuple[str, ...] = () + notes: tuple[str, ...] = () + unsupported_by_type: Mapping[str, int] = field(default_factory=dict) + + def statement(self) -> str: + """One line naming all three counts, because two of them are not enough.""" + parts = [ + f"{self.rows_read} rows read from {self.source}, " + f"{self.loaded} loaded, {len(self.refusals)} refused." + ] + parts.extend(self.notes) + return " ".join(parts) + + def to_jsonable(self) -> dict[str, Any]: + return { + "source": self.source, + "rows_read": self.rows_read, + "loaded": self.loaded, + "refusals": list(self.refusals), + "notes": list(self.notes), + "unsupported_by_type": dict(self.unsupported_by_type), + } + + @classmethod + def from_jsonable(cls, stored: Mapping[str, Any]) -> LoadSummary: + return cls( + source=str(stored["source"]), + rows_read=int(stored["rows_read"]), + loaded=int(stored["loaded"]), + refusals=tuple(str(refusal) for refusal in stored.get("refusals", ())), + notes=tuple(str(note) for note in stored.get("notes", ())), + unsupported_by_type={ + str(name): int(count) + for name, count in dict(stored.get("unsupported_by_type", {})).items() + }, + ) + + @dataclass(frozen=True) class LoadReport: """What a file contained, what came out of it, and what was left behind.""" @@ -91,14 +141,20 @@ def unsupported_by_type(self) -> dict[str, int]: def is_complete(self) -> bool: return not self.refusals + def summary(self) -> LoadSummary: + """This load without its cases, as an artifact stores it.""" + return LoadSummary( + source=self.source, + rows_read=self.rows_read, + loaded=self.row_count, + refusals=tuple(str(refusal) for refusal in self.refusals), + notes=self.notes, + unsupported_by_type=self.unsupported_by_type, + ) + def statement(self) -> str: """One line naming all three counts, because two of them are not enough.""" - parts = [ - f"{self.rows_read} rows read from {self.source}, " - f"{self.row_count} loaded, {len(self.refusals)} refused." - ] - parts.extend(self.notes) - return " ".join(parts) + return self.summary().statement() def require_complete(self) -> None: """Refuse to proceed on a partial dataset, naming every row dropped.""" diff --git a/src/plumbline/report/markdown.py b/src/plumbline/report/markdown.py index c6b5be9..80e6ba6 100644 --- a/src/plumbline/report/markdown.py +++ b/src/plumbline/report/markdown.py @@ -27,7 +27,7 @@ from dataclasses import dataclass from datetime import UTC, date, datetime -from plumbline.datasets.loader import LoadReport +from plumbline.datasets.loader import LoadReport, LoadSummary from plumbline.metrics import baseline, calibration, cascade, cost, latency, recalibration from plumbline.metrics.calibration import Binning from plumbline.metrics.cost import DEFAULT_PRICING_MAX_AGE_DAYS @@ -111,7 +111,7 @@ def clock(self) -> date: def render( results: Sequence[RunResult], *, - load: LoadReport | None = None, + load: LoadReport | LoadSummary | None = None, options: ReportOptions | None = None, ) -> str: """One markdown document covering every arm, grouped and labeled.""" @@ -146,7 +146,7 @@ def render( def _header( - results: Sequence[RunResult], load: LoadReport | None, options: ReportOptions + results: Sequence[RunResult], load: LoadReport | LoadSummary | None, options: ReportOptions ) -> list[str]: first = results[0] datasets = sorted({result.dataset_hash for result in results}) @@ -215,7 +215,7 @@ def _how_to_read() -> list[str]: ] -def _dataset_section(load: LoadReport) -> list[str]: +def _dataset_section(load: LoadReport | LoadSummary) -> list[str]: lines = ["## Dataset", "", f"- {load.statement()}"] for question_type, count in load.unsupported_by_type.items(): lines.append( @@ -563,7 +563,13 @@ def _provenance(result: RunResult, options: ReportOptions) -> list[str]: hits = result.cache_stats.get("hits", 0) if hits: line += f" {hits} of {len(result.records)} rows came from cache and cost nothing." - return [line] + lines = [line] + if result.config.get("semantics_set_by") == "operator": + lines.append( + f"- **Probability semantics**: {_code(result.probability_semantics)}, set by the " + "operator with --semantics rather than declared by the adapter." + ) + return lines def _code(value: str) -> str: diff --git a/src/plumbline/runner/execute.py b/src/plumbline/runner/execute.py index e8fbde2..9d2d147 100644 --- a/src/plumbline/runner/execute.py +++ b/src/plumbline/runner/execute.py @@ -34,6 +34,7 @@ from typing import Any from plumbline.adapters.base import Adapter +from plumbline.datasets.loader import LoadSummary from plumbline.metrics.cost import ( CostBasis, Pricing, @@ -221,6 +222,13 @@ class RunResult: Carries the source and the date that entry was read, so a result opened in a year is not silently re-scored against the prices of the day it is opened. """ + load: LoadSummary | None = None + """What loading the dataset found: rows read, loaded, and refused. + + Kept so a report rendered from this artifact later prints the same Dataset + section as the one written at run time. None for a run started without a + load, and for artifacts written before this was stored. + """ @property def successes(self) -> list[CaseRecord]: @@ -288,6 +296,7 @@ def to_jsonable(self) -> dict[str, Any]: "pricing": self.pricing, "config": self.config, "cache_stats": self.cache_stats, + "load": self.load.to_jsonable() if self.load is not None else None, "records": [ { "case_id": record.case_id, @@ -369,6 +378,7 @@ def _from_stored(cls, stored: dict[str, Any], path: Path) -> RunResult: records=[_record_from_jsonable(row) for row in stored["records"]], cache_stats=stored.get("cache_stats", {}), pricing=stored.get("pricing"), + load=LoadSummary.from_jsonable(stored["load"]) if stored.get("load") else None, ) def write(self, directory: Path | str) -> Path: @@ -516,6 +526,39 @@ def check_guard( return estimate +@dataclass(frozen=True) +class Plan: + """What a run would do, decided before anything is sent.""" + + cases: int + estimated_cost_usd: float | None + """None when the requested model is not in the pricing table.""" + pricing_key: str | None + pricing: Pricing | None + + +def plan( + adapter: Adapter, + cases: Sequence[Case], + *, + guard: CostGuard | None = None, + pricing_table: PricingTable | None = None, +) -> Plan: + """Price a run and apply its guard, sending nothing. + + ``run`` starts here, so a dry run refuses exactly what the run would refuse + and prints the estimate the run would check against its limit. The guard can + only price the requested model, since nothing has answered yet. + """ + if not cases: + raise ValueError("no cases to run") + pricing, pricing_key = pricing_for(pricing_table or {}, None, adapter.model_requested) + estimate = check_guard(cases, guard or CostGuard(), pricing) + return Plan( + cases=len(cases), estimated_cost_usd=estimate, pricing_key=pricing_key, pricing=pricing + ) + + def run( adapter: Adapter, cases: Sequence[Case], @@ -526,6 +569,7 @@ def run( workers: int = DEFAULT_WORKERS, retry: RetryPolicy | None = None, extra_config: Mapping[str, Any] | None = None, + load: LoadSummary | None = None, ) -> RunResult: """Classify every case, in order, with the guard checked before anything is sent.""" if not cases: @@ -537,8 +581,8 @@ def run( guard = guard or CostGuard() table: PricingTable = pricing_table or {} today = datetime.now(UTC).date() - pricing, pricing_key = pricing_for(table, None, adapter.model_requested) - estimate = check_guard(cases, guard, pricing) + planned = plan(adapter, cases, guard=guard, pricing_table=table) + estimate, pricing_key = planned.estimated_cost_usd, planned.pricing_key records: list[CaseRecord | None] = [None] * len(cases) locks = _KeyLocks() if cache is not None and cache.enabled else None @@ -583,6 +627,10 @@ def handle(index: int) -> None: # The server that answered, when it is not the vendor's default. It is # part of what was measured: a self-hosted endpoint is a different system. "endpoint": getattr(adapter, "base_url", None), + # How long one request was allowed, when the adapter was given a limit. + # A tight timeout turns slow answers into failures, which is part of + # what the run measured. + "timeout_seconds": getattr(adapter, "timeout", None), # Whether rows carried option descriptions and whether they were sent, # so the report can say when the dataset's descriptions went nowhere. "label_descriptions": { @@ -609,6 +657,7 @@ def handle(index: int) -> None: config=redact(config), records=finished, cache_stats=cache.stats if cache is not None else {}, + load=load, ) diff --git a/tests/test_cli_run_options.py b/tests/test_cli_run_options.py new file mode 100644 index 0000000..2ed695f --- /dev/null +++ b/tests/test_cli_run_options.py @@ -0,0 +1,230 @@ +"""The run options that change what is sent, and the dry run that sends nothing. + +Everything here stays off the network. The hosted adapters are built with a +dummy key and only ever reach the dry run, which returns before any call. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +from plumbline import cli +from plumbline.adapters.mock import MockAdapter +from plumbline.datasets.loader import LoadSummary +from plumbline.runner import execute +from plumbline.types import Case + +runner = CliRunner() + + +def a_dataset(path: Path, n_rows: int = 12, broken_rows: int = 0) -> Path: + labels = ["billing", "returns", "shipping", "other"] + lines = [ + json.dumps( + { + "id": f"case-{index}", + "text": f"ticket body number {index}", + "labels": labels, + "gold_label": labels[index % len(labels)], + } + ) + for index in range(n_rows) + ] + # A gold label that is not among the row's options is refused on load. + lines += [ + json.dumps({"id": f"bad-{index}", "text": "t", "labels": ["a", "b"], "gold_label": "c"}) + for index in range(broken_rows) + ] + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + return path + + +def invoke(*args: str): + return runner.invoke(cli.app, list(args), catch_exceptions=False) + + +@pytest.fixture +def no_calls(monkeypatch: pytest.MonkeyPatch) -> None: + """Any classify call fails the test: a dry run must not make one.""" + + def refuse(*_args: object, **_kwargs: object) -> None: + raise AssertionError("a dry run called the adapter") + + monkeypatch.setattr(MockAdapter, "classify", refuse) + + +def test_a_dry_run_prices_the_run_and_sends_nothing(tmp_path: Path, no_calls: None) -> None: + dataset = a_dataset(tmp_path / "d.jsonl") + results = tmp_path / "results" + + done = invoke("run", str(dataset), "--dry-run", "--results", str(results)) + + assert done.exit_code == 0, done.stderr + assert "dry run" in done.stdout and "nothing was sent" in done.stdout + assert "12 cases" in done.stdout + assert "not in the pricing table" in done.stdout + assert not results.exists(), "a dry run wrote an artifact" + + +def test_a_dry_run_names_the_estimate_and_the_entry_that_priced_it( + tmp_path: Path, no_calls: None +) -> None: + dataset = a_dataset(tmp_path / "d.jsonl") + pricing = tmp_path / "pricing.json" + pricing.write_text( + json.dumps( + { + "demo-model": { + "input_usd_per_million": 1.0, + "output_usd_per_million": 2.0, + "source": "https://example.invalid/prices", + "as_of": "2026-09-01", + } + } + ), + encoding="utf-8", + ) + + done = invoke( + "run", str(dataset), "--dry-run", "--model", "demo-model", "--pricing", str(pricing) + ) + + assert done.exit_code == 0, done.stderr + assert "USD" in done.stdout and "`demo-model`" in done.stdout + assert "2026-09-01" in done.stdout + + +def test_a_dry_run_refuses_what_the_run_would_refuse(tmp_path: Path, no_calls: None) -> None: + dataset = a_dataset(tmp_path / "d.jsonl") + + done = invoke("run", str(dataset), "--dry-run", "--max-cases", "5") + + assert done.exit_code == 1 + assert "max_cases is 5" in done.stderr + + +def test_an_endpoint_and_a_timeout_reach_the_adapter_and_the_plan( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("TYPESAFE_API_KEY", "dummy-not-a-real-key") + monkeypatch.delenv("TYPESAFE_BASE_URL", raising=False) + dataset = a_dataset(tmp_path / "d.jsonl") + + done = invoke( + "run", + str(dataset), + "--adapter", + "typesafe_wire", + "--base-url", + "http://localhost:8911", + "--timeout", + "12.5", + "--semantics", + "restricted_softmax", + "--dry-run", + ) + + assert done.exit_code == 0, done.stderr + assert "`http://localhost:8911`" in done.stdout + assert "12.5 s" in done.stdout + assert "restricted_softmax, set by --semantics" in done.stdout + assert "dummy-not-a-real-key" not in done.stdout + done.stderr + + +@pytest.mark.parametrize( + ("adapter", "flag", "value"), + [ + ("mock", "--base-url", "http://localhost:1"), + ("mock", "--timeout", "5"), + ("generative", "--semantics", "calibrated_claim"), + ], +) +def test_a_setting_the_adapter_does_not_take_is_named_exactly( + tmp_path: Path, adapter: str, flag: str, value: str +) -> None: + dataset = a_dataset(tmp_path / "d.jsonl") + + done = invoke("run", str(dataset), "--adapter", adapter, flag, value, "--dry-run") + + assert done.exit_code == 1 + assert f"the {adapter} adapter does not take {flag}" in done.stderr + + +def test_a_timeout_must_be_positive(tmp_path: Path) -> None: + dataset = a_dataset(tmp_path / "d.jsonl") + + done = invoke("run", str(dataset), "--timeout", "0", "--dry-run") + + assert done.exit_code == 1 + assert "--timeout" in done.stderr + + +def test_a_report_rebuilt_from_artifacts_keeps_its_dataset_section(tmp_path: Path) -> None: + dataset = a_dataset(tmp_path / "d.jsonl", broken_rows=2) + results = tmp_path / "results" + at_run = tmp_path / "at-run.md" + invoke("run", str(dataset), "--results", str(results), "--report", str(at_run), "--boot", "50") + artifact = next(results.glob("*.json")) + + rebuilt = invoke("report", str(artifact), "--boot", "50") + + assert rebuilt.exit_code == 0, rebuilt.stderr + + def dataset_section(text: str) -> str: + return text.split("## Dataset", 1)[1].split("\n## ", 1)[0] + + written = at_run.read_text(encoding="utf-8") + assert "14 rows read" in dataset_section(written) and "2 refused" in dataset_section(written) + assert dataset_section(rebuilt.stdout) == dataset_section(written) + + +def test_an_artifact_from_before_the_load_was_stored_still_reports(tmp_path: Path) -> None: + dataset = a_dataset(tmp_path / "d.jsonl") + results = tmp_path / "results" + invoke("run", str(dataset), "--results", str(results), "--boot", "50") + artifact = next(results.glob("*.json")) + stored = json.loads(artifact.read_text(encoding="utf-8")) + del stored["load"] + artifact.write_text(json.dumps(stored), encoding="utf-8") + + rebuilt = invoke("report", str(artifact), "--boot", "50") + + assert rebuilt.exit_code == 0, rebuilt.stderr + assert "## Dataset" not in rebuilt.stdout + + +def test_the_load_summary_round_trips_through_the_artifact(tmp_path: Path) -> None: + summary = LoadSummary( + source="d.jsonl", + rows_read=5, + loaded=4, + refusals=("line 5: gold label 'c' is not among the options",), + notes=("a note.",), + unsupported_by_type={"score": 1}, + ) + cases = [Case(id=f"c{i}", text=f"t{i}", labels=("a", "b"), gold_label="a") for i in range(3)] + result = execute.run( + MockAdapter(gold_by_text={case.text: case.gold_label for case in cases}), + cases, + load=summary, + ) + + reread = execute.RunResult.read(result.write(tmp_path)) + + assert reread.load == summary + assert reread.load is not None and reread.load.statement() == summary.statement() + + +def test_the_timeout_an_adapter_used_is_in_the_artifact() -> None: + class Patient(MockAdapter): + timeout = 42.0 + + cases = [Case(id="c", text="t", labels=("a", "b"), gold_label="a")] + + result = execute.run(Patient(gold_by_text={"t": "a"}), cases) + + assert result.config["timeout_seconds"] == 42.0