diff --git a/.env.example b/.env.example index c4cdcd5..0fe43b3 100644 --- a/.env.example +++ b/.env.example @@ -18,7 +18,8 @@ TYPESAFE_API_KEY= # endpoint instead of the hosted service. The endpoint is part of the cache key, # so a self-hosted run is never served a hosted run's answers, and it is # recorded in the artifact and named in the report. ANTHROPIC_BASE_URL does the -# same for the generative arm. +# same for the generative arm. `plumbline run --base-url` sets it for one run +# and takes precedence over either variable. TYPESAFE_BASE_URL= # Generative control arm, which speaks Anthropic's Messages API. Only needed if diff --git a/CHANGELOG.md b/CHANGELOG.md index 06f6aa9..4c4fb30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,23 @@ different event from one that moved because it was wrong. ### Added +- `plumbline run --dry-run` loads and checks the dataset, builds the adapter, + applies the cost guard, and prints the case count, the endpoint, the timeout, + the semantics, and the estimated cost with the pricing entry behind it, then + exits without sending a request or writing a file (#58). It refuses exactly + what the run would refuse, because the run starts from the same plan. +- `--base-url` and `--timeout` on `plumbline run`, for an adapter that takes + them (#58). The endpoint could be changed only through an environment + variable before; it is recorded in the artifact and the cache key either + way, and the timeout is now recorded too. `--semantics` now overrides the + declared semantics for any adapter that takes it, not only the mock, and the + report says when the operator set it. An option an adapter does not take is + refused by name before anything is built. +- An artifact now stores what loading the dataset found, so a report rebuilt + with `plumbline report` prints the same Dataset section, with the read, + loaded, and refused counts, as the one written at run time (#58). Artifacts + written before this still report, without the section. + - A browser calculator for the ECE floor (`site/`), a JavaScript port of `synthetic_floor` that reproduces numpy's seeded random stream draw for draw. `scripts/floor_golden.py` exports golden values from the Python and diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 650bd8e..b82cc53 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -9,7 +9,8 @@ one of them. Adding a vendor should not add a file. - **Anything serving the Jev wire format is a `base_url` for `typesafe_wire`.** A self-hosted endpoint, a compatible server in front of an open model, a - provider that implemented the same shape. It is a config entry. That includes + provider that implemented the same shape. It is `plumbline run --base-url`, + recorded in the artifact and part of the cache key. That includes the open decision models that serve this wire format: the unchanged adapter runs them. - **Any open-weights checkpoint is a config entry for `local_logits`**, as a diff --git a/README.md b/README.md index 611dd95..c824694 100644 --- a/README.md +++ b/README.md @@ -237,6 +237,13 @@ Without a pricing table the run still works; cost reports as unpriced, and `--max-cost-usd` refuses rather than bounding a run it cannot cost. See [Limitations](#limitations). +Add `--dry-run` to either command first to see what the run would do without +spending anything. It loads and checks the dataset, builds the adapter, applies +the guard, and prints the case count, the endpoint, and the estimated cost, then +exits without sending a request or writing a file. Pass `--base-url` to point +`typesafe_wire` at a self-hosted endpoint, and `--timeout` to bound one request; +both are recorded in the artifact. + Your own data goes in `datasets/private/`, which is gitignored, and that is the only path on which the recalibration numbers mean anything. [Your own data](docs/datasets.md) gives the row format, what is refused and why, @@ -290,7 +297,7 @@ and self-hosted servers speak it, which is why one adapter covers all of them. | Adapter | Transport | Semantics | Run for real | Adding one | |---|---|---|---|---| -| `typesafe_wire` | Jev wire format over HTTP | `calibrated_claim` | Yes, 40 rows against a hosted vendor | A `base_url`. Anything serving the same wire format is a config entry, including self-hosted endpoints and open models behind a compatible server. | +| `typesafe_wire` | Jev wire format over HTTP | `calibrated_claim` | Yes, 40 rows against a hosted vendor | `--base-url`. Anything serving the same wire format is one flag, recorded in the artifact, including self-hosted endpoints and open models behind a compatible server. | | `local_logits` | Option-token logits from a local checkpoint | `restricted_softmax` | **No, tests only** | A HuggingFace model id and a pinned revision. Needs the optional `local` extra. | | `generative` | Chat completion, parsed | `none` | **No, tests only** | A model string. | | `mock` | None, seeded | configurable | Yes, it is the example report | Built in. A deterministic stand-in, not a system under test. | @@ -369,8 +376,8 @@ Specific, and none of them are going to surprise you later. - **[Mapika/decider](https://github.com/Mapika/decider)**. One-pass typed decisions with calibrated probabilities, fine-tuned from Qwen3.5-2B, in several sizes. Its own guidance is to check calibration on your own labels before - routing on confidence. It serves the same typed question shape, so it is a - `base_url` config entry here rather than new code. + routing on confidence. It serves the same typed question shape, so it is + `--base-url` on `typesafe_wire` here rather than new code. - **[Bespoke Nimble](https://github.com/bespokelabsai/nimble)**. An open recipe for typed decision models, a LoRA fine-tune on Qwen3.5-9B, trained with contrastive data curation. It serves the Jev wire format, so the unchanged diff --git a/src/plumbline/adapters/registry.py b/src/plumbline/adapters/registry.py index 25ddbff..6e28cdd 100644 --- a/src/plumbline/adapters/registry.py +++ b/src/plumbline/adapters/registry.py @@ -110,6 +110,22 @@ def _missing_arguments(factory: AdapterFactory, config: dict[str, Any]) -> list[ ] +def accepts(name: str, setting: str) -> bool: + """Whether the adapter registered as ``name`` takes ``setting`` when it is built. + + Imports a built-in to read its signature, as ``create`` would, so a caller can + name the one setting an adapter refuses before trying to build it. + """ + try: + parameters = inspect.signature(_factory(name)).parameters.values() + except (TypeError, ValueError): # no signature to read, so let create decide + return True + return any( + parameter.name == setting or parameter.kind is inspect.Parameter.VAR_KEYWORD + for parameter in parameters + ) + + def available() -> tuple[str, ...]: """Registered adapter names, sorted, without importing any of them.""" return tuple(sorted({*_REGISTRY, *_BUILTINS})) diff --git a/src/plumbline/cli.py b/src/plumbline/cli.py index dca08e7..9e7eb1e 100644 --- a/src/plumbline/cli.py +++ b/src/plumbline/cli.py @@ -76,13 +76,34 @@ def run( "truncation: use --limit to run fewer.", ), ] = None, + base_url: Annotated[ + str | None, + typer.Option( + "--base-url", + help="Send requests here instead of the vendor's endpoint, for an adapter that " + "takes one. Recorded in the artifact and part of the cache key.", + ), + ] = None, + timeout: Annotated[ + float | None, + typer.Option( + "--timeout", help="Seconds one request may take, for an adapter that takes it." + ), + ] = None, semantics: Annotated[ str | None, typer.Option( - help="Override probability_semantics, mock only: one of " - f"{', '.join(PROBABILITY_SEMANTICS)}." + help="Override the probability_semantics the adapter declares, for one that " + f"takes it: one of {', '.join(PROBABILITY_SEMANTICS)}. The report says so." ), ] = None, + dry_run: Annotated[ + bool, + typer.Option( + "--dry-run", + help="Load, check, and price the run, print what it would do, and send nothing.", + ), + ] = False, seed: Annotated[int, typer.Option(help="Mock seed.")] = 7, accuracy: Annotated[float, typer.Option(help="Mock target accuracy.")] = 0.8, escalation_cost: Annotated[ @@ -102,12 +123,15 @@ def run( Status lines and errors go to stderr, so stdout carries only the report when no --report is given. The exit code is 1 when no case produced a prediction, - after the artifact and the report are written. + after the artifact and the report are written. With --dry-run, stdout carries + the plan instead, and nothing is sent or written. """ # Everything that can be checked before a call goes out is checked here, # so a mistake costs nothing. if report is not None and report.is_dir(): _fail(f"--report {report} is a directory; give it a file path, such as {report}/report.md") + if timeout is not None and not timeout > 0: + _fail(f"--timeout must be a number of seconds above 0, got {timeout}") load = dataclasses.replace(_load(dataset, data_format), source=_shown(dataset)) _status(load.statement()) for refusal in load.refusals: @@ -124,19 +148,32 @@ def run( cases, model=model, revision=revision, + base_url=base_url, + timeout=timeout, semantics=semantics, seed=seed, accuracy=accuracy, ) + guard = execute.CostGuard(max_cost_usd=max_cost_usd, max_cases=max_cases) + table = _pricing(pricing) + if dry_run: + planned = _guard(lambda: execute.plan(built, cases, guard=guard, pricing_table=table)) + typer.echo(_plan_text(built, planned, guard, semantics_set=semantics is not None)) + return + + extra: dict[str, object] = {"dataset": _shown(dataset), "format": data_format} + if semantics is not None: + extra["semantics_set_by"] = "operator" result = _guard( lambda: execute.run( built, cases, cache=Cache(cache_dir) if cache_dir else None, - guard=execute.CostGuard(max_cost_usd=max_cost_usd, max_cases=max_cases), - pricing_table=_pricing(pricing), + guard=guard, + pricing_table=table, workers=workers, - extra_config={"dataset": _shown(dataset), "format": data_format}, + extra_config=extra, + load=load.summary(), ) ) @@ -192,6 +229,7 @@ def report( document = _guard( lambda: markdown.render( results, + load=_stored_load(results), options=markdown.ReportOptions( n_boot=n_boot, cost_escalation_usd=escalation_cost, @@ -208,6 +246,20 @@ def report( typer.echo(document) +def _stored_load(results: list[execute.RunResult]) -> loader.LoadSummary | None: + """The Dataset section a rebuilt report can print, from what the runs stored. + + Only when every run is over one dataset and they all say the same thing + about loading it. Otherwise there is no single section that is true of the + document, and printing one run's counts over another's figures would be + worse than printing none. + """ + if len({result.dataset_hash for result in results}) != 1: + return None + first = results[0].load + return first if all(result.load == first for result in results) else None + + @app.command() def adapters() -> None: """List the transports this install can run.""" @@ -253,6 +305,8 @@ def _build( *, model: str | None, revision: str | None, + base_url: str | None = None, + timeout: float | None = None, semantics: str | None, seed: int, accuracy: float, @@ -263,6 +317,12 @@ def _build( config["model_requested"] = model if revision is not None: config["revision"] = revision + if base_url is not None: + config["base_url"] = base_url + if timeout is not None: + config["timeout"] = timeout + if semantics is not None: + config["probability_semantics"] = _semantics(semantics) if adapter == "mock": # The mock is told the answer key up front, because classify() is never @@ -270,17 +330,23 @@ def _build( config["gold_by_text"] = {case.text: case.gold_label for case in cases} config["seed"] = seed config["accuracy"] = accuracy - if semantics is not None: - config["probability_semantics"] = _semantics(semantics) - elif semantics is not None: - _fail("--semantics applies to the mock only; a real adapter declares its own.") + + # Name the exact option an adapter will not take, before building it: the + # generative arm reports no probability, so it has no semantics to override. + refused_options = [ + flag + for setting, flag in _OPTIONS.items() + if setting in config and not _guard(partial(registry.accepts, adapter, setting)) + ] + if refused_options: + _fail(f"the {adapter} adapter does not take {', '.join(refused_options)}.") try: return registry.create(adapter, **config) except (PlumblineError, ValueError) as refused: _fail(str(refused)) except TypeError: - given = [f"--{name.replace('_requested', '')}" for name in config if name in _OPTIONS] + given = [flag for setting, flag in _OPTIONS.items() if setting in config] _fail(f"the {adapter} adapter does not take {', '.join(given) or 'these settings'}.") except Exception as unavailable: # an SDK that cannot start, such as a missing key variable = _KEY_VARIABLES.get(adapter) @@ -288,8 +354,56 @@ def _build( _fail(f"could not set up the {adapter} adapter: {unavailable}.{hint}") -#: Settings the CLI passes to an adapter from its own options. -_OPTIONS = ("model_requested", "revision") +#: Settings the CLI passes to an adapter from its own options, and the option +#: each comes from. +_OPTIONS = { + "model_requested": "--model", + "revision": "--revision", + "base_url": "--base-url", + "timeout": "--timeout", + "probability_semantics": "--semantics", +} + + +def _plan_text( + adapter: Adapter, planned: execute.Plan, guard: execute.CostGuard, *, semantics_set: bool +) -> str: + """What a dry run prints: what would be sent, to whom, and what it would cost.""" + endpoint = getattr(adapter, "base_url", None) + timeout = getattr(adapter, "timeout", None) + semantics = adapter.probability_semantics + (", set by --semantics" if semantics_set else "") + if planned.estimated_cost_usd is not None and planned.pricing is not None: + cost = ( + f"about {planned.estimated_cost_usd:.4f} USD, priced by `{planned.pricing_key}` " + f"as read on {planned.pricing.as_of.isoformat()}. The estimate is rough: it counts " + "the case text, the option names, and a fixed overhead." + ) + else: + cost = ( + f"not estimated: `{adapter.model_requested}` is not in the pricing table. Pass " + "--pricing with an entry for it to cost the run in advance." + ) + cost_limit = ( + f"max cost {guard.max_cost_usd} USD" if guard.max_cost_usd is not None else "no cost limit" + ) + case_limit = f"max cases {guard.max_cases}" if guard.max_cases is not None else "no case limit" + revision = f", revision `{adapter.revision}`" if adapter.revision else "" + return "\n".join( + [ + "dry run: nothing was sent and nothing was written.", + "", + f"- {planned.cases} cases for the {adapter.name} adapter, model " + f"`{adapter.model_requested}`{revision}.", + f"- Endpoint: `{endpoint}`." if endpoint else "- Endpoint: the adapter's default.", + f"- Timeout: {timeout:g} s per request." + if timeout + else "- Timeout: the adapter's default.", + f"- Probability semantics: {semantics}.", + f"- Cost: {cost}", + f"- Guard: {cost_limit}, {case_limit}; the run would start.", + ] + ) + #: Where each hosted adapter reads its key, for the message when it is missing. _KEY_VARIABLES = {"typesafe_wire": "TYPESAFE_API_KEY", "generative": "ANTHROPIC_API_KEY"} diff --git a/src/plumbline/datasets/loader.py b/src/plumbline/datasets/loader.py index a629928..5f32c84 100644 --- a/src/plumbline/datasets/loader.py +++ b/src/plumbline/datasets/loader.py @@ -50,6 +50,56 @@ def __str__(self) -> str: return f"line {self.row}{named}: {self.reason}" +@dataclass(frozen=True) +class LoadSummary: + """What a load found, without its cases: the part a report prints. + + An artifact keeps this, so a report rendered from artifacts later still says + how many rows were read, loaded, and refused, as the one written at run time + did. The cases themselves are already in the artifact's records. + """ + + source: str + rows_read: int + loaded: int + refusals: tuple[str, ...] = () + notes: tuple[str, ...] = () + unsupported_by_type: Mapping[str, int] = field(default_factory=dict) + + def statement(self) -> str: + """One line naming all three counts, because two of them are not enough.""" + parts = [ + f"{self.rows_read} rows read from {self.source}, " + f"{self.loaded} loaded, {len(self.refusals)} refused." + ] + parts.extend(self.notes) + return " ".join(parts) + + def to_jsonable(self) -> dict[str, Any]: + return { + "source": self.source, + "rows_read": self.rows_read, + "loaded": self.loaded, + "refusals": list(self.refusals), + "notes": list(self.notes), + "unsupported_by_type": dict(self.unsupported_by_type), + } + + @classmethod + def from_jsonable(cls, stored: Mapping[str, Any]) -> LoadSummary: + return cls( + source=str(stored["source"]), + rows_read=int(stored["rows_read"]), + loaded=int(stored["loaded"]), + refusals=tuple(str(refusal) for refusal in stored.get("refusals", ())), + notes=tuple(str(note) for note in stored.get("notes", ())), + unsupported_by_type={ + str(name): int(count) + for name, count in dict(stored.get("unsupported_by_type", {})).items() + }, + ) + + @dataclass(frozen=True) class LoadReport: """What a file contained, what came out of it, and what was left behind.""" @@ -91,14 +141,20 @@ def unsupported_by_type(self) -> dict[str, int]: def is_complete(self) -> bool: return not self.refusals + def summary(self) -> LoadSummary: + """This load without its cases, as an artifact stores it.""" + return LoadSummary( + source=self.source, + rows_read=self.rows_read, + loaded=self.row_count, + refusals=tuple(str(refusal) for refusal in self.refusals), + notes=self.notes, + unsupported_by_type=self.unsupported_by_type, + ) + def statement(self) -> str: """One line naming all three counts, because two of them are not enough.""" - parts = [ - f"{self.rows_read} rows read from {self.source}, " - f"{self.row_count} loaded, {len(self.refusals)} refused." - ] - parts.extend(self.notes) - return " ".join(parts) + return self.summary().statement() def require_complete(self) -> None: """Refuse to proceed on a partial dataset, naming every row dropped.""" diff --git a/src/plumbline/report/markdown.py b/src/plumbline/report/markdown.py index c6b5be9..80e6ba6 100644 --- a/src/plumbline/report/markdown.py +++ b/src/plumbline/report/markdown.py @@ -27,7 +27,7 @@ from dataclasses import dataclass from datetime import UTC, date, datetime -from plumbline.datasets.loader import LoadReport +from plumbline.datasets.loader import LoadReport, LoadSummary from plumbline.metrics import baseline, calibration, cascade, cost, latency, recalibration from plumbline.metrics.calibration import Binning from plumbline.metrics.cost import DEFAULT_PRICING_MAX_AGE_DAYS @@ -111,7 +111,7 @@ def clock(self) -> date: def render( results: Sequence[RunResult], *, - load: LoadReport | None = None, + load: LoadReport | LoadSummary | None = None, options: ReportOptions | None = None, ) -> str: """One markdown document covering every arm, grouped and labeled.""" @@ -146,7 +146,7 @@ def render( def _header( - results: Sequence[RunResult], load: LoadReport | None, options: ReportOptions + results: Sequence[RunResult], load: LoadReport | LoadSummary | None, options: ReportOptions ) -> list[str]: first = results[0] datasets = sorted({result.dataset_hash for result in results}) @@ -215,7 +215,7 @@ def _how_to_read() -> list[str]: ] -def _dataset_section(load: LoadReport) -> list[str]: +def _dataset_section(load: LoadReport | LoadSummary) -> list[str]: lines = ["## Dataset", "", f"- {load.statement()}"] for question_type, count in load.unsupported_by_type.items(): lines.append( @@ -563,7 +563,13 @@ def _provenance(result: RunResult, options: ReportOptions) -> list[str]: hits = result.cache_stats.get("hits", 0) if hits: line += f" {hits} of {len(result.records)} rows came from cache and cost nothing." - return [line] + lines = [line] + if result.config.get("semantics_set_by") == "operator": + lines.append( + f"- **Probability semantics**: {_code(result.probability_semantics)}, set by the " + "operator with --semantics rather than declared by the adapter." + ) + return lines def _code(value: str) -> str: diff --git a/src/plumbline/runner/execute.py b/src/plumbline/runner/execute.py index e8fbde2..9d2d147 100644 --- a/src/plumbline/runner/execute.py +++ b/src/plumbline/runner/execute.py @@ -34,6 +34,7 @@ from typing import Any from plumbline.adapters.base import Adapter +from plumbline.datasets.loader import LoadSummary from plumbline.metrics.cost import ( CostBasis, Pricing, @@ -221,6 +222,13 @@ class RunResult: Carries the source and the date that entry was read, so a result opened in a year is not silently re-scored against the prices of the day it is opened. """ + load: LoadSummary | None = None + """What loading the dataset found: rows read, loaded, and refused. + + Kept so a report rendered from this artifact later prints the same Dataset + section as the one written at run time. None for a run started without a + load, and for artifacts written before this was stored. + """ @property def successes(self) -> list[CaseRecord]: @@ -288,6 +296,7 @@ def to_jsonable(self) -> dict[str, Any]: "pricing": self.pricing, "config": self.config, "cache_stats": self.cache_stats, + "load": self.load.to_jsonable() if self.load is not None else None, "records": [ { "case_id": record.case_id, @@ -369,6 +378,7 @@ def _from_stored(cls, stored: dict[str, Any], path: Path) -> RunResult: records=[_record_from_jsonable(row) for row in stored["records"]], cache_stats=stored.get("cache_stats", {}), pricing=stored.get("pricing"), + load=LoadSummary.from_jsonable(stored["load"]) if stored.get("load") else None, ) def write(self, directory: Path | str) -> Path: @@ -516,6 +526,39 @@ def check_guard( return estimate +@dataclass(frozen=True) +class Plan: + """What a run would do, decided before anything is sent.""" + + cases: int + estimated_cost_usd: float | None + """None when the requested model is not in the pricing table.""" + pricing_key: str | None + pricing: Pricing | None + + +def plan( + adapter: Adapter, + cases: Sequence[Case], + *, + guard: CostGuard | None = None, + pricing_table: PricingTable | None = None, +) -> Plan: + """Price a run and apply its guard, sending nothing. + + ``run`` starts here, so a dry run refuses exactly what the run would refuse + and prints the estimate the run would check against its limit. The guard can + only price the requested model, since nothing has answered yet. + """ + if not cases: + raise ValueError("no cases to run") + pricing, pricing_key = pricing_for(pricing_table or {}, None, adapter.model_requested) + estimate = check_guard(cases, guard or CostGuard(), pricing) + return Plan( + cases=len(cases), estimated_cost_usd=estimate, pricing_key=pricing_key, pricing=pricing + ) + + def run( adapter: Adapter, cases: Sequence[Case], @@ -526,6 +569,7 @@ def run( workers: int = DEFAULT_WORKERS, retry: RetryPolicy | None = None, extra_config: Mapping[str, Any] | None = None, + load: LoadSummary | None = None, ) -> RunResult: """Classify every case, in order, with the guard checked before anything is sent.""" if not cases: @@ -537,8 +581,8 @@ def run( guard = guard or CostGuard() table: PricingTable = pricing_table or {} today = datetime.now(UTC).date() - pricing, pricing_key = pricing_for(table, None, adapter.model_requested) - estimate = check_guard(cases, guard, pricing) + planned = plan(adapter, cases, guard=guard, pricing_table=table) + estimate, pricing_key = planned.estimated_cost_usd, planned.pricing_key records: list[CaseRecord | None] = [None] * len(cases) locks = _KeyLocks() if cache is not None and cache.enabled else None @@ -583,6 +627,10 @@ def handle(index: int) -> None: # The server that answered, when it is not the vendor's default. It is # part of what was measured: a self-hosted endpoint is a different system. "endpoint": getattr(adapter, "base_url", None), + # How long one request was allowed, when the adapter was given a limit. + # A tight timeout turns slow answers into failures, which is part of + # what the run measured. + "timeout_seconds": getattr(adapter, "timeout", None), # Whether rows carried option descriptions and whether they were sent, # so the report can say when the dataset's descriptions went nowhere. "label_descriptions": { @@ -609,6 +657,7 @@ def handle(index: int) -> None: config=redact(config), records=finished, cache_stats=cache.stats if cache is not None else {}, + load=load, ) diff --git a/tests/test_cli_run_options.py b/tests/test_cli_run_options.py new file mode 100644 index 0000000..2ed695f --- /dev/null +++ b/tests/test_cli_run_options.py @@ -0,0 +1,230 @@ +"""The run options that change what is sent, and the dry run that sends nothing. + +Everything here stays off the network. The hosted adapters are built with a +dummy key and only ever reach the dry run, which returns before any call. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +from plumbline import cli +from plumbline.adapters.mock import MockAdapter +from plumbline.datasets.loader import LoadSummary +from plumbline.runner import execute +from plumbline.types import Case + +runner = CliRunner() + + +def a_dataset(path: Path, n_rows: int = 12, broken_rows: int = 0) -> Path: + labels = ["billing", "returns", "shipping", "other"] + lines = [ + json.dumps( + { + "id": f"case-{index}", + "text": f"ticket body number {index}", + "labels": labels, + "gold_label": labels[index % len(labels)], + } + ) + for index in range(n_rows) + ] + # A gold label that is not among the row's options is refused on load. + lines += [ + json.dumps({"id": f"bad-{index}", "text": "t", "labels": ["a", "b"], "gold_label": "c"}) + for index in range(broken_rows) + ] + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + return path + + +def invoke(*args: str): + return runner.invoke(cli.app, list(args), catch_exceptions=False) + + +@pytest.fixture +def no_calls(monkeypatch: pytest.MonkeyPatch) -> None: + """Any classify call fails the test: a dry run must not make one.""" + + def refuse(*_args: object, **_kwargs: object) -> None: + raise AssertionError("a dry run called the adapter") + + monkeypatch.setattr(MockAdapter, "classify", refuse) + + +def test_a_dry_run_prices_the_run_and_sends_nothing(tmp_path: Path, no_calls: None) -> None: + dataset = a_dataset(tmp_path / "d.jsonl") + results = tmp_path / "results" + + done = invoke("run", str(dataset), "--dry-run", "--results", str(results)) + + assert done.exit_code == 0, done.stderr + assert "dry run" in done.stdout and "nothing was sent" in done.stdout + assert "12 cases" in done.stdout + assert "not in the pricing table" in done.stdout + assert not results.exists(), "a dry run wrote an artifact" + + +def test_a_dry_run_names_the_estimate_and_the_entry_that_priced_it( + tmp_path: Path, no_calls: None +) -> None: + dataset = a_dataset(tmp_path / "d.jsonl") + pricing = tmp_path / "pricing.json" + pricing.write_text( + json.dumps( + { + "demo-model": { + "input_usd_per_million": 1.0, + "output_usd_per_million": 2.0, + "source": "https://example.invalid/prices", + "as_of": "2026-09-01", + } + } + ), + encoding="utf-8", + ) + + done = invoke( + "run", str(dataset), "--dry-run", "--model", "demo-model", "--pricing", str(pricing) + ) + + assert done.exit_code == 0, done.stderr + assert "USD" in done.stdout and "`demo-model`" in done.stdout + assert "2026-09-01" in done.stdout + + +def test_a_dry_run_refuses_what_the_run_would_refuse(tmp_path: Path, no_calls: None) -> None: + dataset = a_dataset(tmp_path / "d.jsonl") + + done = invoke("run", str(dataset), "--dry-run", "--max-cases", "5") + + assert done.exit_code == 1 + assert "max_cases is 5" in done.stderr + + +def test_an_endpoint_and_a_timeout_reach_the_adapter_and_the_plan( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("TYPESAFE_API_KEY", "dummy-not-a-real-key") + monkeypatch.delenv("TYPESAFE_BASE_URL", raising=False) + dataset = a_dataset(tmp_path / "d.jsonl") + + done = invoke( + "run", + str(dataset), + "--adapter", + "typesafe_wire", + "--base-url", + "http://localhost:8911", + "--timeout", + "12.5", + "--semantics", + "restricted_softmax", + "--dry-run", + ) + + assert done.exit_code == 0, done.stderr + assert "`http://localhost:8911`" in done.stdout + assert "12.5 s" in done.stdout + assert "restricted_softmax, set by --semantics" in done.stdout + assert "dummy-not-a-real-key" not in done.stdout + done.stderr + + +@pytest.mark.parametrize( + ("adapter", "flag", "value"), + [ + ("mock", "--base-url", "http://localhost:1"), + ("mock", "--timeout", "5"), + ("generative", "--semantics", "calibrated_claim"), + ], +) +def test_a_setting_the_adapter_does_not_take_is_named_exactly( + tmp_path: Path, adapter: str, flag: str, value: str +) -> None: + dataset = a_dataset(tmp_path / "d.jsonl") + + done = invoke("run", str(dataset), "--adapter", adapter, flag, value, "--dry-run") + + assert done.exit_code == 1 + assert f"the {adapter} adapter does not take {flag}" in done.stderr + + +def test_a_timeout_must_be_positive(tmp_path: Path) -> None: + dataset = a_dataset(tmp_path / "d.jsonl") + + done = invoke("run", str(dataset), "--timeout", "0", "--dry-run") + + assert done.exit_code == 1 + assert "--timeout" in done.stderr + + +def test_a_report_rebuilt_from_artifacts_keeps_its_dataset_section(tmp_path: Path) -> None: + dataset = a_dataset(tmp_path / "d.jsonl", broken_rows=2) + results = tmp_path / "results" + at_run = tmp_path / "at-run.md" + invoke("run", str(dataset), "--results", str(results), "--report", str(at_run), "--boot", "50") + artifact = next(results.glob("*.json")) + + rebuilt = invoke("report", str(artifact), "--boot", "50") + + assert rebuilt.exit_code == 0, rebuilt.stderr + + def dataset_section(text: str) -> str: + return text.split("## Dataset", 1)[1].split("\n## ", 1)[0] + + written = at_run.read_text(encoding="utf-8") + assert "14 rows read" in dataset_section(written) and "2 refused" in dataset_section(written) + assert dataset_section(rebuilt.stdout) == dataset_section(written) + + +def test_an_artifact_from_before_the_load_was_stored_still_reports(tmp_path: Path) -> None: + dataset = a_dataset(tmp_path / "d.jsonl") + results = tmp_path / "results" + invoke("run", str(dataset), "--results", str(results), "--boot", "50") + artifact = next(results.glob("*.json")) + stored = json.loads(artifact.read_text(encoding="utf-8")) + del stored["load"] + artifact.write_text(json.dumps(stored), encoding="utf-8") + + rebuilt = invoke("report", str(artifact), "--boot", "50") + + assert rebuilt.exit_code == 0, rebuilt.stderr + assert "## Dataset" not in rebuilt.stdout + + +def test_the_load_summary_round_trips_through_the_artifact(tmp_path: Path) -> None: + summary = LoadSummary( + source="d.jsonl", + rows_read=5, + loaded=4, + refusals=("line 5: gold label 'c' is not among the options",), + notes=("a note.",), + unsupported_by_type={"score": 1}, + ) + cases = [Case(id=f"c{i}", text=f"t{i}", labels=("a", "b"), gold_label="a") for i in range(3)] + result = execute.run( + MockAdapter(gold_by_text={case.text: case.gold_label for case in cases}), + cases, + load=summary, + ) + + reread = execute.RunResult.read(result.write(tmp_path)) + + assert reread.load == summary + assert reread.load is not None and reread.load.statement() == summary.statement() + + +def test_the_timeout_an_adapter_used_is_in_the_artifact() -> None: + class Patient(MockAdapter): + timeout = 42.0 + + cases = [Case(id="c", text="t", labels=("a", "b"), gold_label="a")] + + result = execute.run(Patient(gold_by_text={"t": "a"}), cases) + + assert result.config["timeout_seconds"] == 42.0