diff --git a/CHANGELOG.md b/CHANGELOG.md index 83db89f..12fd3cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -83,6 +83,10 @@ different event from one that moved because it was wrong. ### Fixed +- The pricing template priced every call at $0, so a copy used unedited let + any run past `--max-cost-usd` (#28). Its prices are now null and its `as_of` + a `YYYY-MM-DD` placeholder that the loader refuses, and any entry pricing + both input and output at 0 is refused unless it says `"free": true`. - Every validation message on the site's calculator and planner read "[object Object]" (#25). Messages now say what is wrong, mark the field invalid, and are tied to it for screen readers; a result is announced as one diff --git a/README.md b/README.md index 0cbdc3a..70ab744 100644 --- a/README.md +++ b/README.md @@ -182,7 +182,9 @@ This one spends money. Set a key, name an adapter, and cap the run. Cost needs a pricing table **you** supply, because plumbline ships no figures for vendors whose terms treat pricing as confidential. Copy the template and -fill in the rates from the vendor's own page: +fill in the rates from the vendor's own page, and the date you read them. +plumbline refuses the copy until you have, so an unedited template can never +price a run at nothing: ``` cp docs/pricing.example.json my-pricing.json diff --git a/docs/PLAN.md b/docs/PLAN.md index 8acabcb..5cd86e7 100644 --- a/docs/PLAN.md +++ b/docs/PLAN.md @@ -336,7 +336,10 @@ So: table and lay it over the shipped one, wired to `plumbline run --pricing`. Every supplied entry must carry its own `source` and `as_of`, on the same reasoning that applies to a shipped one. -- `docs/pricing.example.json` is a template with placeholder figures. +- `docs/pricing.example.json` is a template whose prices are null and whose + `as_of` is the placeholder `YYYY-MM-DD`, which the loader refuses, so an + unedited copy cannot price a run. An entry with both prices at 0 is refused + unless it says `"free": true`. - A test asserts no shipped entry for this vendor carries a numeric price, so a figure cannot drift back in unnoticed. diff --git a/docs/pricing.example.json b/docs/pricing.example.json index 886e25b..97854a6 100644 --- a/docs/pricing.example.json +++ b/docs/pricing.example.json @@ -5,10 +5,14 @@ "price a vendor plumbline ships no figures for, or correct a shipped price", "that has gone stale, without editing the package.", "", - "The numbers below are placeholders. plumbline does not ship a tariff for", - "TypeSafe: its customer agreement treats pricing information as confidential,", - "so reading the published page and writing the figure down is your act, not", - "plumbline's. The page is named in the shipped entry's source.", + "The entries below are unfilled on purpose. plumbline does not ship a tariff", + "for TypeSafe: its customer agreement treats pricing information as", + "confidential, so reading the published page and writing the figure down is", + "your act, not plumbline's. The page is named in each entry's source.", + "", + "Until every as_of is a real date, plumbline refuses this file, so an unedited", + "copy can never price a run. Replace each null with the price per million", + "tokens and each YYYY-MM-DD with the day you read it.", "", "Key every model string that can appear. An alias resolves server-side and", "the response reports a version, and pricing prefers whatever answered, so a", @@ -17,24 +21,25 @@ "", "source and as_of are required on every entry. Use null, never 0, where a", "vendor publishes no price: null is the absence of a claim and 0 is the claim", - "that those tokens are free.", + "that those tokens are free. An entry with both prices at 0 is refused unless", + "it also says \"free\": true.", "", "Keys starting with an underscore are skipped, so this note can stay." ], "jev-1.13.0": { - "input_usd_per_million": 0.0, - "output_usd_per_million": 0.0, + "input_usd_per_million": null, + "output_usd_per_million": null, "source": "TypeSafe published tariff, https://docs.typesafe.ai/models.md, read by me", - "as_of": "2026-01-01", - "note": "Placeholder figures. Replace with what the page says, and set as_of to the day you read it." + "as_of": "YYYY-MM-DD", + "note": "Replace both nulls with what the page says, and as_of with the day you read it." }, "jev-latest": { - "input_usd_per_million": 0.0, - "output_usd_per_million": 0.0, + "input_usd_per_million": null, + "output_usd_per_million": null, "source": "TypeSafe published tariff, https://docs.typesafe.ai/models.md, read by me", - "as_of": "2026-01-01", + "as_of": "YYYY-MM-DD", "note": "The alias, in case a run is costed before any response has reported a version." } } diff --git a/src/plumbline/config.py b/src/plumbline/config.py index 8192ea2..5c32cf7 100644 --- a/src/plumbline/config.py +++ b/src/plumbline/config.py @@ -115,9 +115,13 @@ def _anthropic(input_usd: float, output_usd: float) -> Pricing: #: Fields a supplied entry may carry. Anything else is a typo worth naming. _PRICING_FIELDS = frozenset( - {"input_usd_per_million", "output_usd_per_million", "source", "as_of", "note"} + {"input_usd_per_million", "output_usd_per_million", "source", "as_of", "note", "free"} ) +#: The ``as_of`` that ``docs/pricing.example.json`` ships with. A copy that still +#: carries it has not been filled in, so it is refused rather than priced. +TEMPLATE_AS_OF = "YYYY-MM-DD" + def load_pricing_file(path: Path | str) -> PricingTable: """Read a pricing table the operator wrote, requiring provenance on every entry. @@ -197,11 +201,35 @@ def _entry(name: str, value: Any, path: Path) -> Pricing: "whether that source still says it and a report can age it." ) + as_of = _as_of(name, value["as_of"], path) + input_price = _price(name, value, "input_usd_per_million", path) + output_price = _price(name, value, "output_usd_per_million", path) + free = value.get("free", False) + if not isinstance(free, bool): + raise PricingConfigError( + f"entry {name!r} in {path} has free={free!r}; it is true or absent." + ) + # Both prices at exactly zero is what an unedited template looks like, and + # it would let any run past --max-cost-usd. A model that really is free says + # so, and one that says so must not also carry a price. + both_zero = input_price == 0 and output_price == 0 + if both_zero and not free: + raise PricingConfigError( + f"entry {name!r} in {path} prices input and output at 0, which claims every " + "token is free and would let any run past --max-cost-usd. If that is true, " + 'add "free": true to the entry; if the price is unknown, use null.' + ) + if free and not both_zero: + raise PricingConfigError( + f'entry {name!r} in {path} says "free": true but carries a price. ' + "Set both prices to 0, or remove free." + ) + return Pricing( - input_usd_per_million=_price(name, value, "input_usd_per_million", path), - output_usd_per_million=_price(name, value, "output_usd_per_million", path), + input_usd_per_million=input_price, + output_usd_per_million=output_price, source=str(value["source"]), - as_of=_as_of(name, value["as_of"], path), + as_of=as_of, note=str(value.get("note", "")), ) @@ -221,6 +249,13 @@ def _price(name: str, value: Mapping[str, Any], field: str, path: Path) -> float def _as_of(name: str, value: Any, path: Path) -> date: + if value == TEMPLATE_AS_OF: + raise PricingConfigError( + f"entry {name!r} in {path} still has the template's placeholder " + f"as_of={TEMPLATE_AS_OF!r}. Fill in the prices you read and the date you read " + "them; an unedited copy of docs/pricing.example.json is refused rather than " + "used to price a run." + ) try: return date.fromisoformat(str(value)) except ValueError: diff --git a/tests/test_pricing_config.py b/tests/test_pricing_config.py index 20c39dd..5fae58d 100644 --- a/tests/test_pricing_config.py +++ b/tests/test_pricing_config.py @@ -73,13 +73,45 @@ def test_an_underscore_key_is_a_comment_and_not_a_model(tmp_path: Path) -> None: assert set(table) == {"m"} -def test_the_shipped_example_file_loads(tmp_path: Path) -> None: - """The file the note points operators at must actually parse.""" - example = Path(__file__).resolve().parents[1] / "docs" / "pricing.example.json" - table = config.load_pricing_file(example) +EXAMPLE = Path(__file__).resolve().parents[1] / "docs" / "pricing.example.json" + + +def test_the_example_file_is_refused_until_it_is_filled_in() -> None: + """An unedited copy must not price anything, least of all at $0 (#28).""" + with pytest.raises(config.PricingConfigError, match="placeholder"): + config.load_pricing_file(EXAMPLE) + + +def test_the_example_file_loads_once_it_is_filled_in(tmp_path: Path) -> None: + """The file the README points operators at must parse once they have done their part.""" + raw = json.loads(EXAMPLE.read_text(encoding="utf-8")) + for name, entry in raw.items(): + if not name.startswith("_"): + entry.update(input_usd_per_million=1.0, output_usd_per_million=4.0, as_of="2026-09-01") + table = config.load_pricing_file(write(tmp_path, raw)) assert "jev-1.13.0" in table assert "_comment" not in table + assert table["jev-1.13.0"].is_priced + + +def test_both_prices_at_zero_are_refused_unless_the_entry_says_it_is_free( + tmp_path: Path, +) -> None: + """0 and 0 is the claim that every token is free, which a placeholder also looks like.""" + free = {**ENTRY, "input_usd_per_million": 0, "output_usd_per_million": 0.0} + with pytest.raises(config.PricingConfigError, match="free"): + config.load_pricing_file(write(tmp_path, {"m": free})) + + entry = config.load_pricing_file(write(tmp_path, {"m": {**free, "free": True}}))["m"] + assert entry.is_priced + assert cost.cost_of(1_000, 1_000, entry) == 0.0 + + +def test_one_price_at_zero_is_an_ordinary_price(tmp_path: Path) -> None: + payload = {"m": {**ENTRY, "input_usd_per_million": 0}} + entry = config.load_pricing_file(write(tmp_path, payload))["m"] + assert cost.cost_of(1_000_000, 1_000_000, entry) == pytest.approx(6.0) @pytest.mark.parametrize(