Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,10 @@ different event from one that moved because it was wrong.

### Fixed

- The pricing template priced every call at $0, so a copy used unedited let
any run past `--max-cost-usd` (#28). Its prices are now null and its `as_of`
a `YYYY-MM-DD` placeholder that the loader refuses, and any entry pricing
both input and output at 0 is refused unless it says `"free": true`.
- Every validation message on the site's calculator and planner read
"[object Object]" (#25). Messages now say what is wrong, mark the field
invalid, and are tied to it for screen readers; a result is announced as one
Expand Down
4 changes: 3 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,9 @@ This one spends money. Set a key, name an adapter, and cap the run.

Cost needs a pricing table **you** supply, because plumbline ships no figures
for vendors whose terms treat pricing as confidential. Copy the template and
fill in the rates from the vendor's own page:
fill in the rates from the vendor's own page, and the date you read them.
plumbline refuses the copy until you have, so an unedited template can never
price a run at nothing:

```
cp docs/pricing.example.json my-pricing.json
Expand Down
5 changes: 4 additions & 1 deletion docs/PLAN.md
Original file line number Diff line number Diff line change
Expand Up @@ -336,7 +336,10 @@ So:
table and lay it over the shipped one, wired to `plumbline run --pricing`.
Every supplied entry must carry its own `source` and `as_of`, on the same
reasoning that applies to a shipped one.
- `docs/pricing.example.json` is a template with placeholder figures.
- `docs/pricing.example.json` is a template whose prices are null and whose
`as_of` is the placeholder `YYYY-MM-DD`, which the loader refuses, so an
unedited copy cannot price a run. An entry with both prices at 0 is refused
unless it says `"free": true`.
- A test asserts no shipped entry for this vendor carries a numeric price, so a
figure cannot drift back in unnoticed.

Expand Down
29 changes: 17 additions & 12 deletions docs/pricing.example.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,14 @@
"price a vendor plumbline ships no figures for, or correct a shipped price",
"that has gone stale, without editing the package.",
"",
"The numbers below are placeholders. plumbline does not ship a tariff for",
"TypeSafe: its customer agreement treats pricing information as confidential,",
"so reading the published page and writing the figure down is your act, not",
"plumbline's. The page is named in the shipped entry's source.",
"The entries below are unfilled on purpose. plumbline does not ship a tariff",
"for TypeSafe: its customer agreement treats pricing information as",
"confidential, so reading the published page and writing the figure down is",
"your act, not plumbline's. The page is named in each entry's source.",
"",
"Until every as_of is a real date, plumbline refuses this file, so an unedited",
"copy can never price a run. Replace each null with the price per million",
"tokens and each YYYY-MM-DD with the day you read it.",
"",
"Key every model string that can appear. An alias resolves server-side and",
"the response reports a version, and pricing prefers whatever answered, so a",
Expand All @@ -17,24 +21,25 @@
"",
"source and as_of are required on every entry. Use null, never 0, where a",
"vendor publishes no price: null is the absence of a claim and 0 is the claim",
"that those tokens are free.",
"that those tokens are free. An entry with both prices at 0 is refused unless",
"it also says \"free\": true.",
"",
"Keys starting with an underscore are skipped, so this note can stay."
],

"jev-1.13.0": {
"input_usd_per_million": 0.0,
"output_usd_per_million": 0.0,
"input_usd_per_million": null,
"output_usd_per_million": null,
"source": "TypeSafe published tariff, https://docs.typesafe.ai/models.md, read by me",
"as_of": "2026-01-01",
"note": "Placeholder figures. Replace with what the page says, and set as_of to the day you read it."
"as_of": "YYYY-MM-DD",
"note": "Replace both nulls with what the page says, and as_of with the day you read it."
},

"jev-latest": {
"input_usd_per_million": 0.0,
"output_usd_per_million": 0.0,
"input_usd_per_million": null,
"output_usd_per_million": null,
"source": "TypeSafe published tariff, https://docs.typesafe.ai/models.md, read by me",
"as_of": "2026-01-01",
"as_of": "YYYY-MM-DD",
"note": "The alias, in case a run is costed before any response has reported a version."
}
}
43 changes: 39 additions & 4 deletions src/plumbline/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -115,9 +115,13 @@ def _anthropic(input_usd: float, output_usd: float) -> Pricing:

#: Fields a supplied entry may carry. Anything else is a typo worth naming.
_PRICING_FIELDS = frozenset(
{"input_usd_per_million", "output_usd_per_million", "source", "as_of", "note"}
{"input_usd_per_million", "output_usd_per_million", "source", "as_of", "note", "free"}
)

#: The ``as_of`` that ``docs/pricing.example.json`` ships with. A copy that still
#: carries it has not been filled in, so it is refused rather than priced.
TEMPLATE_AS_OF = "YYYY-MM-DD"


def load_pricing_file(path: Path | str) -> PricingTable:
"""Read a pricing table the operator wrote, requiring provenance on every entry.
Expand Down Expand Up @@ -197,11 +201,35 @@ def _entry(name: str, value: Any, path: Path) -> Pricing:
"whether that source still says it and a report can age it."
)

as_of = _as_of(name, value["as_of"], path)
input_price = _price(name, value, "input_usd_per_million", path)
output_price = _price(name, value, "output_usd_per_million", path)
free = value.get("free", False)
if not isinstance(free, bool):
raise PricingConfigError(
f"entry {name!r} in {path} has free={free!r}; it is true or absent."
)
# Both prices at exactly zero is what an unedited template looks like, and
# it would let any run past --max-cost-usd. A model that really is free says
# so, and one that says so must not also carry a price.
both_zero = input_price == 0 and output_price == 0
if both_zero and not free:
raise PricingConfigError(
f"entry {name!r} in {path} prices input and output at 0, which claims every "
"token is free and would let any run past --max-cost-usd. If that is true, "
'add "free": true to the entry; if the price is unknown, use null.'
)
if free and not both_zero:
raise PricingConfigError(
f'entry {name!r} in {path} says "free": true but carries a price. '
"Set both prices to 0, or remove free."
)

return Pricing(
input_usd_per_million=_price(name, value, "input_usd_per_million", path),
output_usd_per_million=_price(name, value, "output_usd_per_million", path),
input_usd_per_million=input_price,
output_usd_per_million=output_price,
source=str(value["source"]),
as_of=_as_of(name, value["as_of"], path),
as_of=as_of,
note=str(value.get("note", "")),
)

Expand All @@ -221,6 +249,13 @@ def _price(name: str, value: Mapping[str, Any], field: str, path: Path) -> float


def _as_of(name: str, value: Any, path: Path) -> date:
if value == TEMPLATE_AS_OF:
raise PricingConfigError(
f"entry {name!r} in {path} still has the template's placeholder "
f"as_of={TEMPLATE_AS_OF!r}. Fill in the prices you read and the date you read "
"them; an unedited copy of docs/pricing.example.json is refused rather than "
"used to price a run."
)
try:
return date.fromisoformat(str(value))
except ValueError:
Expand Down
40 changes: 36 additions & 4 deletions tests/test_pricing_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,13 +73,45 @@ def test_an_underscore_key_is_a_comment_and_not_a_model(tmp_path: Path) -> None:
assert set(table) == {"m"}


def test_the_shipped_example_file_loads(tmp_path: Path) -> None:
"""The file the note points operators at must actually parse."""
example = Path(__file__).resolve().parents[1] / "docs" / "pricing.example.json"
table = config.load_pricing_file(example)
EXAMPLE = Path(__file__).resolve().parents[1] / "docs" / "pricing.example.json"


def test_the_example_file_is_refused_until_it_is_filled_in() -> None:
"""An unedited copy must not price anything, least of all at $0 (#28)."""
with pytest.raises(config.PricingConfigError, match="placeholder"):
config.load_pricing_file(EXAMPLE)


def test_the_example_file_loads_once_it_is_filled_in(tmp_path: Path) -> None:
"""The file the README points operators at must parse once they have done their part."""
raw = json.loads(EXAMPLE.read_text(encoding="utf-8"))
for name, entry in raw.items():
if not name.startswith("_"):
entry.update(input_usd_per_million=1.0, output_usd_per_million=4.0, as_of="2026-09-01")
table = config.load_pricing_file(write(tmp_path, raw))

assert "jev-1.13.0" in table
assert "_comment" not in table
assert table["jev-1.13.0"].is_priced


def test_both_prices_at_zero_are_refused_unless_the_entry_says_it_is_free(
tmp_path: Path,
) -> None:
"""0 and 0 is the claim that every token is free, which a placeholder also looks like."""
free = {**ENTRY, "input_usd_per_million": 0, "output_usd_per_million": 0.0}
with pytest.raises(config.PricingConfigError, match="free"):
config.load_pricing_file(write(tmp_path, {"m": free}))

entry = config.load_pricing_file(write(tmp_path, {"m": {**free, "free": True}}))["m"]
assert entry.is_priced
assert cost.cost_of(1_000, 1_000, entry) == 0.0


def test_one_price_at_zero_is_an_ordinary_price(tmp_path: Path) -> None:
payload = {"m": {**ENTRY, "input_usd_per_million": 0}}
entry = config.load_pricing_file(write(tmp_path, payload))["m"]
assert cost.cost_of(1_000_000, 1_000_000, entry) == pytest.approx(6.0)


@pytest.mark.parametrize(
Expand Down
Loading