diff --git a/.github/dependabot.yml b/.github/dependabot.yml index ca961f1..eef6eef 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -21,7 +21,9 @@ updates: # so a patch bump moves Cargo.lock and nothing anybody resolves against. patch-updates: update-types: ["patch"] - ignore: + # **There is no `ignore` list any more, and the two entries that were here are + # worth their notes.** + # # The `ratatui` entry that stood here is GONE as of 0.18.0, and the note it # carried is worth keeping because of how it ended. It said the composer was # `tui-textarea` 0.7 built against exactly ratatui 0.29 and crossterm 0.28, @@ -61,16 +63,19 @@ updates: # `tests/dependencies.rs::n1_the_composer_is_free_and_no_lru_is_in_the_advisorys_range` # fails the build if any `lru` in the lockfile falls inside the range. # - # `crossterm` stays pinned, and now for a reason of our own choosing rather - # than a dependency's: ratatui 0.30 is taken with the `crossterm_0_28` - # feature, so a Dependabot bump of the direct `crossterm` to 0.29 would put - # two versions in the tree again. Moving to crossterm 0.29 is a deliberate - # change of that feature flag and this entry together, not a bot's to make. - # It is pre-1.0, so the minor bump is the breaking one — patches still come - # through. - - dependency-name: crossterm - update-types: - ["version-update:semver-minor", "version-update:semver-major"] + # **The `crossterm` entry that stood here is GONE as of 0.22.0**, and it went + # the way it said it would. It held the direct `crossterm` at 0.28 because + # ratatui 0.30 was taken with the `crossterm_0_28` feature and a bot bumping + # one without the other would put two versions in the tree — and it named the + # exit: "moving to crossterm 0.29 is a deliberate change of that feature flag + # and this entry together, not a bot's to make." 0.22.0 made it. `Cargo.toml` + # now carries `crossterm_0_29` and `crossterm = "0.29"`, changed in one + # commit, and there is nothing left for this entry to hold back. + # + # Nothing replaces it. The property that entry was protecting — one crossterm + # in the tree — is not a judgement a list of ignores can keep: it is the two + # lines in `Cargo.toml` agreeing, and a future bot PR that moves the direct + # crate alone fails to build rather than passing quietly. - package-ecosystem: github-actions directory: "/" diff --git a/CHANGELOG.md b/CHANGELOG.md index f4d2686..a5037f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,147 @@ All notable changes to this project are documented here. The format follows ## [Unreleased] +## [0.22.0] - 2026-08-27 + +What the work cost and whether it worked — two questions this interface has been +carrying the answers to since io-harness 0.18.0 and has never asked. + +**`/cost` commits what has been spent: this run, this session, by model, by day.** +Tokens and money, four sections, every figure a row already in `runs.db`. The +harness has recorded one row per provider call for four releases — the model, the +prompt, completion, cache-read, cache-write and reasoning split, the latency and +the time to first token — and io-cli reported token counts and treated the money +question as unanswerable. It was answerable then. + +**Nothing on that page is estimated, and three rules follow from it.** Cache reads +and cache writes are shown as the breakdown of the prompt they already are, never +added on top: they live inside `prompt_tokens`, and adding them would over-report +every cached turn, which is most of them. A call the provider reported no usage +for is **unknown**, never free — io-harness stores it as SQL `NULL` for exactly +that reason, and summing it as zero would report a turn that cost something as a +turn that cost nothing. And a model with no rate in the table makes the total a +**floor** rather than a total, which the page says in that word, with the number +of calls it applies to, and on the row itself in the grouped sections — a reader +scanning for the largest figure has to see which of them is incomplete without +reading past the list. io-harness's own pricing documentation calls a renderer +that hides that count "lying by omission", and it is right. + +**`/stats` commits how the runs have gone**: runs by outcome, runs by day, the +first-try counts, gate failures by phase, recovery, the slowest calls and the time +to first token over the last 200 runs, and what the store holds on disk. **First-try +is io-harness's own definition** — finished *and* successful *and* carrying no +gate failure — and it arrives as three counts rather than as a rate, because the +denominator is a choice: first-try over every run counts runs that are still +going. Where a share is drawn, the row names what it is a share *of*, instead of +printing a percentage whose meaning two readers would reconstruct differently. The +sandbox gate's phases and the review, command and contains gates are two +vocabularies that do not overlap, so they stay two lists; merging them would +produce a chart whose categories mean two things. Nothing here compacts anything — +`Store::compact` is a full `VACUUM` needing free disk roughly equal to the file, +and a page that reported free space and then reclaimed it would be a page that +surprised somebody mid-session. + +**They are two commands rather than one because they are two questions.** Every +agent that has both keeps them apart, and a single screen carrying thirteen +sections is one nobody reads to the end. `/usage` is an alias for `/cost` and +earns no row in any table — it resolved to `/status` until now, which was the +closest thing there was to an answer and was not one. + +**io-cli compiles no prices in, and this is the part worth reading twice.** A rate +baked into a binary is a promise the binary cannot keep: providers move prices +without announcing it, a release cadence is not a pricing cadence, and an operator +reading a confident wrong number is worse off than one reading no number at all. +So the table ships **empty**, and an install that has connected nothing sees +tokens and no currency. It is filled from the model catalogue the operator's own +provider serves — the same fetch `io setup` has made since 0.1.0 to offer a list +of models, whose `price`, `price_tiers` and `price_source` it read off the same +row and threw away. No JSON is parsed here and no dependency is added: io-harness +did the parsing and the unit conversion. The rates are written into `[prices]`, +which is io-harness's own section — `as_of` for the date, one line per model under +`[prices.models]`, a row per line so that correcting one by hand is a one-line +change you can find again. + +**Refreshing shows every rate that would move before it writes anything**, with +what each was and what it would become, and declining writes nothing — the shape +`/import` established in 0.21.0. That is not courtesy. io-cli records what a rate +*is* and never where it came from, so it cannot tell a correction you made by hand +from a value an older catalogue served; it does not guess, it shows you and lets +you refuse the lot. **A refetch that comes back empty, or far shorter than the +table it would replace, is refused and the old table kept** — the one failure in +this area that loses money quietly, because a truncated response replacing a full +table with a handful of rows would turn most of an operator's spending into +"unpriced" and shrink their reported bill with nothing failing anywhere. A first +fill has nothing to compare against and is never refused. Rows for models the +catalogue no longer serves are left alone rather than pruned: io-harness prices a +call by the model name on it, so an old row is what prices an old run correctly. + +**Whose price it is gets said, on every surface that draws money.** OpenAI and +Anthropic publish no prices on any endpoint — their model endpoints +carry capabilities and limits and no cost field, and their cost APIs report what +was *spent* rather than what a token *costs*. So for those two the rates +necessarily come from the reference catalogue rather than from the vendor, and the +page names which and on what date. io-harness models the distinction already, and +io-cli carries it through instead of flattening it into the connected provider's +name. On OpenRouter the two coincide, because the reference catalogue is +OpenRouter's own. + +**The status line carries a cost field**, right of the token count it is derived +from, so the two read in the order they are computed in. It is **absent — not +`$0`** — where there is no price table, where the table prices none of the models +this run called, or where nothing has run. Those are three different things and +none of them means free. + +**Three new keys and one new section for a file you already have.** +`[app.io-cli.prices]` carries `source_url`, which names a catalogue to read +instead of io-harness's default and is the only way an operator on a self-hosted +or `compatible` endpoint gets prices at all; and `source` and `models`, which +record what the last read was and how many models it priced and are written by a +fetch rather than by hand. They are in io-cli's own section rather than beside the +rates because `[prices]` is `deny_unknown_fields` and carries exactly `as_of` and +`models` — a key of io-cli's own put next to them would not be ignored, it would +make the operator's whole configuration file unreadable. `/config` gains +`prices.as_of` and `app.io-cli.prices.source_url` as rows; `[prices.models]` is +deliberately not one, because it is a list rather than a setting. + +**`/image`, `/copy` and `/copy diff` move from inspect to this turn**, and it is a +correction rather than a way of making room — the same sentence 0.19.0 wrote about +`/mcp` and `/provider`, and it is worth being able to say twice. All three act on +the turn that just finished; none of them asks the store a question, which is what +**inspect** means. They were filed there because they *show* something, and +showing is not the same as inspecting. `/cost` and `/stats` are what made it worth +correcting: no group may hold more than ten, **inspect** stood at nine, and the +choice was between re-filing three commands that were in the wrong group and +filing two more that would have been. Weakening the bound would have given up the +thing it protects — the grouped menu not turning back into the flat list of thirty +it replaced. + +**Two status-line priority inversions, both corrected here because this release +adds another counter to that row.** The footer dropped its whole right-hand group +— the policy layer, the containment mode, the planning phase — to keep every +counter, so a narrow terminal gave up the standing modes that explain why nothing +is happening in order to keep a number about what already happened; the counters +yield now. And `fields` pushed `planning` right of the counters, so the narrow +line dropped it before the token count — the exact inversion of the rule written +five lines above it in the same file, standing for four releases. + +**Four defects carried over from 0.21.0 are fixed.** `edit::split_path` is +quote-aware: a dot inside a quoted key is not a separator, TOML spells a bare key +out of `A-Za-z0-9_-` and nothing else, and a model id or MCP server name carrying +a dot was cut in half — the read half answered nothing and the write half appended +a second copy of a table that was already there, surfacing as an unexplainable +"the edit would have produced a file that does not parse". Removing or moving a +TOML entry no longer carries the *next* section's comment block away with it. +`move_entry` into the last position of a file with no trailing newline no longer +concatenates onto the final value. And a loose `CONVENTIONS.md` or `CLAUDE.md` +inside a `skills/` or `plugins/` directory is imported as a **skill**: it matched +on its basename before, and its whole body was appended into the instructions file +loaded on every turn, forever, instead of a named skill read on demand. Where a +file is decides what it is, ahead of what it is called. + +**crossterm moves to 0.29**, taken together with ratatui's `crossterm_0_29` +feature so the two agree on one version of the terminal backend. The dependabot +ignore that held it back is gone. + ## [0.21.0] - 2026-08-27 What you already told another agent tool, brought across once — and two surfaces diff --git a/Cargo.lock b/Cargo.lock index be28b56..0723a64 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -758,22 +758,6 @@ version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" -[[package]] -name = "crossterm" -version = "0.28.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" -dependencies = [ - "bitflags 2.13.1", - "crossterm_winapi", - "mio", - "parking_lot", - "rustix 0.38.44", - "signal-hook", - "signal-hook-mio", - "winapi", -] - [[package]] name = "crossterm" version = "0.29.0" @@ -786,7 +770,7 @@ dependencies = [ "document-features", "mio", "parking_lot", - "rustix 1.1.4", + "rustix", "signal-hook", "signal-hook-mio", "winapi", @@ -1866,10 +1850,10 @@ dependencies = [ [[package]] name = "io-cli" -version = "0.21.0" +version = "0.22.0" dependencies = [ "clap", - "crossterm 0.28.1", + "crossterm", "image", "io-harness", "ratatui", @@ -2150,12 +2134,6 @@ dependencies = [ "bitflags 2.13.1", ] -[[package]] -name = "linux-raw-sys" -version = "0.4.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" - [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -3193,8 +3171,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567584a3b0e6a8203c23de40b4861497266725eb5363dbfd18a1edd603cca9f0" dependencies = [ "cfg-if", - "crossterm 0.28.1", - "crossterm 0.29.0", + "crossterm", "instability", "ratatui-core", ] @@ -3505,19 +3482,6 @@ dependencies = [ "semver", ] -[[package]] -name = "rustix" -version = "0.38.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" -dependencies = [ - "bitflags 2.13.1", - "errno", - "libc", - "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", -] - [[package]] name = "rustix" version = "1.1.4" @@ -3527,7 +3491,7 @@ dependencies = [ "bitflags 2.13.1", "errno", "libc", - "linux-raw-sys 0.12.1", + "linux-raw-sys", "windows-sys 0.61.2", ] @@ -4086,7 +4050,7 @@ dependencies = [ "fastrand", "getrandom 0.4.3", "once_cell", - "rustix 1.1.4", + "rustix", "windows-sys 0.61.2", ] @@ -4098,7 +4062,7 @@ checksum = "9048a889effe34a5cddee0af7f53285198b16dca3be510858d38dfdb3e62a04e" dependencies = [ "bitflags 2.13.1", "parking_lot", - "rustix 1.1.4", + "rustix", "signal-hook", "windows-sys 0.61.2", ] @@ -5074,15 +5038,6 @@ dependencies = [ "windows-targets", ] -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets", -] - [[package]] name = "windows-sys" version = "0.61.2" diff --git a/Cargo.toml b/Cargo.toml index 1f5e156..cc25182 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "io-cli" -version = "0.21.0" +version = "0.22.0" edition = "2021" # io-harness declares 1.95, set by `libsqlite3-sys` calling the std `cfg_select!` # macro from its build script. Nothing here can be lower than what the harness @@ -75,12 +75,15 @@ io-harness = { version = "0.69", features = ["media", "browser", "documents"] } # go. io-cli owns its editing model now (`src/editor.rs`), and `lru` is out of the # tree entirely. # -# `default-features = false` with `crossterm_0_28` rather than the default -# `crossterm`, which resolves to 0.29: that bump is a separate decision from this -# one, and taking both at once would put two crossterm versions in the tree for as -# long as anything disagreed. Of the remaining defaults, `all-widgets` is only the -# calendar and `macros` is only the declarative builders — this crate uses -# neither, and the base widgets are not behind either flag. +# `default-features = false` with `crossterm_0_29` rather than the bare default +# `crossterm`, and the two crates move together or not at all. ratatui chooses the +# crossterm it re-exports by feature, so the direct `crossterm` line below and this +# flag name the same version by construction; the release that let them disagree +# would build two crossterm versions whose key events are different types. 0.18.0 +# held this at 0.28 and said so here and in `.github/dependabot.yml`; 0.22.0 is the +# deliberate change both notes were waiting for. Of the remaining defaults, +# `all-widgets` is only the calendar and `macros` is only the declarative builders +# — this crate uses neither, and the base widgets are not behind either flag. # # `unstable-rendered-line-info` is enabled for one method: `Paragraph::line_count`, # which answers how many rows a wrapped paragraph will occupy. `insert_before` @@ -88,12 +91,12 @@ io-harness = { version = "0.69", features = ["media", "browser", "documents"] } # exactly the content this product exists to keep. It is a feature of a dependency # already taken, not a dependency. ratatui = { version = "0.30", default-features = false, features = [ - "crossterm_0_28", + "crossterm_0_29", "layout-cache", "underline-color", "unstable-rendered-line-info", ] } -crossterm = "0.28" +crossterm = "0.29" clap = { version = "4", features = ["derive"] } tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync", "time"] } serde = { version = "1", features = ["derive"] } diff --git a/README.md b/README.md index f5004ba..ff76570 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ reimplemented here. A test asserts that: `tests/dependencies.rs` fails the build if this crate ever grows an HTTP client, a TLS stack, a database or a sandbox. - [Install](#install) · [First run](#first-run) · [Bringing your setup across](#bringing-your-setup-across) · [While it works](#while-it-works) -- [Keys](#keys) · [Commands](#commands) · [Configuration](#configuration) +- [Keys](#keys) · [Commands](#commands) · [What it costs](#what-it-costs) · [Configuration](#configuration) - [Capability bundles](#capability-bundles) · [Hooks](#hooks) · [The fleet](#the-fleet) - [Pictures](#pictures) · [Documents](#documents) · [Background jobs](#background-jobs) - [Reading it without seeing it](#reading-it-without-seeing-it) · [Headless](#headless) @@ -47,6 +47,7 @@ on one row and the keys and the posture on the next.](docs/screenshot.png) | **Markdown, rendered** | Headings, bullets, code and emphasis drawn as themselves rather than printed as notation | | **Documents, read and written** | Spreadsheets, Word, slide decks, PDFs and barcodes through io-harness's own tools — twelve of them, six of which write, every one under the same gate as any other read or write | | **Your setup, brought across** | `/import` finds the agent tools already on this machine and offers their instructions, MCP servers, skills and model, item by item, with the whole plan shown before a byte is written and no credential read at any point | +| **What it spent, and whether it worked** | `/cost` commits the money and the token split — this run, this session, by model, by day — and `/stats` commits the outcomes, the first-try counts, the gate failures and the latencies. Nothing is estimated: an unpriced model makes a total a floor and the page says so | ## It never takes your terminal @@ -153,6 +154,13 @@ name one — `gpt-5` could be OpenAI or any of the twenty-one presets pointed at compatible endpoint — so io hands you the id and the entry is built once you have chosen the vendor. +**Where a file *is* decides what it is, ahead of what it is called.** A loose +`CONVENTIONS.md` — or `CLAUDE.md`, or `MEMORY.md` — sitting inside a `skills/` or +`plugins/` directory is a skill, and from 0.22.0 it is imported as one. It used to +match on its basename and be appended whole into the instructions file that is +loaded on **every turn, forever**, instead of being a named skill the model reads +on demand. + **The whole plan is on screen before a single byte is written.** One row per thing found, saying where it came from and where it would go, and you accept them item by item. What you did not accept is not written. A cancelled import is not a @@ -224,6 +232,24 @@ path until this release, so what an operator met under a half-finished answer wa `error: step_cap_reached`. All four are successful calls in io-harness and always were. The word stays the harness's; what changed is the weight it is said in. +**From 0.22.0 the status line also carries what the run cost**, immediately right +of the token count it is derived from, so the two read in the order they are +computed in: the tokens are what happened and the money is what they came to. It +is **absent** — not `$0` — where there is no price table, where the table prices +none of the models this run called, or where nothing has run yet. Those three +things are different and none of them means free; `/cost` is where they are told +apart. See [What it costs](#what-it-costs). + +**Two things about that line were corrected in the same release, both of them +priority inversions.** The footer used to drop its whole right-hand group — the +policy layer, the containment mode, the planning phase — to keep every counter, +so a narrow terminal gave up the standing modes that explain *why nothing is +happening* in order to keep a number about what already happened; the counters +yield now. And `planning` was pushed right of the counters, so the narrow line +dropped it before the token count, inverting the rule written five lines above it +in the code. A standing mode that stops the agent writing outranks what the last +turn spent, on both rows. + What lands in the scrollback is designed rather than defaulted. A tool call reads as a verb and a workspace-relative path — `Read src/lib.rs`, not `read_file` and an absolute path — and a tool this release has never seen keeps the name @@ -438,6 +464,9 @@ above a row that ranked there for reasons having nothing to do with it. | `/profile` | switch to a named profile from the configuration, for this session | | `/steer` | send what is queued into the turn that is already running | | `/compact` | fold this conversation into a summary, at the next step | +| `/image` | draw an attached image again: /image 1 | +| `/copy` | put the last answer on the system clipboard | +| `/copy diff` | put the whole run's patch on the system clipboard | **inspect** @@ -449,9 +478,8 @@ above a row that ranked there for reasons having nothing to do with it. | `/expand` | commit the last step's full detail into the scrollback | | `/fleet` | show the children this turn has spawned | | `/skills` | every skill, shipped or yours: what it is for, whether it is on, and its file | -| `/image` | draw an attached image again: /image 1 | -| `/copy` | put the last answer on the system clipboard | -| `/copy diff` | put the whole run's patch on the system clipboard | +| `/cost` | commit what this run, this session and this install have spent | +| `/stats` | commit how the runs have gone: outcomes, first-try, gates, latency | **configure** @@ -476,6 +504,21 @@ here you use once: the others are returned to for the life of the install. It writes files, which is the whole reason it is under **configure**. See [Bringing your setup across](#bringing-your-setup-across). +`/cost` and `/stats` are new in 0.22.0 and are two commands rather than one +because they are two questions: `/cost` says what the work cost, `/stats` says +whether it worked. See [What it costs](#what-it-costs). + +`/image`, `/copy` and `/copy diff` sit under **this turn** from 0.22.0, and it is +a correction rather than a way of making room — the same sentence 0.19.0 wrote +about `/mcp` and `/provider`, and it is worth being able to say twice. All three +act on the turn that just finished: `/image` draws an image attached to it, +`/copy` puts its answer or its diff on the clipboard. None of them asks the store +a question, which is what **inspect** means. They were filed there because they +*show* something, and showing is not the same as inspecting. `/cost` and `/stats` +are what made it worth correcting: no group may hold more than ten, **inspect** +stood at nine, and the choice was between re-filing three commands that were in +the wrong group and filing two more that would have been. + `/mcp` and `/provider` sit under **configure** from 0.19.0, and it is a correction rather than a promotion: both open with a list, and both go on from that list to add, edit and remove entries in the configuration file, which is the @@ -483,12 +526,14 @@ one thing **inspect** promises it never does. That second half was a promise rather than a fact until 0.21.0 — the writers existed and nothing called them, so both panels could only read. They write now. `/steer` and `/compact` are listed under **this turn**, which is the group the code has -always filed them in — the table above said otherwise until this release, and +always filed them in — the table above said otherwise until 0.21.0, and nothing but a reader was misled by it. -`/usage` answers what `/status` answers and is deliberately not listed above: an -alias earns no row of its own, because a second row for one screen reads as a -second screen. +`/usage` opens `/cost` and is deliberately not listed above: an alias earns no +row of its own, because a second row for one screen reads as a second screen. It +resolved to `/status` until 0.22.0, which was the closest thing there was to an +answer and was not one — `/usage` means spend everywhere else it exists, and this +product now has a spend to report. In the palette each row carries a mark saying what it is — `:` runs a command, `+` fills the prompt from a configured template, `*` names one of the agent's own @@ -541,6 +586,94 @@ refuse a large payload without saying so. `/theme` and `/model` change this session only and say so. Making a choice permanent is `io setup`. +## What it costs + +**`/cost` commits what has been spent, and `/stats` commits how the runs have +gone.** Two commands rather than one, because they are two questions — what the +work cost, and whether it worked — and a single screen carrying thirteen sections +is one nobody reads to the end. `/usage` opens `/cost`. + +`/cost` has four sections: **this run**, **this session**, **by model** and **by +day**, each with the calls, the money, and the token split under it. Every figure +is a row io-harness has been writing into `runs.db` since its 0.18.0 and this +interface has never read — the model, the prompt, completion, cache-read, +cache-write and reasoning split, the latency and the time to first token, one row +per provider call. + +**Nothing is estimated.** There is no sampling, no extrapolation from a partial +window, and no model of what a token "usually" costs. Three rules follow from +that and each is on the page rather than in this file: + +- **Cache reads and cache writes are the breakdown of the prompt they are part + of, never an addition to it.** They are already inside `prompt_tokens`, so the + page shows the prompt total and the parts under it — read, written, fresh — and + the parts sum to the total rather than past it. Reasoning is the same + arrangement under completion: a vendor that reports it separately still bills it + as output. +- **A call the provider reported no usage for is *unknown*, never free.** + io-harness stores that as SQL `NULL` and reads it back as `None` for exactly + this reason; summing it as zero would report a turn that cost something as a + turn that cost nothing. The count of them is a line on the page. +- **A model with no rate in the table makes the total a *floor*, and the page + says so** — in that word, with the number of calls it applies to, and on the + row itself in the grouped sections so a reader scanning for the largest figure + can see which of them is incomplete without reading past the list. + +`/stats` is the other half: runs by outcome, runs by day, the first-try counts, +gate failures by phase, recovery, the slowest calls and the time to first token +over the last 200 runs, and what the store holds on disk. **First-try is +io-harness's own definition** — finished *and* successful *and* carrying no gate +failure — and it arrives as three counts rather than as a rate, because the +denominator is a choice. Where a share is shown, the row names what it is a share +*of*. The sandbox gate's phases and the review, command and contains gates use +two vocabularies that do not overlap, so they are two lists and are never merged +into one. Nothing on the page compacts anything: `Store::compact` is a full +`VACUUM` that needs free disk roughly equal to the file, so the free figure is +reported and the reclaiming is not offered here. + +### Where a price comes from + +**io-cli compiles no prices in.** A rate baked into a binary is a promise the +binary cannot keep: providers move prices without announcing it, a release cadence +is not a pricing cadence, and an operator reading a confident wrong number is +worse off than one reading no number at all. So the table ships empty, and **an +install that has connected nothing sees tokens and no currency** — which is the +honest answer, not a bug. + +It is filled from **the model catalogue your connected provider already serves** — +the same fetch `io setup` has made since 0.1.0 to offer you a list of models, +whose prices it read and threw away on the same row. The rates land in `[prices]`, +which is io-harness's own section: `as_of` for the date, and one line per model +under `[prices.models]`. + +**Refreshing is a row on `/config`, and it shows every rate that would move before +it writes anything** — what each was and what it would become — and you can +decline the lot. That is not courtesy. The file records what a rate *is* and never +where it came from, so io-cli cannot tell a correction you made by hand from a +value an older catalogue served; it does not guess, it shows you. A refetch that +comes back empty, or far shorter than the table it would replace, is **refused** +and the old table kept: a truncated response that replaced a full table with a +handful of rows would turn most of your spending into "unpriced" and shrink your +reported bill with nothing failing anywhere. A first fill has nothing to compare +against and is never refused. Rows for models the catalogue no longer serves are +left alone, because io-harness prices a call by the model name on it and an old +row is what prices an old run correctly. + +**Whose price it is gets said, on every surface that draws money.** OpenAI and +Anthropic publish no prices on any endpoint — their model endpoints +carry capabilities and limits and no cost field — so for those two the rates +necessarily come from the reference catalogue rather than from the vendor. The +page names which and on what date. On OpenRouter the two coincide, because the +reference catalogue is OpenRouter's own. `[app.io-cli.prices] source_url` points +at a different catalogue, which is how an operator on a self-hosted or +`compatible` endpoint gets prices at all. + +**The unit is `$`.** io-harness prices in micro-units and names no currency, +because it is not the layer that knows one. io-cli is: every catalogue it can read +quotes USD and every provider it can connect to bills in it. Four decimals below a +unit and two above, in integer arithmetic — a turn costs a fraction of a cent and +a month costs dollars, and one precision cannot show both. + ## Skills **Five of the things io can do have a plain-language door.** Say what you want @@ -1137,7 +1270,13 @@ from the next turn. every section io-cli has no type for come back byte for byte — one value's bytes are replaced and the rest is copied through. The write is staged in a temporary file and renamed over the original, so a failure cannot truncate a configuration, -and the mode is preserved. +and the mode is preserved. **That now holds for a removal as well**: through +0.21.0, removing or moving an entry took the *next* section's comment block away +with it, and moving one into the last position of a file with no trailing newline +concatenated it onto the final value. A key whose name carries a dot — a model id, +an MCP server id — is addressed correctly too; it could only be written quoted, +and the path splitter cut it in half, which surfaced as an unexplainable "the edit +would have produced a file that does not parse". **A project-scoped change that would widen the boundary is refused in io-harness's own words**, and the same value is accepted in `io.local.toml` — @@ -1185,7 +1324,7 @@ bundles](#capability-bundles). **`/profile`** switches to a named `[profile.]` for the session, and `--profile ` picks one for a single run without writing anything. -Eight keys live there, and five tables: +Eight keys live there, and six tables: | Key | Is | | --- | --- | @@ -1202,6 +1341,7 @@ Eight keys live there, and five tables: | `[[app.io-cli.mcp]]` | MCP servers for the turn, in io-harness's own shape. Merged with the top-level `[[mcp]]`, and wins a collision of ids. | | `[[app.io-cli.lsp]]` | language servers for this workspace. Merged with the top-level `[[lsp]]`, and wins a collision of ids. | | `[app.io-cli.browser]` | a browser the agent may drive. Never downloaded — it is one you already have. | +| `[app.io-cli.prices]` | where the rates in `[prices]` came from: `source_url` names a catalogue to read instead of io-harness's default, and `source` and `models` record what the last read was and how many models it priced. The last two are written by a fetch rather than by hand. See [Where a price comes from](#where-a-price-comes-from). | Because the section is unvalidated by design, an unrecognised *value* reads as the default rather than stopping a session from starting. A section io-harness cannot @@ -1332,12 +1472,17 @@ line beside what has been drawn against them. **`[[hook]]` and capability bundles are applied from 0.20.0**, each with the surface the omission was waiting on: `/plugin` for what a bundle brought and what was dropped, and io-harness's own refusal sentence for a hook a project file may -not declare. That leaves two sections and one key still unapplied, and each has a -reason. `[prices]` is not part of a contract at all and belongs with the release -that reads the provider-call rows; there is no `[verify]` section to apply, and +not declare. **`[prices]` is read from 0.22.0**, which is the release that reads +the provider-call rows it prices. That leaves one section and one key still +unapplied, and each has a reason: there is no `[verify]` section to apply, and giving a session verification gates needs its own surface; and `run.templates` is -the thirteenth `[run]` key, reachable only through its own accessor. None of them -is a silent omission — this is where they are named. +the thirteenth `[run]` key, reachable only through its own accessor. Neither is a +silent omission — this is where they are named. + +**A price is never invented, and a missing one is never a zero.** io-cli compiles +no rates in and estimates nothing, so an install that has connected no provider +reports tokens and no currency, and a total containing a model the table does not +price is reported as a floor. See [What it costs](#what-it-costs). **`/import` does not bring a permission boundary across, and never will.** Another tool spells a permission as a command line; io-harness matches a binary name. The diff --git a/docs/config.example.toml b/docs/config.example.toml index 9dfe5ce..77350e7 100644 --- a/docs/config.example.toml +++ b/docs/config.example.toml @@ -291,9 +291,61 @@ net = "deny" # on_failure = "refuse" # timeout_ms = 5000 +# --- What a token costs (0.22.0) -------------------------------------------- +# +# io-harness's own section, and the only place a price comes from. NOTHING IS +# COMPILED IN: a rate baked into a binary is a promise the binary cannot keep, +# because providers move prices without announcing it and a release cadence is +# not a pricing cadence. So this table ships EMPTY, and `/cost` reports tokens +# and no currency until it is filled. +# +# Filling it is a row on `/config`: io-cli reads the model catalogue your +# connected provider already serves — the same fetch that offers the wizard its +# model list, whose prices it used to throw away — and writes what it found here. +# EVERY RATE THAT WOULD MOVE IS SHOWN BEFORE A BYTE IS WRITTEN, with what it was +# and what it would become, and declining writes nothing. That is not courtesy: +# this file records what a rate IS and never where it came from, so io-cli cannot +# tell a correction you made by hand from a value an older catalogue served. It +# does not guess; it shows you and lets you refuse the lot. +# +# A refetch that comes back EMPTY, or far shorter than the table it would +# replace, is refused and the old table kept — a truncated response that replaced +# a full table with a handful of rows would turn most of your spending into +# "unpriced" and shrink your reported bill with nothing failing anywhere. A first +# fill has nothing to compare against and is never refused. +# +# WHOSE PRICE IT IS. OpenAI and Anthropic publish no prices on any +# endpoint — their model endpoints carry capabilities and limits and no cost +# field. So for those two the rates necessarily come from the reference +# catalogue rather than from the vendor, and every surface that draws money says +# which and on what date. On OpenRouter the two coincide, because the reference +# catalogue is OpenRouter's own. +# +# THIS SECTION IS deny_unknown_fields AND CARRIES EXACTLY `as_of` AND `models`. A +# key of io-cli's own put beside them would not be ignored — it would make this +# whole file unreadable. io-cli's bookkeeping lives in [app.io-cli.prices] below. +# +# as_of the date the rates were read. REQUIRED: a price list with no date is +# a claim with no expiry. Written by a fetch rather than typed. +# models one entry per model, in micro-units PER MILLION TOKENS +# +# Rows for models the catalogue no longer serves are left alone rather than +# pruned. They are not stale: io-harness prices a call by the model name on it, so +# an old row is what prices an old run correctly, and `/cost` reports history as +# well as today. +# +# [prices] +# as_of = "2026-08-27" +# +# [prices.models."anthropic/claude-sonnet-4.5"] +# input = 3000000 +# output = 15000000 +# cache_read = 300000 +# cache_write = 3750000 + # --- io-cli's own section --------------------------------------------------- # -# Eight keys and five tables. io-harness does not validate any of it, so an +# Eight keys and six tables. io-harness does not validate any of it, so an # unrecognised VALUE reads as the default rather than stopping a session. A # table io-harness cannot parse at all is different: the session starts on the # defaults and says so, carrying the harness's own message. @@ -374,10 +426,10 @@ fleet = "ctrl+f" # task at all**, and it is absent by default: io-harness has one session entry # point that reaches its spawn loop, and these caps are what selects it. # -# What it costs is smaller than it was, and it is not a mode. No session turn -# takes a steer inbox — none has since 0.11.0 — so a contained turn gives up no -# steering an ordinary one has. `Ctrl+C` still ends it, at the next point where no -# child is in flight, and `/contain off` gives the next turn back. +# What it costs is smaller than it was, and it is not a mode. Every session turn +# carries a steer inbox — every one has since 0.17.0 — so a contained turn gives +# up no steering an ordinary one has. `Ctrl+C` still ends it, at the next point +# where no child is in flight, and `/contain off` gives the next turn back. # # THROUGH 0.11.0 THIS TABLE DECIDED EVERY KEY BELOW IT, AND IT NO LONGER DOES. # io-harness's contained turn was then the only session entry point that took a @@ -475,3 +527,25 @@ fleet = "ctrl+f" # [app.io-cli.browser] # binary = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" # headless = true + +# Where the prices above came from (0.22.0). The rates themselves are io-harness's +# and live in [prices]; this is the bookkeeping io-harness does not model, in the +# one section that will not refuse a key it has never heard of. +# +# source_url the catalogue to read. ABSENT MEANS io-harness's own default, +# which is the reference catalogue. Write this when you are on a +# self-hosted or `compatible` endpoint: the reference catalogue +# cannot speak for a server it has never heard of, and this key is +# the only way such an operator gets prices at all. +# source what the last read was, IN WORDS — the vendor speaking for +# itself, or a named third-party catalogue. WRITTEN BY A FETCH +# RATHER THAN BY HAND: it is what the surfaces that draw money +# cite, and a number attributed to a vendor that never published +# one is the thing it exists to prevent. +# models how many models the last read priced. `PriceTable` has no length +# and no iterator — it answers `price(model)` and nothing else — so +# this is the only count there is, and it is what refuses a refetch +# that comes back far shorter than the table it would replace. +# +# [app.io-cli.prices] +# source_url = "https://openrouter.ai/api/v1/models" diff --git a/src/commands.rs b/src/commands.rs index f8c6b1a..e9e73f9 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -232,6 +232,18 @@ pub const COMMANDS: &[(&str, &str)] = &[ "/clear", "start a new conversation; this one stays in /resume", ), + // The two halves of one question, and they are two commands because they are + // two questions. `/cost` says what the work cost; `/stats` says whether it + // worked. Every agent that has both keeps them apart, and a single screen + // carrying thirteen sections would be one nobody reads to the end. + ( + "/cost", + "commit what this run, this session and this install have spent", + ), + ( + "/stats", + "commit how the runs have gone: outcomes, first-try, gates, latency", + ), ]; /// What an operator is doing when they reach for a command. @@ -306,24 +318,39 @@ pub const GROUPS: &[(Group, &[&str])] = &[ Group::Session, &["/clear", "/resume", "/fork", "/setup", "/exit"], ), + // **`/image`, `/copy` and `/copy diff` moved here in 0.22.0, and it is a + // correction rather than a way of making room** — the same sentence 0.19.0 + // wrote when it moved `/mcp` and `/provider`, and it is worth being able to + // say it twice. All three act on the turn that just finished: `/image` draws + // an image attached to it, `/copy` puts its answer or its diff on the + // clipboard. None of them asks the store a question, which is what `Inspect` + // means. They were filed there because they *show* something, and showing is + // not the same as inspecting. + // + // `/cost` and `/stats` are what made the misfiling worth correcting: `Inspect` + // stood at nine of a bound of ten and both new commands belong in it, so the + // choice was between re-filing three commands that were in the wrong group and + // filing two more that would have been. The bound exists to stop the grouped + // menu becoming the flat list it replaced, and answering it by weakening it + // would have given up the thing it was protecting. ( Group::Turn, &[ - "/model", "/contain", "/plan", "/profile", "/steer", "/compact", + "/model", + "/contain", + "/plan", + "/profile", + "/steer", + "/compact", + "/image", + "/copy", + "/copy diff", ], ), ( Group::Inspect, &[ - "/help", - "/status", - "/context", - "/expand", - "/fleet", - "/skills", - "/image", - "/copy", - "/copy diff", + "/help", "/status", "/context", "/expand", "/fleet", "/skills", "/cost", "/stats", ], ), // **`/mcp` and `/provider` moved here in 0.19.0, and it is a correction rather @@ -815,6 +842,24 @@ pub enum Action { /// budgets in force, the context fill, the servers that came up — so what is /// committed is the state io-harness is in and not io-cli's account of it. Status, + /// Commit what has been spent into the scrollback. + /// + /// The sibling of [`Action::Status`] and drawn the same way, for the same + /// reason: this is a page a reader wants to keep beside the turns it accounts + /// for, and the terminal's own scrollback is where it keeps things. What + /// separates the two is the question. `/status` says what the session *is* + /// — the policy, the caps, the budgets in force; this says what it has *cost*, + /// which is a different fact about the same run and one that only ever climbs. + Cost, + /// Commit how the runs have gone into the scrollback. + /// + /// Deliberately not folded into [`Action::Cost`] under one name. What a run + /// cost and whether it worked are two questions, and the answer to the second + /// — outcomes, the first-try share, gate failures by phase, latency — is + /// eight sections that have no money in them at all. One page carrying both + /// would be thirteen sections deep and a reader looking for either would + /// scroll past the other. + Stats, /// Send what is queued into the turn that is already running. /// /// io-harness delivers it at the next step boundary, so the step in flight @@ -1558,12 +1603,27 @@ pub fn parse(input: &str, keys: &Keys, theme: &Theme) -> Action { // second screen: both open the one surface. "import" | "migrate" => Action::Import, // **An alias earns no row of its own.** `/usage` is what an operator - // coming from another agent types for "what is this costing me", and the - // answer is `/status` — which already commits the spend, the budgets and - // what is left of them. A second row for one screen reads as a second - // screen, so this is answered and never listed: it is absent from - // `COMMANDS`, from the palette and from every group. - "usage" => Action::Status, + // coming from another agent types for "what is this costing me", and a + // second row for one screen reads as a second screen — so this is + // answered and never listed: it is absent from `COMMANDS`, from the + // palette and from every group. + // + // **It answered `/status` until 0.22.0, and the argument for that was + // sound right up until it was not.** The note here read: the answer is + // `/status`, which already commits the spend, the budgets and what is left + // of them. That was the closest thing this program had to a spending + // surface, and it was a token draw against a ceiling rather than a cost. + // The moment a page exists whose whole subject is what has been spent, an + // operator typing the field's word for that page and landing on the + // session's configuration is being answered a question they did not ask. + // + // It stays an alias rather than becoming a third screen. `/usage` means + // plan and rate-limit status everywhere else it exists, and this product + // has no plan and no rate limit of its own to report — so a distinct + // `/usage` here would be a screen with nothing on it. + "usage" => Action::Cost, + "cost" => Action::Cost, + "stats" => Action::Stats, "config" | "settings" => { let mut rest = input.split_whitespace().skip(1); match rest.next() { diff --git a/src/composer.rs b/src/composer.rs index 09a0895..862b450 100644 --- a/src/composer.rs +++ b/src/composer.rs @@ -227,7 +227,7 @@ pub fn pasted_path(text: &str) -> Option { /// /// The mark is chosen rather than escaped into: a path carrying a double quote is /// wrapped in single quotes, one carrying both is left bare. That keeps this the -/// exact inverse of [`crate::attach::unquote`], which takes off one matching pair +/// exact inverse of `crate::attach`'s own `unquote`, which takes off one matching pair /// and knows nothing about escapes — two halves that agree because neither of /// them has anything to agree about. fn quoted(path: &str) -> String { diff --git a/src/configure.rs b/src/configure.rs index b9d3713..dfddaf6 100644 --- a/src/configure.rs +++ b/src/configure.rs @@ -78,6 +78,20 @@ impl Decided { Decided::Default => None, } } + + /// The scope that decided it, where a file did. + /// + /// **What a rewrite of an existing value has to be aimed at.** Writing into a + /// higher-precedence scope than the one that holds a key shadows it rather + /// than updating it: the value in force changes, and the file the operator + /// opens still shows the old one. `None` is io-harness's own default, which no + /// file holds and which a caller therefore has to choose a scope for. + pub fn scope(&self) -> Option { + match self { + Decided::File { scope, .. } => Some(*scope), + Decided::Default => None, + } + } } /// One row of the surface. @@ -137,8 +151,39 @@ pub const CATALOGUE: &[&str] = &[ "app.io-cli.max_parallel_reads", "app.io-cli.spawn_background_after_secs", "app.io-cli.detached_spawns", + // What a turn costs, split across two sections because one of them is not + // ours. `prices.as_of` is io-harness's and dates the whole table; it is + // written by a fetch rather than typed, and it is listed here because a date + // an operator cannot see is a claim with no expiry they cannot check. + // `[prices.models]` is deliberately **not** a row: it is a list rather than a + // setting, and it is reached the way `/provider` and `/mcp` reach a list. + "prices.as_of", + "app.io-cli.prices.source_url", ]; +/// The `/config` row that re-reads the price catalogue. +/// +/// **A row rather than a key, because it is an act and not a setting.** Every +/// other row on that surface names something in a file and puts it in the +/// composer for the operator to type a value after; this one does something. It +/// is a sentinel and not a path so that `settings` and `setting` never see it — +/// a key that is not in any file and never will be would show as "not set" +/// forever on a surface whose whole job is saying what is in force. +/// +/// It lives here rather than in the driver so a test can reach it: nothing under +/// `tests/` can link `src/main.rs`, and a row spelled in the driver is a row no +/// test can assert on. +pub const REFRESH_PRICES: &str = "!refresh-prices"; + +/// The label that sentinel wears on the picker. +pub fn refresh_row(setting: &Setting) -> crate::picker::Row { + let detail = match &setting.value { + Some(as_of) => format!("last read {as_of}"), + None => "no prices are configured".to_string(), + }; + crate::picker::Row::with_detail("prices: re-read the catalogue", detail) +} + /// Whether a key's value is a credential and must never be shown in full. /// /// N2. The test is the key rather than the value, because a credential that @@ -208,11 +253,26 @@ pub fn settings(config: &Config) -> Vec { rows.push(setting(config, key)); } - // Anything a file named that the catalogue does not carry. + // Anything a file named that the catalogue does not carry — **except the + // price table, which is a list and not a set of settings.** + // + // io-harness's origin index records *leaf* keys, so a filled `[prices.models]` + // contributes two to five of them per model: `prices.models..input`, + // `.output`, `.cache_read`, and so on. On an OpenRouter install that is well + // over a thousand rows, and `/config` — a picker an operator opens to change + // one setting — becomes a list of rates with every real setting buried above + // them. The `CATALOGUE` comment already says `[prices.models]` is deliberately + // not a row; this is what makes that true, because the sweep below was putting + // every one of them back. + // + // They would not even render: `record_origins` joins the path with `.` and + // does not quote, so a model id containing a dot arrives as a key that + // `edit::value_at` cannot resolve, and the row draws with no value at all. let mut extra: Vec = config .origins() .map(|(key, _)| key.to_string()) .filter(|key| !seen.contains(key)) + .filter(|key| !key.starts_with("prices.models.")) .collect(); extra.sort(); for key in extra { diff --git a/src/context.rs b/src/context.rs index be60724..c50bf01 100644 --- a/src/context.rs +++ b/src/context.rs @@ -483,48 +483,11 @@ pub fn committed( /// `text` as rows no wider than `width`, indented two then four. /// -/// **Folded and never fitted**, for the reason `crate::status`'s own helper is: a -/// committed surface owns as many rows as it needs, and the characters a fitter -/// would drop here are tool names — which is to say the answer. A word longer -/// than its row is split rather than allowed to overflow, so eighty columns holds. +/// **The body moved to [`crate::page::folded`] in 0.22.0.** This was the second of +/// two copies of the same twenty lines — `crate::status` had the other, under the +/// name `folded`, taking as arguments the two indents this hard-coded — and +/// `/cost` and `/stats` would have made four. The two numbers stay here because +/// they are this surface's decision; the folding is not. fn wrapped(text: &str, width: usize) -> Vec { - let mut rows: Vec = Vec::new(); - let mut indent = 2usize; - let mut row = " ".repeat(indent); - let mut used = 0usize; - for word in text.split_whitespace() { - let mut word = word; - loop { - // At least one cell, so a terminal narrower than the indent still - // makes progress instead of looping forever. - let room = width.saturating_sub(indent).max(1); - let space = usize::from(used > 0); - let length = word.chars().count(); - if used + space + length <= room { - if space == 1 { - row.push(' '); - } - row.push_str(word); - used += space + length; - break; - } - if used > 0 { - rows.push(std::mem::take(&mut row)); - indent = 4; - row = " ".repeat(indent); - used = 0; - continue; - } - let head: String = word.chars().take(room).collect(); - word = &word[head.len()..]; - row.push_str(&head); - rows.push(std::mem::take(&mut row)); - indent = 4; - row = " ".repeat(indent); - } - } - if used > 0 || rows.is_empty() { - rows.push(row); - } - rows + crate::page::folded(text, width, 2, 4) } diff --git a/src/cost.rs b/src/cost.rs new file mode 100644 index 0000000..0440fdc --- /dev/null +++ b/src/cost.rs @@ -0,0 +1,464 @@ +//! What has been spent: `/cost`, and `/usage` under the name the field uses. +//! +//! **Every number here has been in `runs.db` since io-harness 0.18.0 and nothing +//! has ever read one.** The harness records a row per provider call — the model, +//! the prompt, completion, cache-read, cache-write and reasoning token split, the +//! latency and the time to first token — and io-cli reported token counts and +//! called the money question unanswerable. It was answerable four releases ago. +//! +//! # Nothing is estimated +//! +//! Every figure is a row already in the store, or that row multiplied by a rate +//! the operator can point at. There is no sampling, no extrapolation from a +//! partial window and no model of what a token "usually" costs. A call the +//! provider reported no usage for is **unknown**, never zero; a model with no rate +//! in the table is **unpriced**, and a total containing one is a floor rather than +//! a total, which this page says out loud. io-harness's own pricing documentation +//! calls a renderer that hides that count "lying by omission", and it is right. +//! +//! # The three honesty rules, and why each is a rule +//! +//! **Cache reads and writes are a breakdown of the prompt, not an addition to +//! it.** `Usage::cache_read_tokens` and `cache_write_tokens` are already inside +//! `prompt_tokens`; adding them would over-report every cached turn, which is most +//! of them. So this page shows the prompt total and the three parts under it, and +//! the parts sum to the total rather than past it. +//! +//! **Reasoning tokens are a breakdown of completion, likewise** — a vendor that +//! reports reasoning separately still bills it as output, and on Anthropic the +//! count is zero because thinking is billed inside `output_tokens` and never +//! split out. +//! +//! **A `None` usage is not a zero usage.** io-harness stores `total_tokens` as +//! SQL `NULL` for a call the provider said nothing about, and reads it back as +//! `None` for exactly that reason. Summing it as zero would report a turn that +//! cost something as a turn that cost nothing. +//! +//! # The unit +//! +//! io-harness prices in micro-units and does not name a currency, because it is +//! not the layer that knows one. io-cli does: every catalogue it can read quotes +//! USD, and every provider it can connect to bills in it. So `$` is written, and +//! the page names the catalogue the rates came from and the date they were read, +//! which is the claim behind the symbol. + +use io_harness::pricing::{PriceTable, Spend}; +use io_harness::{ProviderCall, Store, Usage}; + +use crate::page::{self, Row}; +use crate::theme::Theme; + +// # What this page costs to draw, stated rather than discovered +// +// The `this run` and `this session` sections are bounded by construction: one +// run's calls, or one conversation's turns. The `by model` and `by day` sections +// are not. `Store::spend_by_model` and `Store::spend_by_day` each read **every** +// `provider_calls` row in the store and group in Rust — pricing is a Rust value +// SQLite cannot see, so it cannot be a `GROUP BY` — which makes both linear in the +// whole store and growing for the life of the install. +// +// **io-cli does not bound them, and the reason is that it cannot bound them +// honestly.** A `LIMIT` would have to be applied to rows rather than to groups, so +// a bounded read would report the most recent N calls under headings that say +// "by model" and "by day" without qualification — a partial total presented as a +// whole one, which is the exact failure the rest of this module is built to +// avoid. There is no public API to ask io-harness for the last N days grouped, and +// `ProviderCall` carries no timestamp of its own for io-cli to group by instead. +// +// So the release measures the cost against a realistically sized store, records +// the number, and leaves the read whole. The bound belongs upstream, beside the +// grouping, and is reported there. + +/// What one set of calls came to. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct Total { + /// Calls made, failed attempts included — a retry costs what it costs. + pub calls: u64, + /// Calls the provider reported no usage for. **Unknown, not zero.** + pub unknown: u64, + /// Calls whose model the table does not price. + pub unpriced: u64, + /// The token split, summed. Cache and reasoning are breakdowns of the prompt + /// and completion figures beside them, never additions to them. + pub usage: Usage, + /// Micro-units. A floor rather than a total whenever `unpriced` is non-zero. + pub micros: u64, +} + +impl Total { + /// Sum a run's calls, priced by `table`. + /// + /// `table` is never `Option`: an operator with no prices gets an empty table, + /// under which every call is unpriced and every token figure is still exact. + /// One code path rather than two, and the no-price case is the same page with + /// the money left off rather than a different page. + pub fn of(calls: &[ProviderCall], table: &PriceTable) -> Self { + let mut total = Self { + calls: calls.len() as u64, + ..Self::default() + }; + for call in calls { + let Some(usage) = call.usage else { + total.unknown += 1; + continue; + }; + add(&mut total.usage, &usage); + let priced = call + .model + .as_deref() + .and_then(|model| table.cost_micros(model, &usage)); + match priced { + Some(micros) => total.micros = total.micros.saturating_add(micros), + None => total.unpriced += 1, + } + } + total + } + + /// Whether the money figure is a floor rather than a total. + pub fn is_floor(&self) -> bool { + self.unpriced > 0 || self.unknown > 0 + } +} + +/// Add `from` into `into`, field by field. +/// +/// Written out rather than derived: `Usage` is `#[non_exhaustive]`-adjacent in +/// spirit — io-harness adds a dimension when a vendor starts billing one — and a +/// field added upstream that this function silently dropped would under-report a +/// bill. Spelling every field means the compiler names the new one. +fn add(into: &mut Usage, from: &Usage) { + into.prompt_tokens += from.prompt_tokens; + into.completion_tokens += from.completion_tokens; + into.total_tokens += from.total_tokens; + into.cache_read_tokens += from.cache_read_tokens; + into.cache_write_tokens += from.cache_write_tokens; + into.reasoning_tokens += from.reasoning_tokens; + into.server_tool_requests += from.server_tool_requests; +} + +/// Where the prices came from, for the line at the foot of the page. +/// +/// **The claim behind every `$` on the page**, and it is carried rather than +/// re-derived so that one sentence cannot drift from the figures above it. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Provenance { + /// The table's own `as_of`, or `None` when there is no table at all. + pub as_of: Option, + /// The catalogue, in words. `None` when nothing has been read yet. + pub source: Option, + /// How many models the table prices, as the last read recorded it, or `None` + /// where nothing recorded it. + /// + /// **Absent is not zero, and this field had the bug the rest of the module is + /// built to prevent.** It was a `usize` defaulting to 0, so a `[prices]` an + /// operator wrote by hand — or one carried forward from an install older than + /// this release, neither of which has an `[app.io-cli.prices]` beside it — + /// made the footer read "0 models" over a table pricing four hundred. + pub models: Option, +} + +impl Provenance { + /// What the configuration in force says about its prices. + /// + /// The date comes from io-harness's own `[prices]` section, because that is + /// the table actually pricing the calls; the source and the count come from + /// `[app.io-cli.prices]`, because io-harness models neither. A file carrying + /// prices but no io-cli section — hand-written, or written by an older + /// release — still gets its date, and the page says the source was not + /// recorded rather than inventing one. + pub fn of(config: &io_harness::Config, stored: Option<&crate::settings::CliSettings>) -> Self { + let prices = stored.and_then(|settings| settings.prices.as_ref()); + Self { + as_of: config.prices().map(|table| table.as_of().to_string()), + source: prices.and_then(|p| p.source.clone()), + models: prices.and_then(|p| p.models), + } + } +} + +/// The price table in force, or an empty one dated to nothing. +/// +/// **One code path rather than two.** An operator with no prices gets a table +/// that prices nothing, under which every call is unpriced and every token figure +/// is still exact — so the page is the same page with the money left off, rather +/// than a second page for the unconfigured case that would go stale unread. +pub fn table(config: &io_harness::Config) -> PriceTable { + config.prices().unwrap_or_else(|| PriceTable::new("")) +} + +/// The `/cost` page. +/// +/// `run` is the turn in flight or the last one; `session` is every turn of this +/// conversation. Both are `None` before anything has run, and the page says so +/// rather than drawing a row of zeroes — the same distinction the status line +/// holds every one of its counters to. +pub fn committed( + store: &Store, + table: &PriceTable, + provenance: &Provenance, + run: Option, + session: Option, + theme: &Theme, + width: u16, +) -> Result>, String> { + let mut rows: Vec = Vec::new(); + + if provenance.as_of.is_none() { + rows.push(Row::caveat( + "no prices are configured, so this page reports tokens and no money. \ + Prices arrive from the catalogue your provider serves when you connect \ + one, and `/config` refreshes them." + .to_string(), + )); + rows.push(Row::Blank); + } + + rows.push(Row::heading("this run".to_string())); + match run { + Some(id) => { + let calls = store.provider_calls(id).map_err(|e| e.to_string())?; + rows.extend(section(&Total::of(&calls, table))); + } + None => rows.push(Row::note("nothing has run in this session yet")), + } + + rows.push(Row::Blank); + rows.push(Row::heading("this session".to_string())); + match session { + Some(id) => { + let turns = store.session_turns(id).map_err(|e| e.to_string())?; + let mut calls: Vec = Vec::new(); + for turn in &turns { + calls.extend( + store + .provider_calls(turn.run_id) + .map_err(|e| e.to_string())?, + ); + } + // **Walked rather than grouped, because io-harness offers no session + // grouping.** `spend_by_run` keys on a run and `spend_by_day` on a + // date; a session is neither, and reporting one run's cost under the + // heading of a whole conversation would be the wrong number under the + // right word. + rows.extend(section(&Total::of(&calls, table))); + rows.push(Row::note(format!( + "across {} turn{}", + turns.len(), + if turns.len() == 1 { "" } else { "s" } + ))); + } + None => rows.push(Row::note("this session has no turns yet")), + } + + rows.push(Row::Blank); + rows.push(Row::heading("by model".to_string())); + rows.extend(grouped( + store.spend_by_model(table).map_err(|e| e.to_string())?, + )); + + rows.push(Row::Blank); + rows.push(Row::heading("by day".to_string())); + rows.extend(grouped( + store.spend_by_day(table).map_err(|e| e.to_string())?, + )); + + rows.push(Row::Blank); + rows.extend(footer(provenance)); + + Ok(page::commit("cost", &rows, theme, width)) +} + +/// The rows for one [`Total`]. +fn section(total: &Total) -> Vec { + if total.calls == 0 { + return vec![Row::note("no provider calls")]; + } + let usage = &total.usage; + let mut rows = vec![Row::fact("calls", total.calls.to_string())]; + // **No currency at all when nothing was priced, rather than `$0`.** A cost row + // reading `$0` over a set of calls the table prices none of would be the + // invented number this whole module exists to avoid — and it would be the + // worst kind, because zero is a number an operator can believe. The token + // figures below are exact either way; it is only the money that is unknown. + if total.micros > 0 { + rows.push(Row::fact("cost", money(total.micros))); + } + rows.push(Row::fact("prompt", tokens(usage.prompt_tokens))); + // The two cache figures are indented under the prompt they are part of, by + // being named as parts of it rather than by being drawn further right — this + // page has no columns to indent into, and a reader in `--plain` has only the + // words. + if usage.cache_read_tokens > 0 || usage.cache_write_tokens > 0 { + rows.push(Row::fact( + "of which cache read", + tokens(usage.cache_read_tokens), + )); + rows.push(Row::fact( + "of which cache written", + tokens(usage.cache_write_tokens), + )); + rows.push(Row::fact( + "of which fresh", + tokens( + usage + .prompt_tokens + .saturating_sub(usage.cache_read_tokens) + .saturating_sub(usage.cache_write_tokens), + ), + )); + } + rows.push(Row::fact("completion", tokens(usage.completion_tokens))); + if usage.reasoning_tokens > 0 { + rows.push(Row::fact( + "of which reasoning", + tokens(usage.reasoning_tokens), + )); + } + if usage.server_tool_requests > 0 { + rows.push(Row::fact( + "provider tool requests", + usage.server_tool_requests.to_string(), + )); + } + rows.extend(caveats(total.unknown, total.unpriced, total.micros > 0)); + rows +} + +/// The rows for a set of io-harness's own [`Spend`] groups. +fn grouped(spend: Vec) -> Vec { + if spend.is_empty() { + return vec![Row::note("nothing recorded")]; + } + let mut rows: Vec = Vec::new(); + for group in spend { + // Same rule as `section` above: no currency at all where nothing was + // priced. `$0` beside a group whose every call used a model the table does + // not know reads as a group that was free, and it is the one wrong number + // an operator has no reason to doubt. + let mut value = if group.cost_micros > 0 { + format!( + "{} · {} call{}", + money(group.cost_micros), + group.calls, + if group.calls == 1 { "" } else { "s" } + ) + } else { + format!( + "{} call{}", + group.calls, + if group.calls == 1 { "" } else { "s" } + ) + }; + if group.usage.total_tokens > 0 { + value.push_str(&format!(" · {} tok", tokens(group.usage.total_tokens))); + } + if group.unpriced_calls > 0 { + // **Never hidden, and on the row rather than in a note below it.** A + // reader scanning for the largest figure has to be able to see which + // of them is incomplete without reading past the list. + // + // **"not priced" rather than "unpriced", because it is a wider set + // than the word means two sections up.** io-harness's + // `Spend::unpriced_calls` counts a call with no usage as unpriced; + // this module's `Total` splits those out as *unknown* and counts only + // a missing rate as unpriced. The same call is therefore in different + // buckets on the two halves of this page, and one word for both would + // have made the numbers look like they disagreed. + value.push_str(&format!(" · {} not priced", group.unpriced_calls)); + } + rows.push(Row::fact(group.key, value)); + } + rows +} + +/// The sentences that qualify a figure, if any qualify it. +fn caveats(unknown: u64, unpriced: u64, drew_cost: bool) -> Vec { + let mut rows: Vec = Vec::new(); + if unpriced > 0 { + // **Two sentences, because the row this used to point at is not always + // there.** `section` suppresses the cost row entirely when nothing was + // priced, so a run every call of which used an unknown model showed no + // cost and a caveat reading "the cost above is a floor" — pointing at + // nothing. + rows.push(Row::caveat(if drew_cost { + format!( + "{unpriced} call{} used a model with no rate in the price table, so the cost \ + above is a floor and not a total", + if unpriced == 1 { "" } else { "s" } + ) + } else { + format!( + "no cost is shown because the price table has a rate for none of the {unpriced} \ + call{} here — the tokens above are exact, the money is simply unknown", + if unpriced == 1 { "" } else { "s" } + ) + })); + } + if unknown > 0 { + rows.push(Row::caveat(format!( + "{unknown} call{} reported no usage at all, which is unknown rather \ + than free — neither its tokens nor its cost is in the figures above", + if unknown == 1 { "" } else { "s" } + ))); + } + rows +} + +/// Where the prices came from and when, at the foot of the page. +fn footer(provenance: &Provenance) -> Vec { + let Some(as_of) = &provenance.as_of else { + return vec![Row::note("prices: none configured".to_string())]; + }; + // **The count is left out when nothing recorded one, rather than shown as + // zero.** A hand-written `[prices]`, or one from an install older than this + // release, has no `[app.io-cli.prices]` beside it — and "0 models" over a + // table pricing four hundred is the same class of invented figure as a `$0` + // over an unpriced call. + let how_many = match provenance.models { + Some(1) => "1 model".to_string(), + Some(n) => format!("{n} models"), + None => "prices".to_string(), + }; + let sentence = match &provenance.source { + Some(source) => format!("prices: {how_many} read from {source} on {as_of}"), + None => { + format!("prices: {how_many}, dated {as_of}, from a source this install did not record") + } + }; + vec![Row::note(sentence)] +} + +/// Micro-units as money. +/// +/// Four decimals below a unit and two above it: a turn costs a fraction of a cent +/// and a month costs dollars, and one precision cannot show both without either +/// rounding the turn to nothing or padding the month with noise. Integer +/// arithmetic throughout — a bill is not a floating-point quantity, and +/// `cost_micros` is exact. +pub fn money(micros: u64) -> String { + // **A real cost smaller than the smallest figure this renders is said to be + // smaller, not rounded to nothing.** Four decimals stop at a hundredth of a + // cent, and a cheap model answering a short prompt genuinely lands under it; + // truncating there would print `$0.0000`, which reads as free for a call that + // was not. The `$0` below is reserved for a cost that really is zero, which a + // free model has. + if micros > 0 && micros < 100 { + return "<$0.0001".to_string(); + } + if micros == 0 { + return "$0".to_string(); + } + let units = micros / 1_000_000; + let rest = micros % 1_000_000; + if units == 0 { + format!("$0.{:04}", rest / 100) + } else { + format!("${units}.{:02}", rest / 10_000) + } +} + +/// A token count, in the spelling the status line already uses. +fn tokens(count: u64) -> String { + crate::status::format_tokens(count) +} diff --git a/src/edit.rs b/src/edit.rs index 712e2e5..4061a58 100644 --- a/src/edit.rs +++ b/src/edit.rs @@ -70,6 +70,8 @@ enum Kind { Set, /// Append a whole `[[path]]` entry to the end of the document. Append, + /// Create the `[path]` section with a whole body, refusing if it exists. + Section, /// Remove a whole `[[path]]` entry, or a `[path]` section, bytes and all. Remove, /// Move one `[[path]]` entry to another position in its array. @@ -108,6 +110,32 @@ impl Edit { } } + /// Create the `[path]` section with `body` as its whole contents. + /// + /// **The shape neither [`Edit::set`] nor [`Edit::append`] can express, and the + /// gap is not academic.** `append` writes `[[path]]`, an array-of-tables + /// entry. `set` writes one key, and when the section it names does not exist + /// it appends a whole new header to carry that one key — which is correct for + /// one key and catastrophic for many: sixty `set`s into a file with no + /// `[prices.models]` each append their own `[prices.models]` header, because + /// every edit in a batch is resolved against the document as it was *before* + /// the batch. The result is sixty duplicate table definitions, and the whole + /// write is refused by the read-back — so the very first fill of a price table + /// could never succeed. + /// + /// `body` is the section's own `key = value` lines without the header; this + /// writes the header. **It refuses when the section already exists**, rather + /// than replacing it: a caller with an existing section wants `set` per key, + /// which preserves every row this one does not name, and silently discarding + /// an operator's hand-added rows is not a thing to do by accident. + pub fn section(path: impl Into, body: impl Into) -> Self { + Self { + path: path.into(), + value: body.into(), + kind: Kind::Section, + } + } + /// Remove the whole `[path]` section or `[[path]][index]` entry. pub fn remove(path: impl Into) -> Self { Self { @@ -203,6 +231,36 @@ pub fn apply(text: &str, edits: &[Edit]) -> Result { splices.push((text.len()..text.len(), block)); continue; } + Kind::Section => { + // **Refused rather than replaced when it already exists**, and the + // refusal is the point: a caller holding an existing section wants + // `set` per key, which leaves every row it does not name alone. + // Replacing here would silently discard whatever the operator had + // added by hand, from a call site whose author believed they were + // adding rows. + let names = segments(&edit.path)?; + if regions.iter().any(|r| r.path == names) { + return Err(format!( + "`[{}]` is already in this file, so it was not written whole — \ + a section that exists is edited key by key, which is what keeps \ + the rows nobody named", + edit.path + )); + } + let header = names + .iter() + .map(|name| spell(name)) + .collect::>() + .join("."); + let mut block = String::new(); + if !text.is_empty() && !text.ends_with('\n') { + block.push('\n'); + } + let body = edit.value.trim_end(); + block.push_str(&format!("\n[{header}]\n{body}\n")); + splices.push((text.len()..text.len(), block)); + continue; + } Kind::Move => { let (from, to) = edit .value @@ -210,7 +268,7 @@ pub fn apply(text: &str, edits: &[Edit]) -> Result { .and_then(|(a, b)| Some((a.parse::().ok()?, b.parse::().ok()?))) .ok_or_else(|| format!("`{}` has no positions to move between", edit.path))?; - let names = vec![edit.path.clone()]; + let names = segments(&edit.path)?; let at = |index: usize| { regions .iter() @@ -221,7 +279,9 @@ pub fn apply(text: &str, edits: &[Edit]) -> Result { continue; } let source = at(from)?; - let block = text[source.start..source.body.end].to_string(); + // Stops before any comment run trailing the entry, which belongs + // to the section below it. See [`removal_end`]. + let taken = source.start..removal_end(text, &source.body); // Both splices are computed against the ORIGINAL text and applied // right to left, which is what makes a move one pass rather than @@ -234,7 +294,25 @@ pub fn apply(text: &str, edits: &[Edit]) -> Result { // Moving up: land where that entry's header begins. destination.start }; - splices.push((source.start..source.body.end, String::new())); + + // **The guard [`Kind::Append`] has, in both directions.** The + // last region's `body.end` is the length of the file, so a move + // into last place against a file that does not end in a newline + // splices the header onto whatever the final line was — and when + // that line is a comment, the comment swallows the header, the + // moved entry's keys join the table above it, and the result + // parses cleanly as a different configuration. The mirror is a + // moved block that ends without a newline, which swallows the + // destination's header the same way. + let mut block = text[taken.clone()].to_string(); + if !block.ends_with('\n') { + block.push('\n'); + } + if insert > 0 && !text[..insert].ends_with('\n') { + block.insert(0, '\n'); + } + + splices.push((taken, String::new())); splices.push((insert..insert, block)); continue; } @@ -263,7 +341,10 @@ pub fn apply(text: &str, edits: &[Edit]) -> Result { .iter() .find(|r| r.path == names && r.index == index) .ok_or_else(|| format!("there is no `{}` to remove in this file", edit.path))?; - splices.push((region.start..region.body.end, String::new())); + // Not `region.body.end`: that is the first byte of the next + // header, and the lines just above it document the next section + // rather than this one. See [`removal_end`]. + splices.push((region.start..removal_end(text, ®ion.body), String::new())); continue; } Kind::Set => {} @@ -300,7 +381,11 @@ pub fn apply(text: &str, edits: &[Edit]) -> Result { // end of this section rather than the end of the file, // where it would silently join whatever section is last. let at = insertion_point(text, ®ion.body); - splices.push((at..at, format!("{key} = {}\n", edit.value))); + // `key` is DECODED, so it is spelled back rather than + // pasted: a name with a dot or a space in it written bare + // is a dotted key or a parse error, never the key asked + // for. + splices.push((at..at, format!("{} = {}\n", spell(&key), edit.value))); } } } @@ -313,12 +398,17 @@ pub fn apply(text: &str, edits: &[Edit]) -> Result { if resolve(&parsed, &table_path.names, &key).is_some() { return Err(dotted_refusal(&edit.path)); } - let header = table_path.names.join("."); + let header = table_path + .names + .iter() + .map(|name| spell(name)) + .collect::>() + .join("."); let mut block = String::new(); if !text.is_empty() && !text.ends_with('\n') { block.push('\n'); } - block.push_str(&format!("\n[{header}]\n{key} = {}\n", edit.value)); + block.push_str(&format!("\n[{header}]\n{} = {}\n", spell(&key), edit.value)); splices.push((text.len()..text.len(), block)); } } @@ -494,8 +584,8 @@ fn split_path(path: &str) -> Result<(TablePath, String), String> { let mut names: Vec = Vec::new(); let mut index = 0usize; - let segments: Vec<&str> = path.split('.').collect(); - let (key, table) = segments + let parts = segments(path)?; + let (key, table) = parts .split_last() .ok_or_else(|| format!("`{path}` names no key"))?; @@ -511,14 +601,123 @@ fn split_path(path: &str) -> Result<(TablePath, String), String> { .map_err(|_| format!("`{path}` has an index that is not a number"))?; names.push(name.to_string()); } else { - names.push(segment.to_string()); + names.push(segment.clone()); } } if key.is_empty() { return Err(format!("`{path}` names no key")); } - Ok((TablePath { names, index }, (*key).to_string())) + Ok((TablePath { names, index }, key.clone())) +} + +/// Cut a dotted TOML path into its segments, **decoded**. +/// +/// The one splitter, used by both halves of this module, and that is the point +/// of it. A dot inside a quoted key is not a separator — TOML spells a bare key +/// out of `A-Za-z0-9_-` and nothing else, so a key carrying a dot can only ever +/// be written quoted (`"gpt-4.1"`, `"github.com/x"`) and `path.split('.')` cuts +/// it in half. [`split_path`] and [`regions`] each did exactly that and then +/// normalised what came out DIFFERENTLY, so a caller's path and the file's own +/// header never matched: the read half answered `None` for any dotted key and +/// the write half fell through to the append arm and emitted a second copy of a +/// table that was already there, caught only by the read-back in [`apply`] and +/// only as "would have produced a file that does not parse". +/// +/// The segments come back decoded, which is the half `trim_matches('"')` cannot +/// do: a basic string takes the full escape set, so `"a\"b"` is a legal key +/// whose name is `a"b`, and trimming strips repeated quotes off both ends and +/// resolves no escape at all. A literal string `'...'` takes no escapes, so a +/// `'` can never appear inside one and finding its end is a search. The decoding +/// is `toml`'s own for the reason [`array()`] spells through `toml` rather than a +/// format string — the escape rules belong to the format, not to this file. +/// [`spell`] is the inverse, for the two places a segment goes back into a +/// document. +/// +/// Whitespace around a dot is legal (`[ a . b ]`), so an unquoted segment is +/// trimmed and a quoted one is taken exactly as the quotes deliver it. +fn segments(path: &str) -> Result, String> { + let bytes = path.as_bytes(); + let mut out: Vec = Vec::new(); + let mut i = 0usize; + + loop { + while matches!(bytes.get(i), Some(b' ' | b'\t')) { + i += 1; + } + match bytes.get(i).copied() { + Some(quote) if quote == b'"' || quote == b'\'' => { + // Find the closing quote. A backslash in a basic string escapes + // whatever follows it, including the quote itself; a literal + // string has no escapes, so the first `'` ends it. + let mut j = i + 1; + let end = loop { + match bytes.get(j).copied() { + None => { + return Err(format!( + "`{path}` has a quoted segment that is never closed" + )) + } + Some(b'\\') if quote == b'"' => j += 2, + Some(c) if c == quote => break j, + Some(_) => j += 1, + } + }; + // `toml` decodes it, because `toml` is what wrote the rules. + let literal = &path[i..=end]; + let decoded = toml::from_str::(&format!("probe = {literal}")) + .ok() + .and_then(|table| Some(table.get("probe")?.as_str()?.to_string())) + .ok_or_else(|| { + format!("`{path}` has a quoted segment that is not a TOML string") + })?; + out.push(decoded); + i = end + 1; + while matches!(bytes.get(i), Some(b' ' | b'\t')) { + i += 1; + } + if !matches!(bytes.get(i), None | Some(b'.')) { + return Err(format!( + "`{path}` has more than a quoted string in one of its segments" + )); + } + } + _ => { + let end = path[i..].find('.').map_or(bytes.len(), |n| i + n); + out.push(path[i..end].trim().to_string()); + i = end; + } + } + if i >= bytes.len() { + return Ok(out); + } + i += 1; // The dot. + } +} + +/// Spell one decoded segment the way a document has to carry it. +/// +/// The inverse of the decoding `segments` does, and it exists because that +/// decoding created the need: a segment in hand is now a NAME, and a name is not +/// always spellable as itself. A bare key is `A-Za-z0-9_-` and nothing else, so +/// anything holding a dot, a space, a slash or a quote has to go back into the +/// file quoted — and writing `[prices.models.gpt-4.1]` instead of +/// `[prices.models."gpt-4.1"]` names a different table two levels deeper, which +/// still parses. +/// +/// The quoting goes through `toml`'s own serializer for the same reason +/// [`array()`] does: a format string cannot escape a `"` or a backslash correctly +/// and would produce either a parse error or a different name. +#[must_use] +pub fn spell(segment: &str) -> String { + let bare = !segment.is_empty() + && segment + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-'); + if bare { + return segment.to_string(); + } + toml::Value::String(segment.to_string()).to_string() } /// Whether the parsed document already holds this path, in any shape. @@ -550,6 +749,43 @@ fn insertion_point(text: &str, body: &Range) -> usize { } } +/// Where a region's bytes end, for the purpose of taking the region away. +/// +/// **A region's `body.end` is the first byte of the NEXT header**, which is what +/// makes it right for parsing the section and wrong for deleting it: everything +/// between this section's last key and that header is inside the range, and what +/// sits there is almost always the comment block an operator wrote to explain the +/// section BELOW. Splicing `region.start..region.body.end` away took somebody +/// else's sentence with it, and a moved entry carried it to a place it was no +/// longer true. +/// +/// So the trailing lines are walked backwards over blank and comment lines, and +/// the answer is the first line of that trailing comment run — `body.end` when +/// there is no comment in it. The blank lines ABOVE the run stay inside the range +/// on purpose: they belonged to the section being taken away, and leaving them +/// would make every removal add an empty line to the file. +fn removal_end(text: &str, body: &Range) -> usize { + let slice = &text[body.clone()]; + let mut starts: Vec = vec![0]; + starts.extend(slice.match_indices('\n').map(|(at, _)| at + 1)); + let mut answer = body.end; + for &start in starts.iter().rev() { + // The empty remainder after a final newline is not a line. + if start >= slice.len() { + continue; + } + let line = slice[start..].split('\n').next().unwrap_or_default().trim(); + if line.is_empty() { + continue; + } + if !line.starts_with('#') { + break; + } + answer = body.start + start; + } + answer +} + /// Cut the document into regions at every table header. /// /// The scan is a character state machine rather than a line filter, because a @@ -663,10 +899,10 @@ fn regions(text: &str) -> Result, String> { .or_else(|| header.strip_prefix('[').and_then(|h| h.strip_suffix(']'))) .ok_or_else(|| format!("`{header}` is not a section header this editor understands"))?; - let path: Vec = inner - .split('.') - .map(|s| s.trim().trim_matches('"').trim_matches('\'').to_string()) - .collect(); + // The same splitter the caller's path goes through, which is the whole + // reason there is only one: two scans that agree about what a segment is + // are the only way a header and a path can ever be compared. + let path = segments(inner)?; let index = seen.entry(path.clone()).or_insert(0); let body_end = header_starts.get(n + 1).copied().unwrap_or(bytes.len()); diff --git a/src/import.rs b/src/import.rs index 5e52bac..bccaa41 100644 --- a/src/import.rs +++ b/src/import.rs @@ -494,6 +494,30 @@ pub fn plan(found: &[Found], home: &Path, scope: Scope) -> Vec { .file_name() .map(|name| name.to_string_lossy().to_string()) .unwrap_or_default(); + + // **Where a file came from decides what it is, and the basename + // match below cannot see that.** [`skill_files`] counts any loose + // `*.md` at the top of a skills root, so `~/.claude/skills/ + // CONVENTIONS.md` — or `CLAUDE.md`, or `MEMORY.md` — arrives here as + // a `Source::Claude` path wearing an anchor file's name, and the arm + // below matched it as standing instructions: the whole body appended + // into the file that is loaded on EVERY turn, forever, instead of a + // named skill read on demand. + // + // Pairing each anchor name with its owning `Source` is the smaller + // diff and it does not close this: `~/.claude/skills/CLAUDE.md` is + // genuinely `Source::Claude` and genuinely named `CLAUDE.md`, and it + // is still a skill. So the guard asks where the file is, ahead of + // asking what it is called. + let rooted = path.parent().and_then(Path::file_name).is_some_and(|dir| { + dir.to_str() + .is_some_and(|dir| dir == "skills" || dir == "plugins") + }); + if name == SKILL_FILE || rooted { + skills.push((one.source, path.clone())); + continue; + } + match (one.source, name.as_str()) { ( _, @@ -509,10 +533,13 @@ pub fn plan(found: &[Found], home: &Path, scope: Scope) -> Vec { (Source::Gemini, "mcp_config.json") => { items.extend(foreign_mcp(one.source, path, scope)); } - // Everything [`files`] can produce is named above except the - // skill files, so this arm is exactly those. Keep it that way: - // a new anchor file added to `files` without a case here would - // silently arrive as a skill. + // Every skill file has already been taken by the guard above, so + // what reaches here is a file [`files`] produces and no arm + // claims. It is treated as a skill, which is the safe end of the + // mistake — a skill is read on demand and an instructions file is + // read on every turn. Keep the guard ahead of this: a new anchor + // file added to `files` without a case above still arrives as a + // skill, and that is a plan an operator can see and decline. _ => skills.push((one.source, path.clone())), } } @@ -669,17 +696,25 @@ fn codex_config(path: &Path, scope: Scope) -> Vec { items.push(model_item(Source::Codex, path, &model)); } + // **Two segments exactly**, which is the server's own table and not its + // `[mcp_servers..env]` child. That was already the right filter and it + // was throwing servers away, because `sections` split `[mcp_servers. + // "github.com/x"]` on the dot inside the quoted name and reported three + // segments — a name that can ONLY be spelled quoted was the one shape it + // could not see. `edit::segments` decides that now, and the name arrives + // decoded, so every path built back out of it goes through `edit::spell`. for name in crate::edit::sections(&text) .into_iter() .filter(|section| section.len() == 2 && section[0] == "mcp_servers") .map(|section| section[1].clone()) { - let Some(command) = crate::edit::value_at(&text, &format!("mcp_servers.{name}.command")) + let quoted = crate::edit::spell(&name); + let Some(command) = crate::edit::value_at(&text, &format!("mcp_servers.{quoted}.command")) .map(|v| unquoted(&v)) else { continue; }; - let args = crate::edit::value_at(&text, &format!("mcp_servers.{name}.args")) + let args = crate::edit::value_at(&text, &format!("mcp_servers.{quoted}.args")) .map(|value| strings(&value)) .unwrap_or_default(); let names = env_names(&text, &name); @@ -710,7 +745,10 @@ fn codex_config(path: &Path, scope: Scope) -> Vec { // config.toml with a `"""` block above its env table is not a thing that exists; // if one turns up, expose `regions` from `edit` and use it here. fn env_names(text: &str, server: &str) -> Vec { - let header = format!("[mcp_servers.{server}.env]"); + // `server` is a decoded name, so it is spelled back before it is compared + // against a line of the file — a name with a dot in it is written quoted in + // the header and matching the raw name would find no table at all. + let header = format!("[mcp_servers.{}.env]", crate::edit::spell(server)); let mut names = Vec::new(); let mut inside = false; for line in text.lines() { diff --git a/src/lib.rs b/src/lib.rs index 1717955..7aa00d8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -41,6 +41,7 @@ pub mod composer; pub mod configure; pub mod context; pub mod contract; +pub mod cost; pub mod diff; pub mod edit; pub mod editor; @@ -57,10 +58,12 @@ pub mod intent; pub mod keys; pub mod markdown; pub mod memory; +pub mod page; pub mod picker; pub mod picture; pub mod plan; pub mod pluginview; +pub mod prices; pub mod provider; pub mod providers; pub mod queue; @@ -74,6 +77,7 @@ pub mod shell; pub mod skills; pub mod skillview; pub mod splash; +pub mod stats; pub mod status; pub mod stdin; pub mod term; diff --git a/src/main.rs b/src/main.rs index 0becfde..16c8ad2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1825,11 +1825,27 @@ async fn loop_over Result>( } } } - Pick::Config(paths) => { - if let Some(key) = paths.get(index) { - app.composer.set(&format!("/config {key} ")); + Pick::Config(paths) => match paths.get(index).map(String::as_str) { + // The one row on this surface that acts rather than + // naming something. It re-reads the catalogue the + // operator's provider serves and writes what moved + // into the scope that already declares the prices — + // or the user scope, for a first fill. + Some(io_cli::configure::REFRESH_PRICES) => { + refresh_prices(screen, &mut app, &config, &spec, session.root()) + .await?; + match io_cli::configure::reload(session.root()) { + Ok((fresh, stored)) => { + capabilities = + io_cli::contract::Capabilities::stored(stored.as_ref()); + config = fresh; + } + Err(error) => app.record(Tone::Error, error), + } } - } + Some(key) => app.composer.set(&format!("/config {key} ")), + None => {} + }, Pick::ConfigScope { key, value, paths } => { // An index past the end puts nothing in the file, // which is the same answer every other picker arm @@ -2428,11 +2444,18 @@ async fn loop_over Result>( } Action::Config(None) => { let settings = io_cli::configure::settings(&config); - let paths: Vec = settings.iter().map(|s| s.path.clone()).collect(); - picker = Some(( - Picker::new("Which setting?", io_cli::configure::rows(&settings)), - Pick::Config(paths), - )); + let mut paths: Vec = settings.iter().map(|s| s.path.clone()).collect(); + let mut rows = io_cli::configure::rows(&settings); + // **Last, because it is the one row that acts.** Every other + // row names a setting and puts it in the composer; a reader + // scanning for `policy.defaults.write` should not have to step + // over something that does work on the way there. + rows.push(io_cli::configure::refresh_row(&io_cli::configure::setting( + &config, + "prices.as_of", + ))); + paths.push(io_cli::configure::REFRESH_PRICES.to_string()); + picker = Some((Picker::new("Which setting?", rows), Pick::Config(paths))); } // A key with no value is a question. Naming what is in force and // which file decided it is the answer, and nothing is written. @@ -2822,6 +2845,37 @@ async fn loop_over Result>( ); screen.commit(&lines).map_err(|error| error.to_string())?; } + // The sibling of `Action::Status` above and wired the same way, + // with one difference worth stating: this arm holds no decision + // at all. Every sentence, every figure and every caveat is + // `io_cli::cost::committed`, because nothing under `tests/` can + // link this file — so a format string here would be a format + // string no test could ever read. + Action::Cost => { + // Asked of the configuration again rather than threaded down + // from the driver's own binding, for the reason stated where + // `settings_in_force` is: `settings::stored` is a pure + // function of the config, so a second call cannot disagree + // with the first, and a threaded copy could go stale behind a + // `/config` write that reloaded one and not the other. + let (settings, _) = settings::stored(&config); + let table = io_cli::cost::table(&config); + let provenance = io_cli::cost::Provenance::of(&config, settings.as_ref()); + let lines = io_cli::cost::committed( + &store, + &table, + &provenance, + last_run(&session, &store).map(|turn| turn.run_id), + Some(session.id()), + &app.theme, + screen.width(), + )?; + screen.commit(&lines).map_err(|error| error.to_string())?; + } + Action::Stats => { + let lines = io_cli::stats::committed(&store, &app.theme, screen.width())?; + screen.commit(&lines).map_err(|error| error.to_string())?; + } // Reached only at an idle prompt: while a turn runs the driver's // own key handler answers `/steer` before `parse` is ever called, // because that is where the inbox lives. So this arm is the @@ -3519,6 +3573,7 @@ async fn turn( // same turn. The trace is the fallback for the window between a // step landing and the first completion call being seen. note_context(app, store, &event, seen, &contract); + note_cost(app, store, config, &event); note_fleet(app, store, &event, &contract); commit_viewed(screen, app, &root, policy, &event)?; commit_fold(app, store, &event, &mut folding); @@ -3854,6 +3909,10 @@ async fn turn( // for: the last step of a turn is exactly the one whose event the select // loop loses to the turn's own return. note_context(app, store, &event, seen, &contract); + // And on the drain too, for the same race: the last step of a turn is the + // one whose event the select loop loses, and it is also the step that + // makes the largest single difference to what the turn cost. + note_cost(app, store, config, &event); note_fleet(app, store, &event, &contract); // And the picture, for the same reason and the same race: a `view_image` // on the turn's last step is exactly the one the drain would otherwise @@ -4082,6 +4141,41 @@ fn note_context( } } +/// Set the status line's cost field from what this run has actually called. +/// +/// Beside [`note_context`] and on the same events, because the two are the same +/// shape of fact: both are read from the store rather than accumulated off the +/// stream, both change only when a step lands, and both are decorative enough +/// that a failed read is a missing field rather than a notice. +/// +/// **The table is asked of the configuration on every call rather than held.** A +/// `/config` write or a price refresh mid-session changes what a turn costs, and a +/// table captured when the session opened would go on reporting the old rate for +/// the rest of it — which is the one thing a figure with a currency in front of it +/// must not do. +fn note_cost( + app: &mut App, + store: &Store, + config: &io_harness::Config, + event: &io_harness::RunEvent, +) { + // **Anchored on a step, for the reason `note_context` three functions up is** + // — and the doc on both this and `Status::note_cost_from` claimed it before + // the code did. Without the gate every event in the stream, token deltas + // included, cost one `provider_calls` read and one full `Total::of`; with it, + // the read happens when the answer can actually have changed. `Finished` + // carries the run's own totals and is the one that settles the last step, + // which the drain would otherwise lose. + if !matches!( + event.kind, + io_harness::EventKind::Step { .. } | io_harness::EventKind::Finished { .. } + ) { + return; + } + let table = io_cli::cost::table(config); + app.status.note_cost_from(store, event.run_id, &table); +} + /// Report a fold that was asked for — once, and only from the event that /// announces it. /// @@ -4233,6 +4327,212 @@ fn to_copy(session: &Session, store: &Store, what: Copied) -> (Option, S } } +/// The ids of a catalogue read, sorted, for a picker. +/// +/// The wizard wants names and [`io_cli::prices`] wants rates, and both come off +/// the same rows. Splitting them here is what stops the catalogue being fetched +/// twice for one screen. +fn ids(served: &[io_harness::ModelInfo]) -> Vec { + let mut ids: Vec = served.iter().map(|model| model.id.clone()).collect(); + ids.sort(); + ids.dedup(); + ids +} + +/// Write a catalogue read into `path` as a `[prices]` section, and say what +/// happened. +/// +/// **The clock is read here because here is the only place it may be read.** +/// `tests/timing.rs` permits `SystemTime::now` in this file and refuses it in +/// every other file under `src/`, so the driver takes the number and +/// [`io_cli::prices::date`] converts it. That is a gate rather than a preference, +/// and it is also what makes the conversion testable. +/// +/// `existing` is how many models the table being replaced priced, which is what +/// [`io_cli::prices::Catalogue::too_short`] refuses a short answer against. Zero +/// for a first fill, which is never refused. +/// +/// Never an error. A catalogue that could not be read is an operator with no +/// prices, which is a supported state — `/cost` draws tokens and no currency and +/// says so. A first run that failed because a price list did not arrive would be a +/// first run failed over a decoration. +fn fill_prices( + path: &std::path::Path, + spec: &io_harness::ProviderSpec, + served: Vec, + existing: usize, + theme: &Theme, +) -> Vec> { + let secs = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|since| since.as_secs()) + .unwrap_or(0); + let catalogue = io_cli::prices::Catalogue::of(spec, served, io_cli::prices::date(secs)); + if catalogue.rows.is_empty() { + return vec![theme.notice( + Tone::Muted, + format!( + "no prices were written: the catalogue served {} model{} and priced none of them, \ + so /cost will report tokens and no money", + catalogue.served, + if catalogue.served == 1 { "" } else { "s" } + ), + )]; + } + // Read rather than assumed, even though the wizard has just written this file + // from `settings::render` and `settings::render` carries no `[prices]` at + // all. The assumption is true today and is exactly the kind that stops being + // true when somebody adds a section to the rendered file, and the failure it + // would cause — a refused write on a first run — is one nobody would connect + // back to here. + let text = std::fs::read_to_string(path).unwrap_or_default(); + if let Some(refusal) = io_cli::prices::refusal(&text) { + return vec![theme.notice(Tone::Warning, refusal)]; + } + // Counted off the file rather than off io-cli's own record of a previous + // write, which is absent on a first fill and on any hand-written table — see + // `prices::priced_in`. + let existing = io_cli::prices::priced_in(&text).max(existing); + if catalogue.too_short(existing) { + return vec![theme.notice( + Tone::Warning, + format!( + "the catalogue answered with {} priced model{} where {existing} were expected, \ + which is short enough to be a truncated read — the prices you have were kept", + catalogue.rows.len(), + if catalogue.rows.len() == 1 { "" } else { "s" } + ), + )]; + } + let source = io_cli::prices::source_word(&catalogue.source); + let mut edits = catalogue.edits(io_cli::prices::has_models_section(&text)); + edits.extend(io_cli::prices::bookkeeping( + &source, + catalogue.rows.len(), + io_cli::edit::sections(&text) + .iter() + .any(|p| p == &["app", "io-cli", "prices"]), + )); + match io_cli::edit::write(path, &edits) { + Ok(()) => vec![theme.notice( + Tone::Success, + format!( + "priced {} of the {} models {source} serves, as of {}", + catalogue.rows.len(), + catalogue.served, + catalogue.as_of + ), + )], + // Reported and not fatal, and the sentence says what was lost rather than + // that something went wrong: an operator who knows they have no prices can + // go and get them, and one told "an error occurred" cannot. + Err(error) => vec![theme.notice( + Tone::Warning, + format!("no prices were written ({error}); /cost will report tokens and no money"), + )], + } +} + +/// Re-read the price catalogue and write what moved. +/// +/// **Everything that would change is committed before anything is written**, which +/// is the shape `/import` established in 0.21.0 and it is here for a sharper +/// reason. io-cli cannot tell a rate the operator corrected by hand from one an +/// older catalogue served — the file records a number, not where it came from — so +/// it does not guess which is which. It shows every rate that would move, with +/// what it was and what it would become, and the operator who sees their own +/// correction in that list can decline the lot. +/// +/// The scope written is the one that already declares `prices.as_of`, so a refresh +/// lands where the last one did rather than shadowing it from a higher layer. A +/// first fill goes to the user scope, which is where the wizard writes. +async fn refresh_prices( + screen: &mut Screen>, + app: &mut App, + config: &Config, + spec: &io_harness::ProviderSpec, + root: &std::path::Path, +) -> Result<(), String> { + let (stored, _) = settings::stored(config); + let source = stored + .as_ref() + .and_then(|s| s.prices.as_ref()) + .and_then(|p| p.source_url.clone()); + // The scope that already holds the prices, so a refresh lands where the last + // one did. A higher-precedence scope would shadow rather than update, and the + // operator would be looking at the old numbers in the file they edit. + let scope = io_cli::configure::setting(config, "prices.as_of") + .decided + .scope() + .unwrap_or(io_harness::config::Scope::User); + // Read of the file this write is aimed at, not of the merged configuration: a + // `[prices.models]` in a lower-precedence scope is not one this scope's file + // can have keys set into, and treating it as though it were would try to edit + // a section that is not in this document. + let text = io_cli::configure::scope_path(root, scope) + .and_then(|path| std::fs::read_to_string(path).ok()) + .unwrap_or_default(); + if let Some(refusal) = io_cli::prices::refusal(&text) { + app.record(Tone::Warning, refusal); + return Ok(()); + } + // Counted off the file, and only then falling back to io-cli's own record. + // The record is absent for a hand-written table and for every install from + // before this release, which is precisely when a truncated read would do the + // most damage. + let existing = io_cli::prices::priced_in(&text).max( + stored + .as_ref() + .and_then(|s| s.prices.as_ref()) + .and_then(|p| p.models) + .unwrap_or(0), + ); + + app.say(Tone::Muted, "re-reading the catalogue…"); + let served = verify::served(source.as_deref()).await; + + let secs = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|since| since.as_secs()) + .unwrap_or(0); + let catalogue = match source.as_deref().filter(|url| !url.is_empty()) { + Some(url) => { + io_cli::prices::Catalogue::named(spec, served, io_cli::prices::date(secs), url) + } + None => io_cli::prices::Catalogue::of(spec, served, io_cli::prices::date(secs)), + }; + + let moved = io_cli::prices::changes(config.prices().as_ref(), &catalogue); + let lines = io_cli::prices::report(&catalogue, &moved, existing, &app.theme, screen.width()); + screen.commit(&lines).map_err(|error| error.to_string())?; + if catalogue.rows.is_empty() || moved.is_empty() || catalogue.too_short(existing) { + return Ok(()); + } + + let source_word = io_cli::prices::source_word(&catalogue.source); + let mut edits = catalogue.edits(io_cli::prices::has_models_section(&text)); + edits.extend(io_cli::prices::bookkeeping( + &source_word, + catalogue.rows.len(), + io_cli::edit::sections(&text) + .iter() + .any(|p| p == &["app", "io-cli", "prices"]), + )); + match io_cli::configure::write(root, scope, &edits) { + Ok(()) => app.record( + Tone::Success, + format!( + "{} rate{} written, dated {}", + moved.len(), + if moved.len() == 1 { "" } else { "s" }, + catalogue.as_of + ), + ), + Err(error) => app.record(Tone::Error, error), + } + Ok(()) +} + /// The first-run wizard. Returns the theme chosen, or `None` if it was abandoned. async fn wizard( screen: &mut Screen>, @@ -4240,6 +4540,14 @@ async fn wizard( theme: Theme, ) -> Result, String> { let mut wizard = Wizard::new(theme); + // **What the catalogue read already returned, kept instead of thrown away.** + // The wizard reads the provider's catalogue to offer a model list, and until + // 0.22.0 mapped every row down to its id and dropped the price on it. Holding + // the rows here is what lets the file this wizard writes arrive with prices in + // it, at the cost of no second call: the fetch is the one that was already + // being made, one step earlier in this same loop. + let mut served: Vec = Vec::new(); + let mut priced_for: Option = None; loop { screen .draw(|frame| wizard.render(frame, frame.area())) @@ -4279,8 +4587,9 @@ async fn wizard( match verify::credential(&spec).await { Ok(()) => { if let Progress::Catalogue(spec) = wizard.verified() { - let models = verify::catalogue(&spec).await; - wizard.catalogue(models); + served = verify::named(&spec, verify::served(None).await); + wizard.catalogue(ids(&served)); + priced_for = Some(spec); } } Err(message) => { @@ -4289,19 +4598,32 @@ async fn wizard( } } Progress::Catalogue(spec) => { - let models = verify::catalogue(&spec).await; - wizard.catalogue(models); + served = verify::named(&spec, verify::served(None).await); + wizard.catalogue(ids(&served)); + priced_for = Some(spec); } Progress::Write(path, contents) => { settings::write(&path, &contents) .map_err(|error| format!("could not write {}: {error}", path.display()))?; let theme = wizard.theme(); - screen - .commit(&[ - theme.notice(Tone::Success, format!("wrote {}", path.display())), - ratatui::text::Line::from(""), - ]) - .map_err(|error| error.to_string())?; + let mut lines = + vec![theme.notice(Tone::Success, format!("wrote {}", path.display()))]; + // **The prices go into the file the wizard just wrote, not into a + // second one.** `settings::render` cannot carry them: it runs + // before the credential is checked and therefore before any + // catalogue has been read, and a section written from a fetch that + // has not happened would be a date with nothing behind it. + if let Some(spec) = &priced_for { + lines.extend(fill_prices( + &path, + spec, + std::mem::take(&mut served), + 0, + &theme, + )); + } + lines.push(ratatui::text::Line::from("")); + screen.commit(&lines).map_err(|error| error.to_string())?; return Ok(Some(theme)); } } diff --git a/src/page.rs b/src/page.rs new file mode 100644 index 0000000..86ebfae --- /dev/null +++ b/src/page.rs @@ -0,0 +1,190 @@ +//! The shape of a page committed into the scrollback. +//! +//! **Four surfaces draw one, and until 0.22.0 two of them each carried their own +//! copy of the folding.** `crate::status::committed` and +//! `crate::context::committed` had the same twenty lines under two names, +//! `folded` and `wrapped`, differing only in that one took its indents as +//! arguments and the other hard-coded the same two numbers. `/cost` and `/stats` +//! would have been the third and fourth copies, so the folding lives here and +//! all four call it. +//! +//! # What a committed page is +//! +//! Three rule glyphs and a title, one fact per row, three rule glyphs and the +//! title again with `ends`. It lands in the terminal's own scrollback beside every +//! earlier turn, which is why it has edges at all: a passage with no edges is one +//! a reader cannot tell the extent of. +//! +//! **It is not a table, and that is a decision about eighty columns rather than +//! about taste** — the argument is written out in full at +//! `crate::status::committed` and it holds identically here. A table has a column +//! width, a column width is decided by the widest cell, and on these two pages the +//! widest cell is a model id. One fact per row, `label: value`, nothing padded +//! into a column and nothing aligned across rows, so there is no width for +//! anything to be squeezed out of. A row too long for the terminal is **folded**, +//! never cut. +//! +//! # Sections, which the older two pages did not need +//! +//! `/status` and `/context` are each one list. `/cost` and `/stats` are seven and +//! eight lists respectively — this run, this session, by model, by day — and a +//! reader has to be able to tell which figure belongs to which question. So a +//! page here is a sequence of [`Row`]s rather than a flat vector of pairs, and a +//! [`Row::Heading`] is what separates them. It is drawn in the accent tone and +//! indented to zero, so it reads as a break rather than as another fact. + +use ratatui::text::{Line, Span}; + +use crate::theme::{Theme, Tone}; + +/// One row of a committed page. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Row { + /// `label: value`, folded at two columns and continued at four. + Fact(String, String), + /// A section break within the page. + Heading(String), + /// A whole sentence, in a tone of its own. + /// + /// What a page says when it has no figure to give: that the price table is + /// empty, that a total is a floor because some calls were unpriced, that a + /// provider reported no usage for a call. **These are the rows that make the + /// figures trustworthy**, so they are rows rather than a footnote somewhere + /// else, and they carry a tone because "this total is incomplete" is not the + /// same kind of statement as "this total is 3,412 tokens". + Note(String, Tone), + /// A blank row, for separating a heading from what came before it. + Blank, +} + +impl Row { + /// `label: value`, taking anything that can spell itself. + pub fn fact(label: impl Into, value: impl Into) -> Self { + Self::Fact(label.into(), value.into()) + } + + /// A section break. + pub fn heading(text: impl Into) -> Self { + Self::Heading(text.into()) + } + + /// A sentence in the ordinary tone. + pub fn note(text: impl Into) -> Self { + Self::Note(text.into(), Tone::Muted) + } + + /// A sentence that qualifies a figure above it. + /// + /// `Warning` rather than `Muted`, because a total that is a floor and a total + /// that is a total must not read the same at a glance. This is the tone the + /// "lying by omission" rule in io-harness's own pricing documentation exists + /// to prevent a renderer from skipping. + pub fn caveat(text: impl Into) -> Self { + Self::Note(text.into(), Tone::Warning) + } +} + +/// A whole page, edged and folded, ready for `Screen::commit`. +/// +/// `title` appears in both rules, the closing one suffixed with `ends`, which is +/// the edge `crate::status::committed` and `crate::transcript` already give a +/// committed passage. +pub fn commit(title: &str, rows: &[Row], theme: &Theme, width: u16) -> Vec> { + let rule = theme.glyphs.rule; + let room = width as usize; + let mut lines = vec![Line::from(Span::styled( + format!("{rule}{rule}{rule} {title}"), + theme.style(Tone::Accent), + ))]; + for row in rows { + match row { + Row::Fact(label, value) => { + for text in folded(&format!("{label}: {value}"), room, 2, 4) { + lines.push(Line::from(Span::styled(text, theme.style(Tone::Normal)))); + } + } + // Indented to zero rather than to two, so a heading is distinguishable + // from a fact by position as well as by colour — which is the whole of + // what a reader in `--plain` or under `NO_COLOR` has to go on. + Row::Heading(text) => { + for text in folded(text, room, 0, 2) { + lines.push(Line::from(Span::styled(text, theme.style(Tone::Accent)))); + } + } + Row::Note(text, tone) => { + for text in folded(text, room, 2, 4) { + lines.push(Line::from(Span::styled(text, theme.style(*tone)))); + } + } + Row::Blank => lines.push(Line::from(String::new())), + } + } + lines.push(Line::from(Span::styled( + format!("{rule}{rule}{rule} {title} ends"), + theme.style(Tone::Accent), + ))); + lines +} + +/// `text` as rows no wider than `width`, indented `first` and then `rest`. +/// +/// **Folded and never fitted**, which is the one thing this differs in from every +/// other width-aware helper in this crate. [`crate::picker::fit`] and +/// [`crate::status::Status::line`] shorten, because a picker row and a status line +/// each own exactly one row of a viewport that cannot grow. A committed surface +/// owns as many rows as it needs, so there is no reason left to lose a character — +/// and the characters most likely to be lost are the tail of a workspace path, the +/// tail of a policy layer's act list and the tail of a model id, which is to say +/// the answer. +/// +/// Broken at spaces, with a hanging indent that says a row is a continuation +/// without aligning anything into a column. A word longer than the room it has — +/// a deep path, an absurd model name — is **split** rather than allowed to +/// overflow: eighty columns is a supported size, and a row that runs past it gets +/// wrapped by the terminal at a place nothing here chose. +pub fn folded(text: &str, width: usize, first: usize, rest: usize) -> Vec { + let mut rows: Vec = Vec::new(); + let mut indent = first; + let mut row = " ".repeat(indent); + // Content characters on this row, which is its width less the indent. Counted + // rather than measured off `row` so the arithmetic is in one unit — the same + // reason `Status::fits` counts characters and not bytes. + let mut used = 0usize; + for word in text.split_whitespace() { + let mut word = word; + loop { + // At least one cell, so a terminal narrower than the indent still + // makes progress instead of looping forever. + let room = width.saturating_sub(indent).max(1); + let space = usize::from(used > 0); + let length = word.chars().count(); + if used + space + length <= room { + if space == 1 { + row.push(' '); + } + row.push_str(word); + used += space + length; + break; + } + if used > 0 { + rows.push(std::mem::take(&mut row)); + indent = rest; + row = " ".repeat(indent); + used = 0; + // Retried whole on the fresh row: a word is only ever split when + // a row of its own cannot hold it. + continue; + } + let head: String = word.chars().take(room).collect(); + word = &word[head.len()..]; + row.push_str(&head); + rows.push(std::mem::take(&mut row)); + indent = rest; + row = " ".repeat(indent); + } + } + if used > 0 || rows.is_empty() { + rows.push(row); + } + rows +} diff --git a/src/prices.rs b/src/prices.rs new file mode 100644 index 0000000..8693c1a --- /dev/null +++ b/src/prices.rs @@ -0,0 +1,615 @@ +//! Where a price comes from, what it claims, and when it was read. +//! +//! **io-cli compiles no prices in, and this module is the reason it does not have +//! to.** io-harness prices a call from a [`io_harness::pricing::PriceTable`] the operator's +//! configuration carries, and that table has to be filled from somewhere. Filling +//! it from a list baked into the binary would be a promise the binary cannot keep: +//! providers move prices without announcing it, a release cadence is not a pricing +//! cadence, and an operator reading a confident wrong number is worse off than one +//! reading no number at all. So the table is filled from the catalogue the +//! operator's own provider serves — which io-cli was already fetching and already +//! throwing away. +//! +//! # The call was already there +//! +//! [`crate::verify::catalogue`] has read the model catalogue since 0.1.0, to offer +//! the wizard a list of models. It mapped every [`ModelInfo`] down to its `id` and +//! discarded the `price`, `price_tiers` and `price_source` on the same row. This +//! module keeps the price and the source. No JSON is parsed here and no dependency +//! is added: io-harness did the parsing and the unit conversion, and +//! `ModelInfo::price` arrives already normalised into [`Price`]'s micro-units per +//! million tokens. +//! +//! **`price_tiers` is read and deliberately dropped, and the consequence is +//! stated rather than hidden.** io-harness's `[prices]` section carries `as_of` +//! and a map of `Price`, and is `deny_unknown_fields` — there is no TOML surface +//! for a tier, and `PriceTier` is constructible only in Rust. So a model whose +//! vendor charges more above a long-prompt threshold is priced here at its base +//! rate, and `/cost` cannot mark that as a floor because nothing in the store says +//! which calls crossed the line. It is an **understated** figure rather than an +//! invented one, and it is in the release record as a known limitation. +//! +//! # Whose price it is +//! +//! For two of the three vendors io-cli can connect to, the answer is not "the +//! vendor's". OpenAI and Anthropic publish no prices on any endpoint — their model +//! endpoints carry capabilities and limits and no cost field, and their cost APIs +//! report what was *spent* rather than what a token *costs*. The reference +//! catalogue does carry rows for their models, which is why +//! [`crate::verify::catalogue`] strips the `anthropic/` and `openai/` prefixes off +//! them to build its list. +//! +//! The fourth `ProviderSpec` is `Compatible`, and the reference catalogue cannot +//! speak for it at all: a list of one vendor's models says nothing about what a +//! server it has never heard of serves. That operator names their own catalogue +//! with `app.io-cli.prices.source_url`, which [`Catalogue::named`] reads without +//! narrowing. +//! +//! So an operator on Anthropic gets prices *about* Anthropic *from* the reference +//! catalogue, and every surface that draws money says which. io-harness models the +//! distinction already — [`PriceSource::Vendor`] against +//! [`PriceSource::Reference`] — and this module carries it through rather than +//! flattening it into the connected provider's name. On OpenRouter the two +//! coincide, which [`crate::verify::catalogue`] already says in as many words: the +//! reference catalogue is OpenRouter's own, so for OpenRouter it is not a +//! reference at all. +//! +//! # Two files, because one of them is not ours +//! +//! The table itself lives under `[prices]`, which io-harness owns and reads +//! through `Config::prices`. That section is `deny_unknown_fields` and carries +//! exactly `as_of` and `models` — so a key of io-cli's own put beside them would +//! not be ignored, it would make the operator's whole configuration file +//! unreadable. Anything this module needs to remember that io-harness does not +//! model therefore lives under `[app.io-cli.prices]`, which is the section +//! io-harness deliberately does not validate. +//! +//! # Nothing here reads a clock +//! +//! `tests/timing.rs` permits `SystemTime::now` in the driver and nowhere else, so +//! the date a table is stamped with is handed in rather than taken. [`date`] is +//! the conversion, and it is a pure function of a number for that reason as much +//! as for testability. + +use io_harness::pricing::Price; +use io_harness::{ModelInfo, PriceSource, ProviderSpec}; + +use crate::edit::Edit; + +/// The catalogue io-harness reads when nothing names another. +/// +/// Re-exported rather than re-spelled: a second copy of this URL in this +/// repository would be a second thing to move when io-harness moves it. +pub use io_harness::provider::catalog::DEFAULT_REFERENCE_URL; + +/// A catalogue read: what it priced, where it came from, and when. +/// +/// `rows` is already filtered to the models the connected provider actually +/// serves and spelled the way that provider names them — the same filter and the +/// same stripping [`crate::verify::catalogue`] applies, because the key a price is +/// stored under has to match the `model` io-harness records on a provider call, +/// and that is the name the operator configured. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Catalogue { + /// The date this was read, `YYYY-MM-DD`. Not a date any provider supplied: + /// every timestamp on offer anywhere is a model *release* date, and no + /// provider publishes when it last changed a price. + pub as_of: String, + /// Where the rows came from, in io-harness's own terms. + pub source: PriceSource, + /// One row per priced model. A model the catalogue served with no price is + /// **absent**, never entered at zero, so `Spend::unpriced_calls` counts it + /// and the page that draws it can say the total is a floor. + pub rows: Vec<(String, Price)>, + /// How many models the catalogue served in total, priced or not. Kept + /// separately from `rows.len()` because a catalogue that answered with plenty + /// of models and no prices is a different failure from one that answered with + /// nothing, and the operator is owed the difference. + pub served: usize, +} + +impl Catalogue { + /// The rows a catalogue read yields, filtered and named for `spec`. + /// + /// `models` is what the catalogue served; `as_of` is the date the caller + /// read it. Separated from the fetch so the whole of this decision is + /// testable without a socket — the fetch itself is one io-harness call and + /// has nothing in it worth a test double. + pub fn of(spec: &ProviderSpec, models: Vec, as_of: impl Into) -> Self { + Self::read(spec, models, as_of, None) + } + + /// The same, from a catalogue the operator named rather than the default. + /// + /// **A named catalogue is not narrowed, and that is the whole of what + /// `app.io-cli.prices.source_url` means.** The default catalogue is one + /// vendor's view of the entire field, so its rows have to be cut down to the + /// provider in force — and for a `compatible` endpoint that cut can only + /// remove everything, because a reference list cannot say what a server it has + /// never heard of serves. An operator who set `source_url` has answered that: + /// they pointed io-cli at the catalogue their own endpoint publishes, so every + /// row of it is theirs and the ids are already spelled the way their provider + /// names them. + /// + /// The source is recorded as that URL rather than as the vendor, because it is + /// still not the vendor speaking — it is a list the operator chose, and a page + /// that drew money under the provider's name would be attributing it wrongly. + pub fn named( + spec: &ProviderSpec, + models: Vec, + as_of: impl Into, + url: &str, + ) -> Self { + Self::read(spec, models, as_of, Some(url)) + } + + fn read( + spec: &ProviderSpec, + models: Vec, + as_of: impl Into, + url: Option<&str>, + ) -> Self { + if let Some(url) = url.filter(|url| !url.is_empty()) { + let served = models.len(); + let rows = crate::verify::priced(models); + return Self { + as_of: as_of.into(), + source: PriceSource::Reference(url.to_string()), + rows, + served, + }; + } + let vendor = matches!(spec, ProviderSpec::OpenRouter { .. }); + // **Counted after the filter, not before it.** The reference catalogue + // serves every model in the field; the number worth reporting is how many + // of them *this* provider serves, because that is the denominator the + // operator is being told their prices cover. Counting before the filter + // told an Anthropic operator their fifteen rates covered fifteen of four + // hundred and seventeen models, which is true of the catalogue and + // meaningless about them. + let mine = crate::verify::named(spec, models); + let served = mine.len(); + let rows = crate::verify::priced(mine); + Self { + as_of: as_of.into(), + // **The one place the two coincide.** The reference catalogue is + // OpenRouter's own, so for an OpenRouter operator it is the vendor + // speaking for itself and calling it a reference would understate it. + // For everyone else it is a third party and saying otherwise would + // attribute a number to a vendor that never published one. + source: if vendor { + PriceSource::Vendor + } else { + PriceSource::Reference(DEFAULT_REFERENCE_URL.to_string()) + }, + rows, + served, + } + } + + /// Whether this read is too short to replace `existing` rows. + /// + /// **The one failure in this release that loses money quietly.** A truncated + /// or partial catalogue response that replaced a full table with a handful of + /// rows would turn most of an operator's spending into "unpriced" and shrink + /// their reported bill without anything failing. So a replacement that comes + /// back empty, or far shorter than what it would replace, is refused and the + /// old table kept. + /// + /// A first fill has nothing to compare against and is never refused: an + /// operator with no table has nothing to lose. + pub fn too_short(&self, existing: usize) -> bool { + if existing == 0 { + return false; + } + self.rows.is_empty() || self.rows.len() * 2 < existing + } + + /// The edits that write this catalogue into a configuration file. + /// + /// A row per model rather than one enormous inline table, because a rate is + /// something the operator is invited to correct by hand and a row per line is + /// what makes correcting one a change they can find again. For OpenRouter that + /// is four hundred rows; as a single inline value it would be one + /// twenty-five-kilobyte line. + /// + /// **`has_section` decides the shape, and getting it wrong does not merely + /// produce a worse file — it produces no file.** Every edit in a batch is + /// resolved against the document as it was before the batch, so `set` on four + /// hundred keys of a `[prices.models]` that does not exist yet appends four + /// hundred `[prices.models]` headers and the read-back refuses the lot. A + /// first fill therefore writes the section whole, once; a refresh sets key by + /// key into the section that is already there. + /// + /// It is a parameter rather than something derived here because **it is a + /// question about the file, and this type has never seen the file.** The + /// obvious substitutes are both wrong: io-cli's own record of how many models + /// it last wrote is zero for a `[prices]` an operator wrote by hand, and + /// asking the `PriceTable` whether it prices any model this catalogue serves + /// answers `false` for a real section whose models the provider has since + /// replaced. [`has_models_section`] asks the file. + /// + /// That split has a second effect worth naming, because it is the behaviour + /// rather than an artefact: on a refresh, **rows for models the catalogue no + /// longer serves are left alone.** They are not stale. io-harness prices a + /// call by the model name recorded on it, so an old row is exactly what prices + /// an old run correctly, and `/cost` reports history as well as today. + pub fn edits(&self, has_section: bool) -> Vec { + let mut edits = vec![Edit::set("prices.as_of", quoted(&self.as_of))]; + if !has_section { + let body = self + .rows + .iter() + .map(|(model, price)| format!("{} = {}", quoted(model), inline(price))) + .collect::>() + .join("\n"); + edits.push(Edit::section("prices.models", body)); + return edits; + } + for (model, price) in &self.rows { + edits.push(Edit::set( + format!("prices.models.{}", quoted(model)), + inline(price), + )); + } + edits + } +} + +/// What a refresh would change about one model's rate. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Change { + /// The model, spelled as the price table keys it. + pub model: String, + /// What the table in force says now, or `None` for a model it does not price. + pub was: Option, + /// What the catalogue served. + pub now: Price, +} + +/// Every rate a refresh would move, against the table in force. +/// +/// **Shown before anything is written, which is why this is separate from +/// [`Catalogue::edits`].** io-cli cannot tell a rate the operator corrected by +/// hand from one an older catalogue served — the file records a number and not +/// where it came from — so it does not guess. It shows what would move and lets +/// the operator decline the lot, which is the shape `/import` established in +/// 0.21.0: everything shown before anything is written. +/// +/// A model the table does not price yet is a change with `was: None`. A rate that +/// has not moved is not a change and is not listed, so a refresh that found +/// nothing new says so in one line rather than in four hundred. +pub fn changes( + existing: Option<&io_harness::pricing::PriceTable>, + fresh: &Catalogue, +) -> Vec { + fresh + .rows + .iter() + .filter_map(|(model, now)| { + let was = existing.and_then(|table| table.price(model)); + (was != Some(*now)).then(|| Change { + model: model.clone(), + was, + now: *now, + }) + }) + .collect() +} + +/// How a file spells its price table, which decides how it can be written to. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Shape { + /// No `[prices.models]` anywhere. The section is created whole. + Absent, + /// `[prices.models]` with a row per model, which is what io-cli writes. Rows + /// are set key by key, so models nobody named survive. + Table, + /// A `[prices.models.""]` sub-table per model — legal TOML that + /// io-harness reads perfectly well, and that io-cli cannot safely rewrite. + SubTables, +} + +/// How `text` spells its price table. +/// +/// **The question [`Catalogue::edits`] needs answered, asked of the only thing +/// that can answer it**, and it has three answers rather than two. +/// +/// A `[prices]` carrying an `as_of` and no models is a real state — a refresh +/// whose catalogue priced nothing leaves exactly that — so the test is for the +/// child table and not the parent. +/// +/// [`Shape::SubTables`] is the one worth spelling out. `Price` deserializes from a +/// sub-table as happily as from an inline one, so an operator writing this section +/// by hand may well write `[prices.models."gpt-4.1"]` with the rates under it, and +/// io-harness will read it. io-cli cannot update it: setting a key named +/// `"gpt-4.1"` beside a table of the same name is a duplicate-key error, and +/// creating `[prices.models]` above the sub-tables collides with every id in both. +/// **So it is refused rather than attempted** — the operator owns this section, +/// and a writer that cannot express their shape should say so and leave it alone, +/// not produce a refusal from the TOML parser that names neither. +/// +/// Reads through [`crate::edit::sections`], the same header walk `edit::apply` +/// itself does, so the two cannot disagree about what is in the file. +/// Whether `text` carries a `[prices.models]` table io-cli can set keys into. +/// +/// The two-answer form of [`shape`], for [`Catalogue::edits`], which only needs to +/// know whether to create the section or write into it. [`Shape::SubTables`] +/// answers `false` here and must be refused by the caller *before* it gets this +/// far — see [`shape`] for why io-cli cannot write that spelling at all. +pub fn has_models_section(text: &str) -> bool { + matches!(shape(text), Shape::Table) +} + +pub fn shape(text: &str) -> Shape { + let mut found = Shape::Absent; + for path in crate::edit::sections(text) { + if path.len() < 2 || path[0] != "prices" || path[1] != "models" { + continue; + } + if path.len() > 2 { + // Any sub-table settles it: this file is not one io-cli can edit, and + // no later `[prices.models]` header makes it one. + return Shape::SubTables; + } + found = Shape::Table; + } + found +} + +/// Why io-cli will not write prices into `text`, if it will not. +/// +/// One sentence for the operator, or `None` to go ahead. The only refusal today is +/// [`Shape::SubTables`], and it is a refusal rather than an attempt because the +/// operator owns this section: a writer that cannot express their spelling should +/// say so and leave their file alone. +pub fn refusal(text: &str) -> Option { + match shape(text) { + Shape::SubTables => Some( + "no prices were written: this file spells its models as \ + `[prices.models.\"\"]` sub-tables, which io-harness reads and io-cli cannot \ + safely rewrite — the rates already there go on pricing your calls, and a refresh \ + means editing them by hand" + .to_string(), + ), + Shape::Absent | Shape::Table => None, + } +} + +/// How many models `text`'s own `[prices.models]` table prices. +/// +/// **What [`Catalogue::too_short`] should have been comparing against all along.** +/// It was handed `app.io-cli.prices.models`, io-cli's record of its own last write +/// — which is absent for a hand-written `[prices]`, absent for every install that +/// predates this release, and absent on the first fill. So `existing` was zero in +/// exactly the cases the guard exists for, and the guard was off. `PriceTable` has +/// no length, but the file has a row per model and can simply be counted. +pub fn priced_in(text: &str) -> usize { + let Ok(document) = text.parse::() else { + return 0; + }; + document + .get("prices") + .and_then(|prices| prices.get("models")) + .and_then(|models| models.as_table()) + .map_or(0, |models| models.len()) +} + +/// The `[app.io-cli.prices]` edits that record where a table came from. +/// +/// **The same defect as [`Catalogue::edits`]'s, one section over, and it is here +/// rather than at the call site because that is where it was.** The driver used to +/// push two bare `Edit::set`s for `source` and `models`. Neither section exists in +/// any file io-cli has ever written — `settings::render` writes `prices: None` and +/// the field is `skip_serializing_if` — so both fell to the append arm, each +/// emitted its own `[app.io-cli.prices]` header, and the read-back refused the +/// whole write. Every first fill would have ended in "the edit would have produced +/// a file that does not parse", including the one the wizard makes, and `/cost` +/// would have reported tokens forever. +/// +/// It survived the fix that added [`crate::edit::Edit::section`] because the two +/// edits it applies to are added by the driver, and nothing under `tests/` can +/// link the driver. Putting them here is what makes them testable. +pub fn bookkeeping(source: &str, models: usize, has_section: bool) -> Vec { + if has_section { + return vec![ + Edit::set("app.io-cli.prices.source", quoted(source)), + Edit::set("app.io-cli.prices.models", models.to_string()), + ]; + } + vec![Edit::section( + "app.io-cli.prices", + format!("source = {}\nmodels = {models}", quoted(source)), + )] +} + +/// What a refresh found, committed before anything is written. +/// +/// **Everything shown before a byte moves, and the reason is sharper than +/// courtesy.** io-cli cannot tell a rate the operator corrected by hand from one +/// an older catalogue served: the file records a number, not where it came from. +/// So it does not guess. It lists every rate that would move, with what it was +/// and what it would become, and an operator who finds their own correction in +/// that list can decline the whole refresh — which is the same answer `/import` +/// gives to the same problem, and the only honest one available. +/// +/// A refusal is reported here too rather than by the caller, so the sentence that +/// explains a short catalogue lives beside the rule that refuses it. +pub fn report( + catalogue: &Catalogue, + moved: &[Change], + existing: usize, + theme: &crate::theme::Theme, + width: u16, +) -> Vec> { + let mut rows: Vec = Vec::new(); + let source = source_word(&catalogue.source); + rows.push(crate::page::Row::note(format!( + "{source}, read {}: {} model{} served, {} priced", + catalogue.as_of, + catalogue.served, + if catalogue.served == 1 { "" } else { "s" }, + catalogue.rows.len(), + ))); + + if catalogue.too_short(existing) { + rows.push(crate::page::Row::caveat(format!( + "that is short enough against the {existing} model{} you already have to be a \ + truncated read rather than a price change, so nothing was written and the prices \ + you have were kept", + if existing == 1 { "" } else { "s" } + ))); + return crate::page::commit("prices", &rows, theme, width); + } + + // **A catalogue that answered with nothing is not a catalogue that agreed with + // you**, and this arm exists because the one below it used to swallow the + // case. `verify::served` returns an empty vector for a network failure, a + // refused key and an unparseable body alike, so with no rows there is nothing + // to compare and `moved` is empty — under which the next arm reported "no rate + // has moved since the last read", a positive claim about a read that did not + // happen. An operator on a flaky connection would have been told their prices + // were confirmed current. + if catalogue.rows.is_empty() { + rows.push(crate::page::Row::caveat(format!( + "the catalogue could not be read, or served no prices — {} model{} came back. \ + Nothing was written and the prices you have are unchanged, which means they are \ + as old as their date says and not as fresh as this attempt.", + catalogue.served, + if catalogue.served == 1 { "" } else { "s" } + ))); + return crate::page::commit("prices", &rows, theme, width); + } + if moved.is_empty() { + rows.push(crate::page::Row::note( + "no rate has moved since the last read, so nothing was written", + )); + return crate::page::commit("prices", &rows, theme, width); + } + + rows.push(crate::page::Row::Blank); + rows.push(crate::page::Row::heading(format!( + "{} rate{} would change", + moved.len(), + if moved.len() == 1 { "" } else { "s" } + ))); + for change in moved { + let value = match change.was { + Some(was) => format!("{} -> {}", rate(&was), rate(&change.now)), + // A model the table has never priced. Said as "new" rather than shown + // as a change from nothing, because an operator scanning for what + // moved is asking a different question from one asking what arrived. + None => format!("new, {}", rate(&change.now)), + }; + rows.push(crate::page::Row::fact(change.model.clone(), value)); + } + rows.push(crate::page::Row::Blank); + rows.push(crate::page::Row::caveat( + "if one of these is a rate you corrected by hand, it is about to be replaced: \ + io-cli records what a rate is and not where it came from, so it cannot tell yours \ + from an older catalogue's. Decline and edit the file if so.", + )); + crate::page::commit("prices", &rows, theme, width) +} + +/// One rate, as input and output per million tokens. +/// +/// The two dimensions that move a bill, and not the other three: a refresh report +/// is read to answer "did this get more expensive", and five numbers per row is a +/// table nobody scans. The whole rate is in the file for anyone who wants it. +fn rate(price: &Price) -> String { + format!( + "{} in / {} out", + crate::cost::money(price.input), + crate::cost::money(price.output) + ) +} + +/// A `Price` as a one-line TOML inline table, naming only what is charged for. +/// +/// A dimension the vendor does not charge for is left out rather than written as +/// a zero, because `Price` defaults every field and a file that spells three +/// zeros to say "nothing" is three lines an operator has to read to learn nothing. +/// `..Price::ZERO` is io-harness's own advice for constructing one, and this is +/// the same decision spelled in TOML. +fn inline(price: &Price) -> String { + let mut parts: Vec = Vec::new(); + for (name, value) in [ + ("input", price.input), + ("output", price.output), + ("cache_read", price.cache_read), + ("cache_write", price.cache_write), + ("per_server_tool_request", price.per_server_tool_request), + ] { + if value != 0 { + parts.push(format!("{name} = {value}")); + } + } + if parts.is_empty() { + // A model every dimension of which is free still has to round-trip as a + // `Price`, and `{}` is the inline table that means exactly that. + return "{}".to_string(); + } + format!("{{ {} }}", parts.join(", ")) +} + +/// A string as TOML source, escaped by the TOML crate rather than by hand. +/// +/// Model ids carry dots, slashes and colons, and a dot in a key can only be +/// spelled quoted — bare keys are letters, digits, `_` and `-` and nothing else. +/// So this is not decoration: an unquoted `gpt-4.1` is two path segments and +/// reaches the wrong place, or no place. `edit::array` sets the precedent of +/// spelling TOML through `toml` rather than with `format!`. +/// +/// Public because the driver writes two values of its own beside the table — the +/// source and the count — and `format!("\"{text}\"")` at a call site is a quoting +/// rule reimplemented, which is how the wrong one eventually gets written. +pub fn quoted(text: &str) -> String { + toml::Value::String(text.to_string()).to_string() +} + +/// `YYYY-MM-DD`, UTC, from a count of seconds since the Unix epoch. +/// +/// **A pure function of a number, and that is a gate rather than a preference.** +/// `tests/timing.rs` permits `SystemTime::now` in the driver and refuses it in +/// every other file under `src/`, so the driver reads the clock and hands the +/// number here. It also happens to be the only shape in which a date conversion +/// can be tested without owning the machine's clock. +/// +/// Howard Hinnant's `civil_from_days`, whose era begins on 0000-03-01 so that the +/// leap day falls at the end of a year and the month arithmetic needs no table. +/// Correct for every date the epoch can express; io-cli has no use for one before +/// 1970 and does not pretend to handle one. +/// +/// ``` +/// assert_eq!(io_cli::prices::date(0), "1970-01-01"); +/// assert_eq!(io_cli::prices::date(1_772_150_400), "2026-02-27"); +/// ``` +pub fn date(secs: u64) -> String { + let days = (secs / 86_400) as i64; + let z = days + 719_468; + let era = if z >= 0 { z } else { z - 146_096 } / 146_097; + let doe = z - era * 146_097; + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let year = yoe + era * 400; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let day = doy - (153 * mp + 2) / 5 + 1; + let month = if mp < 10 { mp + 3 } else { mp - 9 }; + let year = if month <= 2 { year + 1 } else { year }; + format!("{year:04}-{month:02}-{day:02}") +} + +/// How a price source reads on a surface that draws money. +/// +/// `PriceSource` is `#[non_exhaustive]`, so the `_` arm is required rather than +/// defensive — and it says "a catalogue" rather than guessing, because a variant +/// io-harness gains and this release has not seen is one whose name io-cli would +/// be inventing. +pub fn source_word(source: &PriceSource) -> String { + match source { + PriceSource::Vendor => "the provider's own catalogue".to_string(), + PriceSource::Reference(url) => format!("the reference catalogue at {url}"), + _ => "a catalogue this release does not recognise".to_string(), + } +} diff --git a/src/settings.rs b/src/settings.rs index 3dba157..b1530b4 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -126,6 +126,53 @@ pub struct CliSettings { /// child's whole life in it, which is what a detached child gives up. #[serde(default, skip_serializing_if = "Option::is_none")] pub detached_spawns: Option, + + /// What io-cli remembers about prices: `[app.io-cli.prices]`. + /// + /// **The table itself is not here, and that is not a filing decision.** The + /// prices live under `[prices]`, which io-harness owns and reads through + /// `Config::prices`. That section is `deny_unknown_fields` and carries exactly + /// `as_of` and `models`, so a key of io-cli's own put beside them would not be + /// ignored — it would make the operator's whole configuration file fail to + /// parse, taking the policy, the providers and the run budgets down with it. + /// `[app.io-cli]` is the one section io-harness deliberately does not + /// validate, so anything io-cli needs to remember that the harness does not + /// model belongs here and nowhere else. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub prices: Option, +} + +/// Where prices come from, and what the last read was. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct PriceSettings { + /// The catalogue to read: `source_url = "..."`. + /// + /// Absent means io-harness's own default. This key is the only way an + /// operator on a self-hosted or `compatible` endpoint gets prices at all: the + /// reference catalogue cannot speak for a server it has never heard of, and + /// `Reference::at` takes any URL serving the shape. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_url: Option, + /// What the last read was, in words, for the surfaces that draw money. + /// + /// Written by a fetch rather than by hand. It records whether the rates came + /// from the provider speaking for itself or from a third party's catalogue, + /// which for two of the three vendors io-cli can connect to is the second — + /// OpenAI and Anthropic publish no prices on any endpoint. A page that drew a + /// figure without saying which would be attributing a number to a vendor that + /// never published one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source: Option, + /// How many models the last read priced. + /// + /// **Bookkeeping io-harness does not model, which is what this section is + /// for.** `PriceTable` has no length and no iterator — it answers `price(model)` + /// and nothing else — so io-cli cannot count what it holds by asking it. The + /// count is needed twice: to say on `/cost` how many models the rates cover, + /// and to refuse a refetch that comes back far shorter than what it would + /// replace, which is the one failure in this area that loses money quietly. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub models: Option, } /// The caps this session runs its turns under, if any. @@ -447,6 +494,13 @@ pub fn render( max_parallel_reads: None, spawn_background_after_secs: None, detached_spawns: None, + // Left out because the wizard has not read a catalogue yet when + // it renders this. Prices arrive from the read that follows the + // credential check, as their own edits against the file this + // wrote — which is also what keeps the shape identical for an + // operator who runs the wizard and one who adds a provider from + // `/provider` later. + prices: None, }, }, }; diff --git a/src/stats.rs b/src/stats.rs new file mode 100644 index 0000000..26d5e3f --- /dev/null +++ b/src/stats.rs @@ -0,0 +1,255 @@ +//! How the work has gone: `/stats`. +//! +//! **The other half of the question `/cost` asks.** `/cost` says what the work +//! cost; this says whether it worked. They are the field's own two names and they +//! answer different questions, which is why every agent that has both keeps them +//! apart — and why `/usage` is an alias for `/cost` here rather than a third +//! screen, since `/usage` means plan and rate-limit status everywhere it exists +//! and this product has no plan to report. +//! +//! Every figure is a `Store` call io-cli has never made: `runs_by_outcome`, +//! `runs_by_day`, `first_try`, `gate_failures_by_phase`, `recovery` and +//! `store_size`, plus the latencies read off the provider calls themselves. +//! +//! # First-try is io-harness's definition, not one derived here +//! +//! It is tempting to count successful runs and call it a first-try rate. That +//! would count a run which failed three gates and then passed as a first-try +//! success. io-harness's `first_try` is finished **and** successful **and** +//! carrying no `gate_phase_failed` event, which is the question an operator is +//! actually asking, and it is a `Store` method precisely so nobody re-derives it +//! wrongly. +//! +//! `FirstTry` is three counts and deliberately not a rate, because the denominator +//! is a choice: first-try over every run counts runs that are still going, and +//! first-try over successful runs answers a different question again. This page +//! picks one and **names it in the row**, rather than printing a percentage whose +//! meaning a reader has to guess. +//! +//! # The two gate vocabularies stay apart +//! +//! io-harness records gate failures twice, in two tables, with two phase +//! vocabularies that do not overlap. `SandboxEvent` phases are `subject-compile`, +//! `criterion-compile`, `test-run` and `subject-emptied`; `GateAttempt` phases are +//! `review`, `command` and `contains`. They are different mechanisms and merging +//! them into one list would produce a chart whose categories mean two things. So +//! they get two headings. +//! +//! # Nothing here compacts anything +//! +//! `Store::compact` is a full `VACUUM`: it needs free disk roughly equal to the +//! database file, rewrites the whole thing, and is not quick. A page that reported +//! free space and then reclaimed it on its own would be a page that surprised +//! somebody mid-session. The free figure is reported and the reclaiming is not +//! offered here. + +use io_harness::Store; + +use crate::page::{self, Row}; +use crate::theme::Theme; + +/// How many of the most recent runs the latency sections read. +/// +/// **A bound this module can actually keep, unlike the one `/cost` could not.** +/// `Store::runs` returns run ids newest first, and io-cli takes the head of that +/// list itself, so the sample is genuinely the last `N` runs and the page says so +/// in the heading. That is the difference between a bound and a truncation: a +/// heading reading "slowest calls of the last 200 runs" is a true statement about +/// a subset, where "by model" over a truncated read would have been a false +/// statement about the whole. +/// +/// The same shape and the same reason as `crate::sessions::MAX_RUNS_SCANNED`. +pub const RECENT_RUNS: usize = 200; + +/// How many slow calls to name. +const SLOWEST: usize = 5; + +/// The `/stats` page. +pub fn committed( + store: &Store, + theme: &Theme, + width: u16, +) -> Result>, String> { + let mut rows: Vec = Vec::new(); + + rows.push(Row::heading("runs by outcome".to_string())); + rows.extend(tallies( + store.runs_by_outcome().map_err(|e| e.to_string())?, + "no runs recorded", + )); + + rows.push(Row::Blank); + rows.push(Row::heading("runs by day".to_string())); + rows.extend(tallies( + store.runs_by_day().map_err(|e| e.to_string())?, + "no runs recorded", + )); + + rows.push(Row::Blank); + rows.push(Row::heading("finished on the first try".to_string())); + let first = store.first_try().map_err(|e| e.to_string())?; + rows.push(Row::fact("runs finished", first.runs.to_string())); + rows.push(Row::fact( + "of those, succeeded", + first.succeeded.to_string(), + )); + rows.push(Row::fact( + "of those, with no gate failure", + first.first_try.to_string(), + )); + // **The denominator is named in the row, not implied by a percent sign.** + // io-harness gives three counts and refuses to pick one for exactly this + // reason; a bare `73%` would be a number whose meaning the reader has to + // reconstruct, and two readers would reconstruct it differently. + if let Some(share) = (first.first_try * 100).checked_div(first.succeeded) { + rows.push(Row::fact( + "first-try share of successful runs", + format!("{share}%"), + )); + } + + rows.push(Row::Blank); + rows.push(Row::heading("gate failures by phase".to_string())); + rows.extend(tallies( + store.gate_failures_by_phase().map_err(|e| e.to_string())?, + "no gate has failed", + )); + rows.push(Row::note( + "these are the sandbox gate's phases; the review, command and contains \ + gates record separately and are not merged into this list" + .to_string(), + )); + + rows.push(Row::Blank); + rows.push(Row::heading("recovery".to_string())); + let recovery = store.recovery().map_err(|e| e.to_string())?; + rows.push(Row::fact("fallbacks", recovery.fallbacks.to_string())); + rows.push(Row::fact("replans", recovery.replans.to_string())); + rows.push(Row::fact("resumes", recovery.resumes.to_string())); + + let ids = store.runs().map_err(|e| e.to_string())?; + let scanned = ids.len().min(RECENT_RUNS); + // **The head, because `Store::runs` is newest first.** Its own documentation + // says so — `ORDER BY id DESC` — and taking the tail would have read the + // *oldest* two hundred runs under a heading promising the most recent, which + // is a false statement about a subset rather than a true one. That is the + // whole difference this bound rests on. + let recent = &ids[..scanned]; + let mut calls = Vec::new(); + for id in recent { + calls.extend(store.provider_calls(*id).map_err(|e| e.to_string())?); + } + + rows.push(Row::Blank); + rows.push(Row::heading(format!( + "slowest calls, of the last {scanned} run{}", + if scanned == 1 { "" } else { "s" } + ))); + let mut slow: Vec<(u64, String)> = calls + .iter() + .map(|call| { + ( + call.latency_ms, + call.model.clone().unwrap_or_else(|| call.provider.clone()), + ) + }) + .collect(); + // Descending, so `take(SLOWEST)` below is the slowest and not the fastest. + slow.sort_by_key(|(latency, _)| std::cmp::Reverse(*latency)); + if slow.is_empty() { + rows.push(Row::note("no provider calls in those runs")); + } else { + for (ms, model) in slow.iter().take(SLOWEST) { + rows.push(Row::fact(model.clone(), format!("{ms} ms"))); + } + } + + rows.push(Row::Blank); + rows.push(Row::heading("time to first token".to_string())); + // **An unmeasured time to first token is `None`, never 0**, and it is counted + // separately rather than averaged in as a zero — which would drag every mean + // toward an instant nothing ever took. + let measured: Vec = calls.iter().filter_map(|call| call.ttft_ms).collect(); + if measured.is_empty() { + rows.push(Row::note( + "no call in those runs reported one, which is unmeasured rather than instant", + )); + } else { + let mut sorted = measured.clone(); + sorted.sort_unstable(); + rows.push(Row::fact( + "median", + format!("{} ms", sorted[sorted.len() / 2]), + )); + rows.push(Row::fact( + "slowest", + format!("{} ms", sorted[sorted.len() - 1]), + )); + rows.push(Row::fact( + "measured on", + format!("{} of {} calls", measured.len(), calls.len()), + )); + } + + rows.push(Row::Blank); + rows.push(Row::heading("what the store holds".to_string())); + let size = store.store_size().map_err(|e| e.to_string())?; + rows.push(Row::fact("file", bytes(size.file_bytes))); + rows.push(Row::fact("free inside it", bytes(size.free_bytes))); + rows.push(Row::fact("sessions", size.sessions.to_string())); + rows.push(Row::fact("runs", size.runs.to_string())); + // **Absent rather than zeroed.** io-harness returns an empty table list when + // the SQLite build lacks `dbstat`, which is a build that cannot answer the + // question — not a database whose tables are all empty. + if size.tables.is_empty() { + rows.push(Row::note( + "this SQLite build cannot break the file down by table", + )); + } else { + for (name, count) in size.tables.iter().take(5) { + rows.push(Row::fact(name.clone(), bytes(*count))); + } + } + if size.free_bytes > 0 { + rows.push(Row::note(format!( + "{} could be returned to the filesystem, which is a full rewrite of the \ + database and is not done from this page", + bytes(size.free_bytes) + ))); + } + + Ok(page::commit("stats", &rows, theme, width)) +} + +/// Rows for a set of io-harness's own `Tally` groups. +fn tallies(tallies: Vec, empty: &str) -> Vec { + if tallies.is_empty() { + return vec![Row::note(empty.to_string())]; + } + tallies + .into_iter() + .map(|tally| Row::fact(tally.key, tally.count.to_string())) + .collect() +} + +/// A byte count in the largest unit that leaves a whole number in front. +/// +/// Powers of 1024 and named as such: this is a file on a disk, and a reader +/// comparing it against what `ls` said should not have to convert. +fn bytes(count: u64) -> String { + const UNITS: [(&str, u64); 4] = [ + ("GiB", 1024 * 1024 * 1024), + ("MiB", 1024 * 1024), + ("KiB", 1024), + ("B", 1), + ]; + for (name, size) in UNITS { + if count >= size { + if size == 1 { + return format!("{count} B"); + } + return format!("{}.{} {name}", count / size, (count % size) * 10 / size); + } + } + "0 B".to_string() +} diff --git a/src/status.rs b/src/status.rs index e7dd019..82df576 100644 --- a/src/status.rs +++ b/src/status.rs @@ -341,10 +341,33 @@ pub struct Status { /// None`, and rendering that as `0` would report a full ceiling as an /// exhausted one. /// - /// Tokens, never money. `Containment::max_total_cost` is documented inert - /// because the crate has no price telemetry, so a figure with a currency in - /// front of it would be one this interface invented. + /// Tokens, never money — and the money is [`Status::cost`], one field down. + /// + /// This one is the tree's draw against the tree's ceiling, which io-harness + /// expresses in tokens and enforces in tokens. A currency figure here would be + /// a conversion of a ceiling nobody set in currency. + /// + /// **The sentence that stood here until 0.22.0 said something stronger and it + /// had stopped being true.** It read: a figure with a currency in front of it + /// would be one this interface invented, because the crate has no price + /// telemetry. The crate has had price telemetry since io-harness 0.18.0 — + /// `provider_calls` records the whole token split per call and `pricing` + /// derives money from it — and io-cli read none of it for four releases while + /// this comment explained why it could not. pub spend: Option<(u64, Option)>, + /// What this run has cost, in micro-units, or `None` for no answer. + /// + /// **Absent rather than zero, and there are three different ways to have no + /// answer** — no price table configured, a table that prices none of the + /// models this run used, or a run that has made no provider call yet. All + /// three draw nothing, because a `$0` on this line would say the run was free, + /// and none of the three means that. `/cost` is where the three are told + /// apart, because there is room there to say which. + /// + /// The run's own, not the session's, unlike [`Status::tokens`] beside it. What + /// an operator watches while a turn runs is what the turn is costing; the + /// session total is a keystroke away on `/cost`, which carries both. + pub cost: Option, /// How many background shells this run has started and not yet finished. /// /// A `shell_start` outlives the step that launched it, which is the whole @@ -473,6 +496,7 @@ impl Status { policy: None, tokens: None, run_tokens: None, + cost: None, context: None, containment: None, spend: None, @@ -525,6 +549,14 @@ impl Status { /// would leave the line contradicting prompts that are still going to run. pub fn forget_run(&mut self) { self.tokens = None; + // **The money goes with the tokens it was derived from.** `start_run` + // clears this too, and clearing it there alone is not enough: `/clear`, + // `/resume`, `/fork` and a rewind all reach here *without* starting a run, + // so a session cleared at an idle prompt would have gone on drawing the + // previous conversation's cost beside a blank token count — a figure with + // a currency in front of it attributed to a session that has spent + // nothing, which is the invented number this field exists not to print. + self.cost = None; // Both new in 0.11.0, and both run facts. The provider is the one that // reads as a session fact and is not: a resumed conversation may have // been served by another provider entirely, and `Started` sets it again @@ -561,6 +593,46 @@ impl Status { pub fn start_run(&mut self) { self.elapsed = Duration::ZERO; self.run_tokens = None; + // Back to "no answer" rather than to zero, and it is the same distinction + // `run_tokens` above is held to: a turn that has not called a provider yet + // has not cost zero, it has not cost anything anybody can report. + self.cost = None; + } + + /// Set [`Status::cost`] from what this run has actually called, priced by + /// `table`. + /// + /// **Read from the store rather than accumulated off the event stream, and + /// that is forced rather than chosen.** `EventKind::Step` carries a scalar + /// token count, and a price needs the split — fresh prompt against cache read + /// against cache write against completion — which lives only on the + /// `provider_calls` row. Counting the scalar at an input rate would over-report + /// every cached turn and every reasoning turn, which is to say most of them. + /// + /// The cost shape, stated: one indexed read of one run's calls, on the events + /// that can change the answer and no others — the same place and the same + /// cadence as [`Status::note_context_from`], which has read the store per step + /// since 0.16.0. + /// + /// Silent on every failure. A store that cannot be read is a status line with + /// no cost field, which is what an operator with no prices already sees; a + /// notice about a failed read of a decorative field would be worse than the + /// field's absence. + pub fn note_cost_from( + &mut self, + store: &io_harness::Store, + run_id: i64, + table: &io_harness::pricing::PriceTable, + ) { + let Ok(calls) = store.provider_calls(run_id) else { + return; + }; + let total = crate::cost::Total::of(&calls, table); + // **Nothing priced is not zero priced.** A run whose models are all + // outside the table has a real cost that this program cannot state, and + // stating it as `$0` would be the invented number the whole of `/cost` is + // built to avoid. + self.cost = (total.micros > 0).then_some(total.micros); } /// Set [`Status::context`] from an observation section of `est_tokens`. @@ -796,6 +868,22 @@ impl Status { (self.queued_prompts > 0).then(|| format!("queued {}", self.queued_prompts)) } + /// This run's cost, drawn, or nothing at all. + /// + /// **One method, reached by both renderers**, which is the shape + /// [`Status::budgets_left`] and [`Status::queued_left`] above already have and + /// for the reason written where they are: this file has shipped a field into + /// one renderer twice — 0.8.0's spend, 0.12.0's planning — and both times the + /// field was invisible on the row the binary actually draws. + /// + /// `None` covers all three ways of having no answer and does not distinguish + /// them, because this line has no room to. `/cost` is the surface that tells + /// an empty price table from an unpriced model from a run that has called + /// nothing, and it is one keystroke away. + pub fn cost_field(&self) -> Option { + self.cost.map(crate::cost::money) + } + /// The fields, most important first. pub fn fields(&self, theme: &Theme) -> Vec { // The WORD is the state, and the animation is only beside it. A spinner @@ -882,6 +970,32 @@ impl Status { }; fields.push(Field::new(text, tone)); } + // **Left of the containment word, which is where the design put it and + // where a live run proved it has to be.** Drafted to the right of it, the + // field never appeared: a real containment word is `workspace-write/ + // macos-sandbox-exec`, thirty-three characters, and at a hundred columns + // beside the model, the posture, the state, the clock and the token count + // there was nothing left — so the one field this release exists to fill + // was the first one dropped, on the first terminal it was run in. + // + // **Left of the numbers, because it is not one** — and as of 0.22.0 the + // code is finally where this sentence has always said it was. A standing + // mode that stops the agent writing outranks what the last turn spent: if + // a narrow terminal can hold one of them, it should hold the one that + // explains why nothing is happening. It was pushed *after* `steps`, `tok` + // and `ctx` for four releases, and since `fits` drops from the right that + // meant `line` gave up the planning phase before it gave up the token + // count — the exact inversion of the rule written directly above it. The + // footer had the same inversion by a different mechanism, dropping its + // whole right-hand group to keep every counter, and both are corrected in + // the release that adds another counter to that row. + // + // `Normal` rather than `Muted` for the same reason the background-job + // count is — it is a fact about what the agent may do, not a footnote + // about what it did. + if self.planning { + fields.push(Field::new("planning".to_string(), Tone::Normal)); + } // Beside the token count rather than anywhere else, because the two are // the same kind of fact — what this run has spent — and they used to sit // together in the `Finished` row this release removed. @@ -900,26 +1014,18 @@ impl Status { if let Some(context) = self.context { fields.push(Field::new(format!("ctx {context}%"), Tone::Muted)); } - // **Left of the containment word, which is where the design put it and - // where a live run proved it has to be.** Drafted to the right of it, the - // field never appeared: a real containment word is `workspace-write/ - // macos-sandbox-exec`, thirty-three characters, and at a hundred columns - // beside the model, the posture, the state, the clock and the token count - // there was nothing left — so the one field this release exists to fill - // was the first one dropped, on the first terminal it was run in. + // **Right of the token count it is derived from**, so the two read in the + // order they are computed in: the tokens are what happened and the money + // is what they came to. `Muted` like the counters beside it, because it is + // one — a fact about what the last turn spent, which is exactly the class + // this row's own priority rule says yields to a standing mode. // - // What it outranks is the honest ordering too: the containment word says - // how commands are sandboxed and does not change during a turn; this says - // what the fan-out is spending, and changes every step. - // **Left of the numbers, because it is not one.** A standing mode that - // stops the agent writing outranks what the last turn spent: if a narrow - // terminal can hold one of them, it should hold the one that explains why - // nothing is happening. `Normal` rather than `Muted` for the same reason - // the background-job count is — it is a fact about what the agent may do, - // not a footnote about what it did. - if self.planning { - fields.push(Field::new("planning".to_string(), Tone::Normal)); - } + // From the same method as the footer's, four lines of which are the whole + // lesson of 0.8.0's spend field and 0.12.0's planning field: a field + // rendered in one of the two renderers is a field the operator never sees, + // because the binary draws the footer on every terminal seven rows or + // taller. + fields.extend(self.cost_field().map(|text| Field::new(text, Tone::Muted))); // **Right of the counters they bound and left of the tree's own spend.** // A budget is read against the number beside it — `3 steps` and `left // 17/20 steps` are one fact split in two — so the two travel together and @@ -1075,6 +1181,12 @@ impl Status { if let Some(context) = self.context { counts.push(format!("ctx {context}%")); } + // Here as well as on `Status::line`, out of the same method, for the + // reason the budgets below are — this is the row the binary draws at an + // ordinary prompt, so a cost added to `line` alone would be a cost no + // operator ever saw. 0.12.0's planning field, again, and 0.8.0's spend + // field before it. + counts.extend(self.cost_field()); // **Here as well as on `Status::line`, from the same method, and that is // deliberate rather than tidy.** This is the row the binary draws at an // ordinary prompt — `Status::render` takes the footer on any terminal @@ -1110,14 +1222,21 @@ impl Status { // The keys that mean something at this exact moment, and only those. A // footer that listed every binding would be a help screen; `/help` is // the help screen. - counts.push( - if self.working { - "esc stops" - } else { - "/ for commands" - } - .to_string(), - ); + // + // **Held out of `counts` until after the narrowing below, because it is + // not a counter.** It used to be pushed here, which put it last — and the + // narrowing pops from the end, so the first thing a crowded row gave up + // was `esc stops`: the only place the footer tells an operator how to + // interrupt a turn, dropped exactly when the row is full because a turn is + // running. It is appended after the loop instead, and its width is counted + // during the loop so the arithmetic still describes the row that gets + // drawn. + let hint = if self.working { + "esc stops" + } else { + "/ for commands" + } + .to_string(); // What the agent is allowed to do, on the right, because that is the // question a reader asks of a footer when they ask anything of it. @@ -1149,6 +1268,35 @@ impl Status { } allowed.push(Span::styled("planning", muted)); } + // **When the two groups cannot both fit, the counters yield — not the + // group.** `row` fits its right-hand group all or nothing, so a counts + // row one character too wide took the posture, the containment word AND + // the planning phase off the screen together and kept every counter. + // That is the failure the comment directly above records, arriving from + // the other direction: 0.13.1's row was `4 steps · 14.2k tok · / for + // commands` and the group fit beside it at a hundred columns; `ctx N%` + // made the row ten characters wider, and in the 0.21.0 live capture the + // finished turn's row carries no right group at all. + // + // Which side gives is not a judgement call — this module states it + // where `planning` is ordered on `Status::fields`: a standing mode that + // stops the agent writing outranks what the last turn spent. So + // counters come off the right of `counts`, rightmost first, which is + // the order `fits` drops fields in and the order they are pushed in + // — least load-bearing last — until the group fits. The + // leftmost counter is never dropped: a row that had given up every + // number would be reporting the mode by erasing the session. + let wanted: usize = allowed.iter().map(|s| s.content.chars().count()).sum(); + // The hint is not in `counts` and is not droppable, so its width is added + // here rather than being carried by the join. + let hinted = hint.chars().count() + separator.chars().count(); + while !allowed.is_empty() + && !counts.is_empty() + && counts.join(separator).chars().count() + hinted + wanted >= room + { + counts.pop(); + } + counts.push(hint); let counted = row( vec![Span::styled(counts.join(separator), muted)], allowed, @@ -1275,6 +1423,11 @@ fn fits(fields: &[Field], width: usize, theme: &Theme) -> usize { /// reader can find elsewhere: the posture is on the wizard's own screen and in /// the configuration, and the clock is beside the word `working` on the row that /// says a turn is running at all. +/// +/// **That last-resort rule is not the counts row's policy**, and a caller who +/// has a droppable left group is expected to narrow it before calling: the +/// counts row trims counters off its own right until the group fits, because +/// the group it would otherwise lose carries the planning phase. fn row(left: Vec>, right: Vec>, width: usize) -> Line<'static> { let measure = |spans: &[Span<'static>]| -> usize { spans.iter().map(|span| span.content.chars().count()).sum() @@ -1658,62 +1811,10 @@ pub fn committed( /// `text` as rows no wider than `width`, indented `first` and then `rest`. /// -/// **Folded and never fitted**, which is the one thing this differs in from every -/// other width-aware helper in this crate. `crate::picker::fit` and -/// [`Status::line`] shorten, because a picker row and a status line each own -/// exactly one row of a viewport that cannot grow. A committed surface owns as -/// many rows as it needs, so there is no reason left to lose a character — and -/// the characters most likely to be lost here are the tail of a workspace path -/// and the tail of a policy layer's act list, which is to say the answer. -/// -/// Broken at spaces, with a hanging indent that says a row is a continuation -/// without aligning anything into a column. A word longer than the room it has — -/// a deep path, an absurd model name — is **split** rather than allowed to -/// overflow: eighty columns is a supported size, and a row that runs past it gets -/// wrapped by the terminal at a place nothing here chose. -fn folded(text: &str, width: usize, first: usize, rest: usize) -> Vec { - let mut rows: Vec = Vec::new(); - let mut indent = first; - let mut row = " ".repeat(indent); - // Content characters on this row, which is its width less the indent. Counted - // rather than measured off `row` so the arithmetic is in one unit — the same - // reason `fits` counts characters and not bytes. - let mut used = 0usize; - for word in text.split_whitespace() { - let mut word = word; - loop { - // At least one cell, so a terminal narrower than the indent still - // makes progress instead of looping forever. - let room = width.saturating_sub(indent).max(1); - let space = usize::from(used > 0); - let length = word.chars().count(); - if used + space + length <= room { - if space == 1 { - row.push(' '); - } - row.push_str(word); - used += space + length; - break; - } - if used > 0 { - rows.push(std::mem::take(&mut row)); - indent = rest; - row = " ".repeat(indent); - used = 0; - // Retried whole on the fresh row: a word is only ever split when - // a row of its own cannot hold it. - continue; - } - let head: String = word.chars().take(room).collect(); - word = &word[head.len()..]; - row.push_str(&head); - rows.push(std::mem::take(&mut row)); - indent = rest; - row = " ".repeat(indent); - } - } - if used > 0 || rows.is_empty() { - rows.push(row); - } - rows -} +/// **The body moved to [`crate::page::folded`] in 0.22.0 and this is the name it +/// answered to here.** `crate::context` carried the same twenty lines under the +/// name `wrapped`, differing only in hard-coding the two indents this signature +/// takes as arguments, and `/cost` and `/stats` would have been the third and +/// fourth copies. The argument for folding rather than fitting is written out +/// where the code now lives. +use crate::page::folded; diff --git a/src/verify.rs b/src/verify.rs index c4ef093..02f4123 100644 --- a/src/verify.rs +++ b/src/verify.rs @@ -69,35 +69,109 @@ pub async fn credential(spec: &ProviderSpec) -> Result<(), String> { } } -/// The provider's model catalogue, as ids. +/// A catalogue read, whole rows and not just ids, and **unfiltered**. /// -/// io-harness reads and prices catalogues already; this filters the reference -/// catalogue down to the models the chosen provider actually serves. An error is -/// not fatal — the wizard offers the provider's usual default instead, because a -/// catalogue that cannot be read is a reason to make the user type a model, not a -/// reason to stop. -pub async fn catalogue(spec: &ProviderSpec) -> Vec { - let Ok(models) = Reference::new().models().await else { - return Vec::new(); +/// **The rows carry prices, and until 0.22.0 this read threw them away.** It read +/// the same catalogue, mapped every [`io_harness::ModelInfo`] down to its `id`, +/// and dropped the `price`, `price_tiers` and `price_source` on the same row — +/// while the interface over it reported token counts and called the money question +/// unanswerable. [`crate::prices`] is what keeps them. +/// +/// `source` names the catalogue to read, for the operator on a self-hosted or +/// `compatible` endpoint the reference catalogue has never heard of. `None` is +/// io-harness's own default. +/// +/// **Narrowing happens at the caller and not here, and that is the whole of what +/// makes `source_url` mean anything.** [`named`] narrows the default catalogue — +/// one vendor's view of the entire field — down to the provider in force, and for +/// a `compatible` endpoint it can only answer "none of these", because a reference +/// list cannot say what a server it has never heard of serves. An operator who +/// sets `source_url` has already answered that question: they have pointed io-cli +/// at the catalogue their own endpoint publishes, and every row of it is theirs. +/// Narrowing here would have applied the wrong filter to the right catalogue, +/// which is what it did — the key was inert, over a test asserting the empty +/// result, while `src/settings.rs` and the README both called it the only way a +/// self-hosted operator gets prices at all. +/// +/// An error is not fatal and comes back as an empty vector: a catalogue that +/// cannot be read is a reason to make the user type a model, not a reason to stop. +/// A caller that needs to tell "nothing was served" from "nothing was priced" has +/// [`crate::prices::Catalogue::served`] for it. +pub async fn served(source: Option<&str>) -> Vec { + let reference = match source { + Some(url) if !url.is_empty() => Reference::at(url), + _ => Reference::new(), }; - let mut ids: Vec = match spec { + reference.models().await.unwrap_or_default() +} + +/// The catalogue filtered to what `spec` serves, spelled the way `spec` names it. +/// +/// Separated from the fetch so the whole of this decision is testable without a +/// socket. The **spelling matters beyond the wizard's list**: the id here is the +/// key a price is stored under, and it has to match the `model` io-harness records +/// on a provider call — which is the name the operator configured, not the +/// catalogue's namespaced one. +pub fn named( + spec: &ProviderSpec, + models: Vec, +) -> Vec { + match spec { // The reference catalogue is OpenRouter's own, so for OpenRouter it is not // a reference at all — it is the provider speaking for itself. - ProviderSpec::OpenRouter { .. } => models.into_iter().map(|model| model.id).collect(), + ProviderSpec::OpenRouter { .. } => models, ProviderSpec::Anthropic { .. } => strip(models, "anthropic/"), ProviderSpec::OpenAi { .. } => strip(models, "openai/"), // Anything else serves whatever it serves and the reference cannot say. _ => Vec::new(), - }; + } +} + +/// The models this provider serves, as ids, sorted and deduplicated. +/// +/// What the wizard puts in front of the user. Unchanged in behaviour since 0.1.0; +/// it is now one `map` over [`served`] rather than its own read. +pub async fn catalogue(spec: &ProviderSpec) -> Vec { + let mut ids: Vec = named(spec, served(None).await) + .into_iter() + .map(|model| model.id) + .collect(); ids.sort(); ids.dedup(); ids } -fn strip(models: Vec, prefix: &str) -> Vec { +/// The priced rows of a catalogue, sorted by model. +/// +/// **Takes rows [`named`] has already spelled, and does not filter again.** +/// Filtering is not idempotent — `named` strips a provider's prefix, and running +/// it twice on an Anthropic catalogue finds no `anthropic/` left to strip and +/// discards every row. So the caller filters once and hands the result both here +/// and to whatever else needs a count of it. +/// +/// **A model served with no price is absent, never entered at zero.** +/// io-harness's `PriceTable::price` returning `None` is what makes +/// `Spend::unpriced_calls` count that call, which is what lets `/cost` say its +/// total is a floor rather than reporting a partial sum as a total. A zero here +/// would silently claim the model is free. +pub fn priced(models: Vec) -> Vec<(String, io_harness::pricing::Price)> { + let mut rows: Vec<(String, io_harness::pricing::Price)> = models + .into_iter() + .filter_map(|model| model.price.map(|price| (model.id, price))) + .collect(); + rows.sort_by(|a, b| a.0.cmp(&b.0)); + rows.dedup_by(|a, b| a.0 == b.0); + rows +} + +fn strip(models: Vec, prefix: &str) -> Vec { models .into_iter() - .filter_map(|model| model.id.strip_prefix(prefix).map(str::to_string)) + .filter_map(|mut model| { + let id = model.id.strip_prefix(prefix)?.to_string(); + model.id = id; + Some(model) + }) .collect() } diff --git a/tests/commands.rs b/tests/commands.rs index cf693d4..1682024 100644 --- a/tests/commands.rs +++ b/tests/commands.rs @@ -82,6 +82,13 @@ fn the_commands_are_the_commands() { "/fleet", "/image", "/clear", + // 0.22.0 — the two halves of one question, and two commands because + // they are two questions. `/cost` says what the work cost and + // `/stats` says whether it worked; every agent that has both keeps + // them apart. `/usage` is an alias for the first and is absent from + // this list, which is the rule every alias here follows. + "/cost", + "/stats", ], "the fuzzy palette is still 0.7.0; this list is written out so that adding \ a command is a decision somebody makes rather than a line somebody adds", @@ -1073,8 +1080,17 @@ fn the_memory_page_separates_the_two_memories_and_qualifies_a_cut_draw_count() { fn f17_usage_answers_and_is_never_listed() { use io_cli::commands::{group_of, palette, GROUPS}; - // It answers, with what `/status` answers. - assert_eq!(commands::parse("usage", &defaults(), &DARK), Action::Status); + // **It answers `/cost`, and until 0.22.0 it answered `/status`.** The + // argument for that was that `/status` was the closest thing this program had + // to a spending surface — it commits the token draw, the budgets and what is + // left of them. It stopped holding the moment a page existed whose whole + // subject is what has been spent: an operator typing the field's own word for + // that page and landing on the session's configuration is being answered a + // question they did not ask. + assert_eq!(commands::parse("usage", &defaults(), &DARK), Action::Cost); + assert_eq!(commands::parse("cost", &defaults(), &DARK), Action::Cost); + // And `/status` still answers itself, which is the half of this that did not + // change and the half a careless repoint would have taken with it. assert_eq!( commands::parse("status", &defaults(), &DARK), Action::Status diff --git a/tests/cost.rs b/tests/cost.rs new file mode 100644 index 0000000..20d79e2 --- /dev/null +++ b/tests/cost.rs @@ -0,0 +1,869 @@ +//! What has been spent, and the three ways of not knowing it. +//! +//! **Every number on `/cost` is a row already in `runs.db` or that row multiplied +//! by a rate the operator can point at.** There is no sampling, no extrapolation +//! and no model of what a token "usually" costs — which means the interesting +//! failures in this module are not arithmetic failures at all. They are the three +//! ways a page can report a number it does not have: a call the provider said +//! nothing about summed as zero, a model outside the price table priced at +//! nothing, and a total containing either of those drawn as though it were +//! complete. io-harness's own pricing documentation calls the last one "lying by +//! omission", and it is right. +//! +//! So the arithmetic is asserted once, against a figure this file computes from +//! the rates and the token split rather than against a literal — a wrong constant +//! baked into both sides of an `assert_eq!` proves nothing at all — and the rest +//! of the file is about what the page says when it cannot answer. +//! +//! The store is real throughout. `ProviderCall` rows go in through +//! `Store::record_provider_call` and come back out through the same reads the +//! page makes, because `usage: None` survives the round trip as SQL `NULL` and +//! that column is the whole of the unknown/free distinction. + +use io_harness::pricing::{Price, PriceTable}; +use io_harness::{ProviderCall, Store, Usage}; + +use io_cli::cost::{self, Provenance, Total}; +use io_cli::glyphs::ASCII; +use io_cli::theme::{Theme, DARK}; + +// --------------------------------------------------------------------------- +// Fixtures +// --------------------------------------------------------------------------- + +/// Wide enough that nothing folds, so a `contains` here fails for the reason it +/// names rather than because a row broke across two. +const ROOMY: u16 = 200; + +/// The rates. Five dimensions, all different, and none of them a round number of +/// dollars — a table where every rate was the same could not tell an +/// implementation that priced completion at the input rate from one that did not. +const INPUT: u64 = 3_000_000; +const OUTPUT: u64 = 15_000_000; +const CACHE_READ: u64 = 300_000; +const CACHE_WRITE: u64 = 3_750_000; +const PER_REQUEST: u64 = 10_000; + +/// The model the table prices, and the one it does not. +const PRICED: &str = "claude-sonnet-4.5"; +const UNPRICED: &str = "some-lab/experimental-preview"; + +fn table() -> PriceTable { + PriceTable::new("2026-08-27").with( + PRICED, + Price { + input: INPUT, + output: OUTPUT, + cache_read: CACHE_READ, + cache_write: CACHE_WRITE, + per_server_tool_request: PER_REQUEST, + }, + ) +} + +/// The token split every arithmetic assertion below runs over. +/// +/// **The cache figures are inside the prompt, not beside it.** Ten thousand +/// prompt tokens of which four thousand were served from cache and one thousand +/// written into it leaves five thousand read fresh — and a page that added the +/// three would report fifteen thousand prompt tokens for a turn that had ten. +fn split() -> Usage { + Usage { + prompt_tokens: 10_000, + completion_tokens: 2_000, + total_tokens: 12_000, + cache_read_tokens: 4_000, + cache_write_tokens: 1_000, + reasoning_tokens: 500, + server_tool_requests: 3, + } +} + +fn call(model: Option<&str>, usage: Option) -> ProviderCall { + ProviderCall { + step: 1, + attempt: 0, + provider: "anthropic".into(), + model: model.map(str::to_string), + usage, + latency_ms: 1_200, + ttft_ms: Some(300), + // `..Default::default()` rather than every field, because io-harness adds + // one to this struct when a vendor starts reporting something new and a + // test that spelled all nine would stop compiling on a harness bump for a + // reason that has nothing to do with what it asserts. + ..Default::default() + } +} + +/// A store with one run inside one session, and the calls recorded against it. +struct Seeded { + store: Store, + run: i64, + session: i64, +} + +fn seeded(calls: &[ProviderCall]) -> Seeded { + let store = Store::memory().expect("an in-memory store"); + let run = store + .start_run("summarise the module", "/repo") + .expect("a run"); + let session = store.create_session("/repo").expect("a session"); + store + .record_turn(session, None, run, "summarise the module") + .expect("a turn"); + for call in calls { + store + .record_provider_call(run, call) + .expect("the call is recorded"); + } + Seeded { + store, + run, + session, + } +} + +fn ascii() -> Theme { + DARK.with_glyphs(ASCII) +} + +/// The `/cost` page as a reader sees it: every row, spans concatenated. +fn page(seeded: &Seeded, table: &PriceTable, provenance: &Provenance) -> Vec { + cost::committed( + &seeded.store, + table, + provenance, + Some(seeded.run), + Some(seeded.session), + &ascii(), + ROOMY, + ) + .expect("the page draws") + .iter() + .map(|line| { + line.spans + .iter() + .map(|span| span.content.as_ref()) + .collect::() + }) + .collect() +} + +/// The whole page as one string, for asking whether a word is anywhere on it. +fn text(rows: &[String]) -> String { + rows.join("\n") +} + +/// The value of the first row reading `label: …`, or a failure naming the page. +fn field<'a>(rows: &'a [String], label: &str) -> &'a str { + let prefix = format!("{label}: "); + rows.iter() + .find_map(|row| row.trim_start().strip_prefix(&prefix)) + .unwrap_or_else(|| panic!("no `{label}` row on the page:\n{}", text(rows))) +} + +/// The provenance a configured operator has. +fn provenance() -> Provenance { + Provenance { + as_of: Some("2026-08-27".into()), + source: Some("the reference catalogue at https://openrouter.ai/api/v1/models".into()), + models: Some(417), + } +} + +// --------------------------------------------------------------------------- +// The arithmetic +// --------------------------------------------------------------------------- + +/// **The total is the rates multiplied by the split, and the expected figure is +/// computed here rather than copied out of a run.** +/// +/// A test that asserted `total.micros == 79_950` would go green on the day it was +/// written and would keep going green if somebody changed a rate constant in the +/// fixture — the two sides would move together and the assertion would be that +/// the function is the function. So the expected value is built from the same +/// five rates and the same seven token counts an operator would multiply by hand, +/// and the fixture is chosen so the division is exact: io-harness's rounding rule +/// is io-harness's to test, and re-implementing it here would smuggle it into an +/// assertion about io-cli's summing. +/// +/// The cross-check against `PriceTable::cost_micros` is what stops this file +/// inventing its own pricing model. If the arithmetic here and the harness's +/// disagree, this test says so before anything else does. +/// +/// Sabotage: sum `usage.prompt_tokens` at the input rate rather than +/// `prompt_tokens - cache_read - cache_write`, which is the natural mistake and +/// over-reports every cached turn — which is most of them. Under it this fails by +/// exactly the cache tokens times the input rate and no rendered page changes +/// shape. +#[test] +fn the_total_is_the_rates_times_the_split_and_not_a_figure_typed_here() { + let usage = split(); + let fresh = usage.prompt_tokens - usage.cache_read_tokens - usage.cache_write_tokens; + assert_eq!(fresh, 5_000, "the fixture's fresh prompt tokens"); + + // Per-million everywhere except the server tool requests, which vendors bill + // per request and which a token-only sum silently loses. + let mtok = fresh as u128 * INPUT as u128 + + usage.completion_tokens as u128 * OUTPUT as u128 + + usage.cache_read_tokens as u128 * CACHE_READ as u128 + + usage.cache_write_tokens as u128 * CACHE_WRITE as u128; + assert_eq!( + mtok % 1_000_000, + 0, + "the fixture is chosen so that no rounding rule is under test here", + ); + let expected = + (mtok / 1_000_000 + usage.server_tool_requests as u128 * PER_REQUEST as u128) as u64; + assert!(expected > 0, "a fixture that costs nothing proves nothing"); + + // io-harness agrees about what one call costs, so what follows is a claim + // about io-cli's summing rather than about pricing. + assert_eq!( + table().cost_micros(PRICED, &usage), + Some(expected), + "this file and io-harness disagree about what one call costs", + ); + + let total = Total::of(&[call(Some(PRICED), Some(usage))], &table()); + assert_eq!(total.calls, 1); + assert_eq!(total.unknown, 0); + assert_eq!(total.unpriced, 0); + assert_eq!(total.micros, expected); + assert!( + !total.is_floor(), + "a total with nothing missing from it was reported as a floor", + ); + + // **Reasoning is inside completion, never beside it.** A vendor that reports + // reasoning separately still bills it as output, so it is carried for the + // page to break down and contributes nothing of its own to the money. + let mut without_reasoning = usage; + without_reasoning.reasoning_tokens = 0; + assert_eq!( + Total::of(&[call(Some(PRICED), Some(without_reasoning))], &table()).micros, + expected, + "reasoning tokens were billed a second time on top of completion", + ); + + // And the sum over calls is a sum. Three of the same call cost three times as + // much, and the token counters add rather than replace. + let three = Total::of( + &[ + call(Some(PRICED), Some(usage)), + call(Some(PRICED), Some(usage)), + call(Some(PRICED), Some(usage)), + ], + &table(), + ); + assert_eq!(three.micros, expected * 3); + assert_eq!(three.usage.prompt_tokens, usage.prompt_tokens * 3); + assert_eq!(three.usage.cache_read_tokens, usage.cache_read_tokens * 3); + assert_eq!( + three.usage.server_tool_requests, + usage.server_tool_requests * 3 + ); +} + +/// **A call the provider said nothing about is unknown, and unknown is not +/// free.** +/// +/// io-harness stores `total_tokens` as SQL `NULL` for a call whose provider +/// reported no usage and reads it back as `None` for exactly that reason. Summing +/// it as a zeroed `Usage` would report a turn that cost something as a turn that +/// cost nothing — and it would do it silently, because a zero adds correctly to +/// every other figure on the page. +/// +/// Three things therefore have to hold at once: the call is counted (it happened, +/// and it was billed by somebody), it contributes no tokens (there are none to +/// contribute), and the total is marked a floor (there is a cost nobody can +/// state). A page that got the first two right and the last one wrong would draw +/// a complete-looking total that is missing a turn. +/// +/// Sabotage: `let usage = call.usage.unwrap_or_default();` — one method call, +/// reads as a tidy-up of an `Option`, and turns every unreported call into a free +/// one. Under it `unknown` stays zero, `is_floor` goes false, and the caveat +/// leaves the page. +#[test] +fn a_call_the_provider_said_nothing_about_is_unknown_and_never_free() { + let calls = [ + call(Some(PRICED), Some(split())), + call(Some(PRICED), None), + call(None, None), + ]; + let total = Total::of(&calls, &table()); + + assert_eq!( + total.calls, 3, + "a call that reported nothing still happened" + ); + assert_eq!(total.unknown, 2); + assert_eq!( + total.unpriced, 0, + "an unreported call is unknown, which is a different gap from unpriced — \ + counting it as both would report one missing call twice", + ); + assert_eq!( + total.usage, + split(), + "the unreported calls contributed tokens they never reported", + ); + assert!( + total.is_floor(), + "two calls are missing from this total and it does not say so", + ); + + // And the page says it in words, in the vocabulary that distinguishes the two + // gaps. `unknown rather than free` is the sentence; a page that said "0 + // tokens" would be stating the thing that is not true. + let fixture = seeded(&calls); + let rows = page(&fixture, &table(), &provenance()); + let drawn = text(&rows); + assert!( + drawn.contains("reported no usage at all"), + "the page hides the calls it could not read:\n{drawn}", + ); + assert!( + drawn.contains("unknown rather than free"), + "the page does not distinguish an unreported call from a free one:\n{drawn}", + ); +} + +/// **A model outside the table is unpriced, and a total containing one is a floor +/// rather than a total.** +/// +/// The gap the whole module is shaped around. io-cli compiles no prices in, so an +/// operator's table covers whatever their catalogue happened to price on the day +/// they read it — a model added upstream last week is a real cost with no rate, +/// and it is the ordinary case rather than the exotic one. +/// +/// The tokens are still exact. That is the point of counting the two gaps +/// separately: an unpriced call contributes every token it reported and no money, +/// so the token figures on the page are complete and the money figure is a floor, +/// and the page has to be able to say both at once. +/// +/// Sabotage: price a missing model at `Price::ZERO` rather than counting it — +/// under which `unpriced` stays zero, `is_floor` goes false, the caveat leaves the +/// page, and the operator reads a total that is confidently short. +#[test] +fn a_model_the_table_does_not_price_is_a_floor_and_says_so() { + let calls = [ + call(Some(PRICED), Some(split())), + call(Some(UNPRICED), Some(split())), + // No model at all, which the provider is allowed not to name. It cannot + // be priced either, and for the same reason it cannot be attributed. + call(None, Some(split())), + ]; + let total = Total::of(&calls, &table()); + + assert_eq!(total.calls, 3); + assert_eq!(total.unknown, 0, "all three reported their usage"); + assert_eq!(total.unpriced, 2, "two of the three cannot be priced"); + assert!( + total.is_floor(), + "a total missing two calls' money is a floor" + ); + assert_eq!( + total.usage.prompt_tokens, + split().prompt_tokens * 3, + "an unpriced call still reported its tokens, and they are still exact", + ); + assert_eq!( + total.micros, + table() + .cost_micros(PRICED, &split()) + .expect("the priced one"), + "the money is exactly the one call the table can price", + ); + + let fixture = seeded(&calls); + let rows = page(&fixture, &table(), &provenance()); + let drawn = text(&rows); + assert!( + drawn.contains("no rate in the price table"), + "the page does not say why the total is short:\n{drawn}", + ); + assert!( + drawn.contains("floor and not a total"), + "the page draws an incomplete total as though it were complete:\n{drawn}", + ); + // **On the grouped rows too, and on the row rather than under it.** A reader + // scanning `by model` for the largest figure has to be able to see which of + // them is incomplete without reading past the list. + // + // "not priced" and not "unpriced", deliberately: io-harness's + // `Spend::unpriced_calls` counts a call with no usage as unpriced, while this + // module's `Total` splits those out as *unknown*. The same call falls in + // different buckets on the two halves of this page, and one word for both + // would make the two sections look like they disagreed about a number. + assert!( + drawn.contains("not priced"), + "the `by model` rows hide the calls they could not price:\n{drawn}", + ); + assert!( + rows.iter() + .any(|row| row.contains(UNPRICED) && row.contains("not priced")), + "the unpriced model's own row does not say it could not be priced:\n{drawn}", + ); +} + +/// **The cache figures are a breakdown of the prompt, and the three parts sum to +/// it rather than past it.** +/// +/// `Usage::cache_read_tokens` and `cache_write_tokens` are already inside +/// `prompt_tokens`, so a page that listed them as separate figures would +/// over-report every cached turn — and on a long agentic run almost every turn is +/// cached. The page draws the prompt total and the three parts beneath it, named +/// as parts (`of which cache read`) rather than drawn further right, because this +/// page has no columns to indent into and a reader in `--plain` has only the +/// words. +/// +/// Asserted on the underlying counts as well as on the rendered strings: the +/// rendered figures are rounded to one decimal by `format_tokens`, so agreement +/// between them is necessary and not sufficient. +/// +/// Sabotage: add the two cache counters to the prompt figure — under which +/// `prompt` reads 15.0k for a turn that had 10.0k of prompt, the three parts sum +/// past it, and every other test in this file still passes. +#[test] +fn the_cache_figures_are_inside_the_prompt_and_never_added_to_it() { + let usage = split(); + let fresh = usage.prompt_tokens - usage.cache_read_tokens - usage.cache_write_tokens; + assert_eq!( + usage.cache_read_tokens + usage.cache_write_tokens + fresh, + usage.prompt_tokens, + "the fixture's own parts do not sum to its prompt", + ); + + let fixture = seeded(&[call(Some(PRICED), Some(usage))]); + let rows = page(&fixture, &table(), &provenance()); + + let spelled = io_cli::status::format_tokens; + assert_eq!( + field(&rows, "prompt"), + spelled(usage.prompt_tokens), + "the prompt figure is not the prompt the provider reported:\n{}", + text(&rows), + ); + assert_eq!( + field(&rows, "of which cache read"), + spelled(usage.cache_read_tokens), + ); + assert_eq!( + field(&rows, "of which cache written"), + spelled(usage.cache_write_tokens), + ); + assert_eq!( + field(&rows, "of which fresh"), + spelled(fresh), + "the fresh figure is not the prompt less what came from cache", + ); + assert_eq!(field(&rows, "completion"), spelled(usage.completion_tokens)); + assert_eq!( + field(&rows, "of which reasoning"), + spelled(usage.reasoning_tokens), + "reasoning is a breakdown of completion and is drawn as one", + ); + + // A turn with no cache at all draws no cache rows, rather than three zeroes + // an operator has to read to learn nothing. + let plain = Usage { + prompt_tokens: 900, + completion_tokens: 100, + total_tokens: 1_000, + ..Default::default() + }; + let fixture = seeded(&[call(Some(PRICED), Some(plain))]); + let drawn = text(&page(&fixture, &table(), &provenance())); + assert!( + !drawn.contains("of which cache"), + "a turn that cached nothing drew a cache breakdown of zeroes:\n{drawn}", + ); + assert!( + !drawn.contains("of which reasoning"), + "a turn that reasoned about nothing drew a reasoning row:\n{drawn}", + ); +} + +// --------------------------------------------------------------------------- +// `money` +// --------------------------------------------------------------------------- + +/// **Money is integer arithmetic at every magnitude, and the precision changes +/// with the magnitude because one precision cannot show both ends.** +/// +/// A turn costs a fraction of a cent and a month costs dollars. Two decimals +/// round the turn to `$0.00`, which reads as free; four decimals pad the month +/// with `$412.9100`, which reads as noise. So it is four below a unit and two +/// above it, and the switch is at the unit rather than at some threshold nobody +/// could name. +/// +/// Every figure here is a division of `u64` by `u64`. A bill is not a +/// floating-point quantity: `cost_micros` is exact, and rendering it through an +/// `f64` would reintroduce a drift the harness went to the trouble of summing in +/// `u128` to avoid. +/// +/// Sabotage: render through `micros as f64 / 1_000_000.0` — under which most of +/// these still pass and the multi-dollar cases drift in the last place. +#[test] +fn money_is_integer_arithmetic_at_every_magnitude() { + // Zero is bare. A total of nothing is not `$0.0000`, which reads as a small + // number somebody rounded. + assert_eq!(cost::money(0), "$0"); + + // **Below the smallest figure four decimals can show, the answer says so + // rather than rounding to a zero.** `$0.0000` reads as free, and a cheap model + // answering a short prompt lands under a hundredth of a cent every time — so + // that form would have printed "free" for most single turns on a small model. + // `$0` above is reserved for a cost that really is zero, which a free model + // has and this does not. + assert_eq!(cost::money(1), "<$0.0001"); + assert_eq!(cost::money(99), "<$0.0001"); + assert_eq!(cost::money(100), "$0.0001"); + assert_eq!(cost::money(12_345), "$0.0123"); + + // Sub-dollar, up to the last micro-unit before the switch. + assert_eq!(cost::money(999_999), "$0.9999"); + + // At and above a unit, two decimals. + assert_eq!(cost::money(1_000_000), "$1.00"); + assert_eq!(cost::money(1_234_567), "$1.23"); + assert_eq!(cost::money(12_345_678_901), "$12345.67"); + + // **Truncated and never rounded up**, which is the honest direction for a + // floor: a figure that rounded to `$2.00` from `$1.999999` would report more + // than the store can account for. + assert_eq!(cost::money(1_999_999), "$1.99"); + assert_eq!(cost::money(99_999), "$0.0999"); + + // The rendering never loses a digit or gains one: every sub-dollar figure it + // can express has four decimals, at every magnitude in the sweep. The + // below-a-hundred-micro-units form is deliberately a different width — it is a + // different *claim*, "smaller than this" rather than "this", and a bound that + // looked identical to a figure would be read as one. + for micros in [100u64, 500, 99_999, 999_999] { + let drawn = cost::money(micros); + assert_eq!( + drawn.len(), + "$0.0000".len(), + "a sub-dollar figure changed width: {drawn}", + ); + } + for micros in [1u64, 7, 99] { + assert_eq!(cost::money(micros), "<$0.0001"); + } + for micros in [1_000_000u64, 9_999_999, 123_456_789] { + let drawn = cost::money(micros); + assert_eq!( + drawn.split('.').next_back().map(str::len), + Some(2), + "a figure at or above a unit did not carry two decimals: {drawn}", + ); + } +} + +// --------------------------------------------------------------------------- +// The page as a whole +// --------------------------------------------------------------------------- + +/// **An empty price table draws every token figure and no money at all.** +/// +/// This is the state every operator is in before their first catalogue read, and +/// the state an operator on a self-hosted endpoint stays in permanently. The +/// module's own argument for having one code path rather than two is written on +/// `cost::table`: "the page is the same page with the money left off, rather than +/// a second page for the unconfigured case that would go stale unread." +/// +/// *Left off* is the claim. A page that draws `cost: $0` under a caveat saying the +/// figure is a floor has not left the money off — it has stated a number nobody +/// measured, in the one situation where no number is available, which is the +/// invented figure the whole module exists to refuse. `Status::note_cost_from` +/// already holds the status line to exactly this rule and says so in as many +/// words: "Nothing priced is not zero priced. A run whose models are all outside +/// the table has a real cost that this program cannot state, and stating it as +/// `$0` would be the invented number the whole of `/cost` is built to avoid." The +/// two surfaces have to agree, and the one-row form is the one that currently +/// does. +/// +/// Sabotage: draw the `cost` row unconditionally, and draw `money(0)` on every +/// grouped row whose calls could none of them be priced. +#[test] +fn an_empty_price_table_draws_tokens_and_no_money() { + let empty = PriceTable::new(""); + let fixture = seeded(&[ + call(Some(PRICED), Some(split())), + call(Some(UNPRICED), Some(split())), + ]); + // No `[prices]` section at all, which is what `Provenance::of` reports for a + // file that has never had one. + let rows = page(&fixture, &empty, &Provenance::default()); + let drawn = text(&rows); + + // The page says so, in the sentence that also says what to do about it. + assert!( + drawn.contains("no prices are configured"), + "the page does not say the table is empty:\n{drawn}", + ); + assert!( + drawn.contains("tokens and no money"), + "the page does not say what it can still answer:\n{drawn}", + ); + + // Every token figure is still exact, because nothing about counting tokens + // needs a rate. + let spelled = io_cli::status::format_tokens; + assert_eq!( + field(&rows, "prompt"), + spelled(split().prompt_tokens * 2), + "the token figures went missing with the prices:\n{drawn}", + ); + assert_eq!( + field(&rows, "completion"), + spelled(split().completion_tokens * 2) + ); + + // And no currency anywhere. Not a zero, not a floor of zero, not a dollar + // sign on a grouped row: there is no rate on this install and every figure + // that would need one is absent. + assert!( + !drawn.contains('$'), + "a page with no prices drew a money figure, which is the invented number \ + `Status::note_cost_from` refuses on the status line:\n{drawn}", + ); +} + +/// **Before anything has run, the page says so rather than drawing a row of +/// zeroes.** +/// +/// The same distinction the status line holds every one of its counters to: a +/// session that has run nothing has no numbers, and a row of zeroes is a claim +/// that it ran and cost nothing. `run` and `session` are both `None` before the +/// first turn, and they are different absences — a session can have turns behind +/// it with no run in flight. +/// +/// Sabotage: `unwrap_or_default()` either id and read run zero, which exists in +/// no store and reads back as an empty call list — under which the page draws +/// `no provider calls` for a session that has never had one, losing the +/// difference between "nothing yet" and "nothing this time". +#[test] +fn a_session_that_has_run_nothing_says_so_rather_than_drawing_zeroes() { + let store = Store::memory().expect("an in-memory store"); + let lines = cost::committed(&store, &table(), &provenance(), None, None, &ascii(), ROOMY) + .expect("the page draws with nothing in the store"); + let drawn: String = lines + .iter() + .map(|line| { + line.spans + .iter() + .map(|span| span.content.as_ref()) + .collect::() + }) + .collect::>() + .join("\n"); + + assert!( + drawn.contains("nothing has run in this session yet"), + "the run section drew something for a session with no run:\n{drawn}", + ); + assert!( + drawn.contains("this session has no turns yet"), + "the session section drew something for a session with no turns:\n{drawn}", + ); + assert!( + drawn.contains("nothing recorded"), + "the grouped sections drew something for an empty store:\n{drawn}", + ); + assert!( + !drawn.contains("calls: 0"), + "an empty store was reported as a run that made no calls:\n{drawn}", + ); +} + +/// **The page's sections are the questions an operator asks, and each figure is +/// under the heading that names its scope.** +/// +/// Seven lists on one page — this run, this session, by model, by day — and a +/// reader has to be able to tell which figure belongs to which question. A page +/// that reported one run's cost under the heading of a whole conversation would be +/// the wrong number under the right word, which is why the session total is walked +/// turn by turn rather than taken from `spend_by_run`. +/// +/// Sabotage: report `Total::of` over the run's calls under `this session` as well +/// — which on a one-turn fixture like this one produces the same number, so the +/// second turn below is what makes the test able to tell. +#[test] +fn the_session_total_is_every_turn_and_not_the_run_in_flight() { + let store = Store::memory().expect("an in-memory store"); + let session = store.create_session("/repo").expect("a session"); + let mut runs = Vec::new(); + for _ in 0..2 { + let run = store.start_run("summarise", "/repo").expect("a run"); + store + .record_turn(session, None, run, "summarise") + .expect("a turn"); + store + .record_provider_call(run, &call(Some(PRICED), Some(split()))) + .expect("recorded"); + runs.push(run); + } + + let table = table(); + let one = table + .cost_micros(PRICED, &split()) + .expect("one call's cost"); + let lines = cost::committed( + &store, + &table, + &provenance(), + Some(runs[1]), + Some(session), + &ascii(), + ROOMY, + ) + .expect("the page draws"); + let rows: Vec = lines + .iter() + .map(|line| { + line.spans + .iter() + .map(|span| span.content.as_ref()) + .collect::() + }) + .collect(); + + // Both sections exist, and they carry different numbers: one run against two + // turns. The `cost` rows come in section order, so the first belongs to the + // run in flight and the second to the conversation. + let costs: Vec<&str> = rows + .iter() + .filter_map(|row| row.trim_start().strip_prefix("cost: ")) + .collect(); + assert_eq!( + costs.first().copied(), + Some(cost::money(one).as_str()), + "the run section is not this run:\n{}", + text(&rows), + ); + assert_eq!( + costs.get(1).copied(), + Some(cost::money(one * 2).as_str()), + "the session section is not every turn of the conversation:\n{}", + text(&rows), + ); + assert!( + text(&rows).contains("across 2 turns"), + "the session section does not say how many turns it walked:\n{}", + text(&rows), + ); +} + +/// **Every `$` on the page is backed by a sentence naming where the rate came +/// from and when it was read.** +/// +/// io-cli reads prices from a catalogue the operator's own provider serves, and +/// for three of the four providers it knows that catalogue is a third party's — +/// OpenAI, Anthropic and Google publish no prices on any endpoint. A figure drawn +/// without that sentence attributes a number to a vendor who never published one. +/// +/// The three cases are three different states of an install, and the middle one +/// is the one a hand-written file produces: prices with no record of where they +/// came from. It gets its date and an honest admission rather than an invented +/// source. +/// +/// Sabotage: fall through to "prices: none configured" whenever `source` is +/// `None`, and a hand-written table reads as no table at all. +#[test] +fn the_footer_names_the_catalogue_and_the_date_or_admits_it_cannot() { + let fixture = seeded(&[call(Some(PRICED), Some(split()))]); + + let full = text(&page(&fixture, &table(), &provenance())); + assert!( + full.contains("417 models") && full.contains("2026-08-27"), + "the footer does not say how many models the rates cover, or when:\n{full}", + ); + assert!( + full.contains("openrouter.ai"), + "the footer does not name the catalogue the rates came from:\n{full}", + ); + + let dated_only = Provenance { + as_of: Some("2026-08-27".into()), + source: None, + models: Some(3), + }; + let hand_written = text(&page(&fixture, &table(), &dated_only)); + assert!( + hand_written.contains("2026-08-27"), + "a hand-written table lost its date:\n{hand_written}", + ); + assert!( + hand_written.contains("did not record"), + "a table with no recorded source got one invented for it:\n{hand_written}", + ); + + let none = text(&page( + &fixture, + &PriceTable::new(""), + &Provenance::default(), + )); + assert!( + none.contains("prices: none configured"), + "an install with no prices does not say so at the foot of the page:\n{none}", + ); +} + +/// **`cost::table` gives one code path rather than two, and `Provenance::of` +/// reads the date from the section that is actually pricing the calls.** +/// +/// The date lives under `[prices]`, which io-harness owns and validates; the +/// source and the model count live under `[app.io-cli.prices]`, which io-harness +/// deliberately does not. That split is not filing: `[prices]` is +/// `deny_unknown_fields` and carries exactly `as_of` and `models`, so a key of +/// io-cli's own put beside them would not be ignored — it would make the +/// operator's whole configuration file fail to parse, taking the policy, the +/// providers and the run budgets down with it. +/// +/// Sabotage: read `as_of` off io-cli's own section, and a table an operator +/// edited by hand reports the date of the last catalogue read instead of the date +/// on the rates in force. +#[test] +fn the_table_in_force_and_its_date_come_from_the_section_that_prices_the_calls() { + let configured = io_harness::Config::from_toml( + "[prices]\nas_of = \"2026-08-27\"\n\n[prices.models]\n\ + \"claude-sonnet-4.5\" = { input = 3000000, output = 15000000 }\n", + ) + .expect("io-harness accepts the fixture"); + + let table = cost::table(&configured); + assert_eq!(table.as_of(), "2026-08-27"); + assert!(table.price(PRICED).is_some()); + + let provenance = Provenance::of(&configured, None); + assert_eq!(provenance.as_of.as_deref(), Some("2026-08-27")); + assert_eq!( + provenance.source, None, + "a file with no io-cli section got a source invented for it", + ); + // **`None`, and asserting `0` here was asserting the defect.** A `[prices]` + // an operator wrote by hand has no `[app.io-cli.prices]` beside it, so nothing + // recorded how many models it prices — which is not the same as its pricing + // none. The field was a `usize` defaulting to zero, and the footer read + // "0 models" over a table pricing four hundred. + assert_eq!( + provenance.models, None, + "an unrecorded count was reported as a measured zero", + ); + + // No `[prices]` at all: an empty table dated to nothing, under which every + // call is unpriced and every token figure is still exact. + let bare = io_harness::Config::from_toml("").expect("an empty configuration"); + let empty = cost::table(&bare); + assert_eq!(empty.as_of(), ""); + assert_eq!(empty.price(PRICED), None); + assert_eq!( + Provenance::of(&bare, None).as_of, + None, + "an install with no prices reported a date it does not have", + ); +} diff --git a/tests/docs.rs b/tests/docs.rs index 058dc20..bb70725 100644 --- a/tests/docs.rs +++ b/tests/docs.rs @@ -240,6 +240,11 @@ fn the_readme_documents_every_key_of_the_io_cli_section() { max_parallel_reads: Some(16), spawn_background_after_secs: Some(120), detached_spawns: Some(true), + prices: Some(io_cli::settings::PriceSettings { + source_url: Some("https://example.invalid/models".into()), + source: Some("the reference catalogue".into()), + models: Some(417), + }), }; let value = serde_json::to_value(&every).expect("[app.io-cli] serializes"); let keys = value.as_object().expect("a table"); diff --git a/tests/edit.rs b/tests/edit.rs index c98f360..8dd680a 100644 --- a/tests/edit.rs +++ b/tests/edit.rs @@ -385,3 +385,326 @@ max_steps = 30 "a move is not reversible" ); } + +/// **A dot inside a quoted key is not a path separator, and the two sides of this +/// module used to disagree about that.** +/// +/// `split_path` split the caller's path on every `.` and normalised nothing; +/// `regions` split the file's header on every `.` and then stripped quotes with +/// `trim_matches`. Two different wrong answers, so `prices.models."gpt-4.1"` never +/// matched itself: the read half answered `None` and the write half fell through +/// to the append arm and emitted a SECOND `[prices.models."gpt-4.1"]` table, which +/// only the read-back caught, and only as "the edit would have produced a file +/// that does not parse" — for a perfectly legal TOML key. +/// +/// Sabotage: put `path.split('.')` back in `split_path`, or +/// `.trim_matches('"').trim_matches('\'')` back in `regions`, or drop either +/// `spell` call from the two spelling sites, and this test fails. +#[test] +fn f1_a_dot_inside_a_quoted_key_is_one_segment_on_both_sides() { + // Two quoted keys, and the second is the one `trim_matches` cannot ever get + // right: a basic string takes the full escape set, so `"a\"b"` is a legal key + // whose NAME is `a"b`. Trimming quotes off both ends of that yields `a\"b`, + // which is a key no file holds. + const QUOTED: &str = "\ +[prices.models.\"gpt-4.1\"] +input = 3.0 +output = 12.0 + +[prices.models.\"a\\\"b\"] +input = 1.0 +"; + + // 1. The header is cut into segments, decoded, and the escape is resolved. + let sections = edit::sections(QUOTED); + assert!( + sections.contains(&vec![ + "prices".to_string(), + "models".to_string(), + "gpt-4.1".to_string() + ]), + "the quoted header was split on the dot inside it: {sections:?}" + ); + assert!( + sections.contains(&vec![ + "prices".to_string(), + "models".to_string(), + "a\"b".to_string() + ]), + "the escaped quote was not decoded: {sections:?}" + ); + + // 2. A caller's path is cut the same way, so the two sides meet. + assert_eq!( + edit::value_at(QUOTED, "prices.models.\"gpt-4.1\".input").as_deref(), + Some("3.0"), + "a dotted key showed a file with no value in it" + ); + // A literal string spells the same key and takes no escapes at all. + assert_eq!( + edit::value_at(QUOTED, "prices.models.'gpt-4.1'.output").as_deref(), + Some("12.0"), + "a literal-string segment did not reach the key a basic string reaches" + ); + assert_eq!( + edit::value_at(QUOTED, "prices.models.\"a\\\"b\".input").as_deref(), + Some("1.0"), + "the escape was not resolved on the caller's side" + ); + // Whitespace around a dot is legal TOML and an unquoted segment is trimmed. + assert_eq!( + edit::value_at(QUOTED, "prices . models . \"gpt-4.1\" . input").as_deref(), + Some("3.0"), + "whitespace around a dot became part of a segment name" + ); + + // 3. The write half reaches the same section, and reaches it exactly once. + // `4.5` rather than `4.0`, and the digit after the point is the point. + // `assert_only_span_changed` reports the span between the longest common + // prefix and the longest common suffix, so replacing `3.0` with `4.0` reports + // `3` -> `4`: the shared `.0` is swallowed by the suffix scan. That would be a + // test asserting against its own helper's arithmetic rather than against the + // writer, and it would go on passing if the writer started truncating floats. + let after = edit::apply( + QUOTED, + &[Edit::set("prices.models.\"gpt-4.1\".input", "4.5")], + ) + .unwrap(); + assert_eq!( + assert_only_span_changed(QUOTED, &after, "prices.models.\"gpt-4.1\".input"), + "4.5" + ); + assert_eq!( + after.matches("[prices.models.\"gpt-4.1\"]").count(), + 1, + "the write fell through to the append arm and duplicated the table:\n{after}" + ); + + // 4. Because a segment is DECODED now, the two sites that spell one back into + // a document have to re-quote anything that is not a bare key. A bare key is + // `A-Za-z0-9_-` and nothing else, so both of these need quotes. + let key = edit::apply( + QUOTED, + &[Edit::set( + "prices.models.\"gpt-4.1\".\"cached input\"", + "1.5", + )], + ) + .unwrap(); + assert!( + key.contains("\"cached input\" = 1.5"), + "the new key was spelled bare and the space in it would not parse:\n{key}" + ); + + let header = edit::apply( + QUOTED, + &[Edit::set("prices.models.\"gpt-4.5\".input", "5.0")], + ) + .unwrap(); + assert!( + header.contains("[prices.models.\"gpt-4.5\"]"), + "the new header was spelled bare and the dot in it would name a third table:\n{header}" + ); + // And the proof that both spellings are right is that the document still says + // what it was asked to say when it is read back as TOML. + let parsed: toml::Value = toml::from_str(&header).expect("the appended header parses"); + assert_eq!( + parsed["prices"]["models"]["gpt-4.5"]["input"].as_float(), + Some(5.0), + "the appended section landed under a different key than the one asked for" + ); +} + +/// **A section's trailing comment run belongs to the section BELOW it.** +/// +/// A region's `body.end` is the first byte of the NEXT header, so splicing +/// `region.start..region.body.end` away deletes everything between this entry's +/// last key and that header — including the comment block an operator wrote to +/// document the next section. `Kind::Move` carried the same bytes away with it. +/// +/// Sabotage: use `region.body.end` at either splice instead of `removal_end`, and +/// the comment naming the entry that survives is deleted along with the one that +/// was asked for. +#[test] +fn f1_removing_or_moving_an_entry_leaves_the_next_one_s_comment_behind() { + const DOCUMENTED: &str = "\ +[[mcp]] +id = \"docs\" + +# search is the one the team actually uses; do not remove it +[[mcp]] +id = \"search\" +"; + + let removed = edit::apply(DOCUMENTED, &[Edit::remove("mcp[0]")]).unwrap(); + assert!( + removed.contains("# search is the one the team actually uses"), + "removing the first entry deleted the second entry's comment:\n{removed}" + ); + assert!(!removed.contains("docs"), "the wrong entry survived"); + // The blank line that separated the removed entry from that comment goes with + // it, so repeated removals do not leave a growing stack of empty lines. + assert!( + removed.starts_with("# search"), + "removal accumulated whitespace above the comment:\n{removed:?}" + ); + + // The same bytes, carried away by a move rather than a removal. + let moved = edit::apply(DOCUMENTED, &[Edit::move_entry("mcp", 0, 1)]).unwrap(); + assert!( + moved.contains("# search is the one the team actually uses"), + "the moved entry took the next entry's comment with it:\n{moved}" + ); + let comment = moved.find("# search").unwrap(); + let docs = moved.find("docs").unwrap(); + assert!( + comment < docs, + "the comment moved down with the entry it does not describe:\n{moved}" + ); +} + +/// **A move has no newline guard, and `Kind::Append` right beside it does.** +/// +/// The last region's `body.end` is the length of the file, so moving an entry +/// into last place splices its header straight onto whatever the final line was. +/// Usually that is a parse refusal; when the final line is a COMMENT with no +/// newline after it the comment swallows the header, the moved entry's keys land +/// inside the previous table, and the file still parses — as something else. +/// +/// Sabotage: delete either half of the guard in `Kind::Move`. The down arm loses +/// the `\n` in front of the block and the up arm loses the one behind it. +#[test] +fn f1_a_move_against_a_file_with_no_trailing_newline_still_parses() { + // No newline after the last line, and the last line is a comment. + const COMMENTED: &str = "\ +[[provider]] +kind = \"openrouter\" +model = \"a\" + +[[provider]] +kind = \"compatible\" +preset = \"groq\" +model = \"b\" +# groq answers last, and there is no newline after this line"; + + let down = edit::apply(COMMENTED, &[Edit::move_entry("provider", 0, 1)]).unwrap(); + let config = io_harness::Config::from_toml(&down).expect("the moved file loads"); + assert_eq!( + config.fallback_specs().len(), + 1, + "the moved entry's keys were absorbed by the entry above it:\n{down}" + ); + assert!( + down.find("groq").unwrap() < down.find("openrouter").unwrap(), + "the move did not reorder the chain:\n{down}" + ); + assert!( + down.contains("# groq answers last"), + "the trailing comment was overwritten:\n{down}" + ); + + // The mirror: the block itself ends without a newline, and the destination's + // header is spliced onto the end of it. + const TAIL: &str = "\ +[[provider]] +kind = \"openrouter\" +model = \"a\" + +[[provider]] +kind = \"compatible\" +preset = \"groq\" +model = \"b\""; + + let up = edit::apply(TAIL, &[Edit::move_entry("provider", 1, 0)]).unwrap(); + let config = io_harness::Config::from_toml(&up).expect("the moved file loads"); + assert_eq!(config.fallback_specs().len(), 1, "{up}"); + assert!( + up.find("groq").unwrap() < up.find("openrouter").unwrap(), + "the move did not reorder the chain:\n{up}" + ); +} + +/// **A whole new section, once — the shape `set` cannot express and the one the +/// price fill could not do without.** +/// +/// Every edit in a batch is resolved against the document as it was *before* the +/// batch: [`edit::apply`] walks the headers once and answers every `set` from that +/// walk. So N `set`s addressing keys of a section that does not exist yet each +/// fall through to the append arm, and each emits its own copy of the header. The +/// file gains N definitions of one table and the read-back refuses the lot — which +/// is what happened to the first price fill, at four hundred models, before this +/// existed. Nothing about the failure named the cause: the operator was told "the +/// edit would have produced a file that does not parse". +/// +/// `Kind::Section` is the answer, and the constraint on it is as important as the +/// capability. It **refuses** a section that is already there rather than replacing +/// it, because a caller holding an existing section wants `set` per key — that +/// leaves every row it did not name alone, which for `[prices.models]` is every +/// model the catalogue stopped serving and every rate the operator corrected by +/// hand. A replace here would delete both from a call site whose author believed +/// they were adding rows. +/// +/// Sabotage: drop the existence check and splice unconditionally, under which the +/// second half of this test passes by writing a file with two `[app.io-cli]` +/// headers in it. Or emit the header per key rather than once, which is the shape +/// that made the failure this exists to remove. +#[test] +fn f1_a_whole_section_is_written_once_and_refused_when_it_is_already_there() { + let body = "\"gpt-4.1\" = { input = 2000000, output = 8000000 }\n\ + \"gpt-4o\" = { input = 2500000, output = 10000000 }\n\ + \"o3\" = { input = 2000000, output = 8000000 }"; + let after = edit::apply( + OPERATORS_FILE, + &[ + Edit::set("prices.as_of", "\"2026-08-27\""), + Edit::section("prices.models", body), + ], + ) + .expect("a section the file does not have is written whole"); + + // One header, whatever the row count. The count is asserted rather than left + // to the parse, because a duplicate is only usually a parse error. + assert_eq!( + after.matches("[prices.models]").count(), + 1, + "the section was written more than once:\n{after}" + ); + + // The operator's own file is still their file, byte for byte, with the new + // sections after it — the same preservation property every other edit here + // keeps. + assert!( + after.starts_with(OPERATORS_FILE), + "writing a new section rewrote the document above it:\n{after}" + ); + + // And it says what it was asked to say. Three dotted ids, each a key rather + // than a path, read back through a parser that has no opinion about io-cli. + let parsed: toml::Value = toml::from_str(&after).expect("the result parses"); + assert_eq!(parsed["prices"]["as_of"].as_str(), Some("2026-08-27")); + for model in ["gpt-4.1", "gpt-4o", "o3"] { + assert!( + parsed["prices"]["models"][model]["input"] + .as_integer() + .is_some(), + "`{model}` did not land as a key of the section:\n{after}" + ); + } + + // The refusal. `[app.io-cli]` is in the fixture, so writing it whole would + // discard the theme — and the message says which section and why rather than + // reporting a parse failure the caller cannot act on. + let error = edit::apply( + OPERATORS_FILE, + &[Edit::section("app.io-cli", "theme = \"light\"")], + ) + .expect_err("a section that already exists must not be written whole"); + assert!( + error.contains("app.io-cli") && error.contains("already in this file"), + "the refusal does not name the section it refused: {error}" + ); + assert!( + error.contains("key by key"), + "the refusal does not say what to do instead: {error}" + ); +} diff --git a/tests/import.rs b/tests/import.rs index 15ab26d..6e9d21a 100644 --- a/tests/import.rs +++ b/tests/import.rs @@ -21,6 +21,17 @@ const SECRET: &str = "sk-live-this-must-never-be-copied"; /// A variable no machine sets, so `${env:…}` for it is unresolvable. const UNSET: &str = "IO_CLI_IMPORT_TEST_TOKEN"; +/// A variable that is always exported, for the tests whose subject is something +/// other than an unresolvable reference. +/// +/// **[`UNSET`] is a premise, not a default.** io-harness treats an unset +/// `${env:…}` as a hard parse error and `configure::write` rolls the whole file +/// back when it hits one, so a test that uses `UNSET` while asserting a server +/// *arrives* is asserting two incompatible things and fails on the rollback — +/// which is correct behaviour reported as a broken test. A test about a quoted +/// server header wants an env reference that resolves. +const ALWAYS_SET: &str = "PATH"; + fn put(path: &Path, text: &str) { if let Some(parent) = path.parent() { std::fs::create_dir_all(parent).expect("a fixture directory"); @@ -349,6 +360,70 @@ fn a_codex_mcp_table_round_trips_into_an_mcp_server() { assert!(env.is_empty()); } +/// **A codex server whose name has a dot in it can only be spelled quoted, and +/// io-cli used to drop it in silence.** +/// +/// `[mcp_servers."github.com/x"]` is how every marketplace-style server id is +/// written, and `crate::edit::sections` reported it as THREE segments because the +/// header was split on every `.`. The `section.len() == 2` filter here then threw +/// it away — no item, no refusal, no line in the report. The two `value_at` paths +/// built from the name have the mirror problem: a decoded name spliced raw into a +/// dotted path names a table nobody has. +/// +/// Sabotage: put `path.split('.')` back in `edit::split_path` or `edit::regions`, +/// or drop `edit::spell` from the paths `codex_config` and `env_names` build. +#[test] +fn a_codex_server_whose_name_must_be_quoted_still_arrives() { + let home_root = tempfile::tempdir().expect("a fake home"); + let workspace = tempfile::tempdir().expect("a workspace"); + let home = tempfile::tempdir().expect("io's home"); + + put( + &home_root.path().join(".codex/config.toml"), + &format!( + "[mcp_servers.\"docs.v2\"]\n\ + command = \"docs-mcp\"\n\ + args = [\"--port\", \"8080\"]\n\ + \n\ + [mcp_servers.\"docs.v2\".env]\n\ + {ALWAYS_SET} = \"{SECRET}\"\n" + ), + ); + + let found = detect(home_root.path(), workspace.path()); + let items: Vec<_> = plan(&found, home.path(), Scope::Project) + .into_iter() + .filter(|item| item.kind == Kind::Mcp) + .collect(); + assert_eq!( + items.len(), + 1, + "a server whose name needed quoting was dropped without a word: {items:?}", + ); + + let report = apply(&items, workspace.path()); + assert!(report.refused.is_empty(), "{report:?}"); + + let servers = written_servers(&workspace.path().join("io.toml")); + assert_eq!(servers.len(), 1); + assert_eq!( + servers[0].id, "docs.v2", + "the id arrived with quotes still on it or split in half", + ); + let (command, args, env) = stdio(&servers[0]); + assert_eq!(command, "docs-mcp"); + assert_eq!(args, vec!["--port", "8080"]); + assert_eq!( + env, + vec![ALWAYS_SET], + "the env table of a quoted server was looked for under an unquoted header", + ); + assert!( + holds(workspace.path(), SECRET).is_none(), + "the secret was copied", + ); +} + // --------------------------------------------------------------------------- // D. Credentials // --------------------------------------------------------------------------- @@ -740,6 +815,94 @@ fn a_skill_name_that_is_already_taken_is_refused_rather_than_overwritten() { ); } +/// **A loose file in a skills root is a skill, whatever it happens to be called.** +/// +/// `skill_files` counts any loose `*.md` at the top of `~/.claude/skills` and +/// `~/.claude/plugins`, so a `CONVENTIONS.md` sitting there is produced as a +/// `Source::Claude` path. `plan`'s first arm matched on the BASENAME with `_` for +/// the source, so it was planned as `Kind::Instructions` — the whole body appended +/// into the standing instructions file that is loaded on every turn, forever, +/// rather than a named skill copied under `/skills` and read on demand. +/// `~/.claude/skills/CLAUDE.md` is the case that proves pairing each anchor name +/// with its owning source would not have closed this: that file genuinely IS +/// `Source::Claude` and genuinely IS named `CLAUDE.md`, and it is still a skill. +/// +/// Sabotage: delete the skills-root guard at the top of `plan`'s loop. +#[test] +fn a_loose_file_in_a_skills_root_is_a_skill_and_never_standing_instructions() { + let home_root = tempfile::tempdir().expect("a fake home"); + let workspace = tempfile::tempdir().expect("a workspace"); + let home = tempfile::tempdir().expect("io's home"); + std::fs::create_dir_all(home.path().join("skills")).expect("a skills directory"); + + // Three loose skills wearing three anchor-file names, and the one real anchor + // file a directory above them. + put( + &home_root.path().join(".claude/skills/CONVENTIONS.md"), + "# conventions\n\nA skill about conventions.\n", + ); + put( + &home_root.path().join(".claude/skills/CLAUDE.md"), + "# claude\n\nA skill that happens to be named like the anchor file.\n", + ); + put( + &home_root.path().join(".claude/plugins/MEMORY.md"), + "# memory\n\nA plugin's loose skill.\n", + ); + put( + &home_root.path().join(".claude/CLAUDE.md"), + "# standing\n\nThe operator's own standing guidance.\n", + ); + + let found = detect(home_root.path(), workspace.path()); + let items = plan(&found, home.path(), Scope::Project); + + let instructions: Vec<_> = items + .iter() + .filter(|item| item.kind == Kind::Instructions) + .collect(); + assert_eq!( + instructions.len(), + 1, + "a skill was planned as standing instructions: {instructions:?}", + ); + assert_eq!( + instructions[0].from, + home_root.path().join(".claude/CLAUDE.md"), + "the anchor file that is instructions is the one OUTSIDE the skills roots", + ); + + let mut names: Vec = items + .iter() + .filter(|item| item.kind == Kind::Skill) + .map(|item| match &item.to { + Destination::File(path) => path + .parent() + .and_then(|parent| parent.file_name()) + .map(|name| name.to_string_lossy().to_string()) + .unwrap_or_default(), + other => panic!("a loose skill file has nowhere to go: {other:?}"), + }) + .collect(); + names.sort(); + assert_eq!(names, vec!["CLAUDE", "CONVENTIONS", "MEMORY"]); + + let report = apply(&items, workspace.path()); + assert!(report.refused.is_empty(), "{report:?}"); + let landed = home.path().join("skills/CONVENTIONS/SKILL.md"); + assert_eq!( + std::fs::read_to_string(&landed).expect("the skill was copied"), + "# conventions\n\nA skill about conventions.\n", + ); + assert!( + io_harness::Skills::discover(home.path().join("skills")) + .expect("it discovers") + .get("CONVENTIONS") + .is_some(), + "the loose file did not become a skill io-harness can find on demand", + ); +} + #[test] fn sixty_three_present_and_three_incoming_is_reported_and_nothing_is_written() { let home_root = tempfile::tempdir().expect("a fake home"); diff --git a/tests/page.rs b/tests/page.rs new file mode 100644 index 0000000..fa5e2a7 --- /dev/null +++ b/tests/page.rs @@ -0,0 +1,528 @@ +//! The shape of a committed page, and the folding four surfaces now share. +//! +//! **This module is a de-duplication, which is the most dangerous kind of change +//! to test.** `status::committed` and `context::committed` each carried their own +//! copy of the folding until 0.22.0 — twenty lines under two names, `folded` and +//! `wrapped`, differing only in that one took its indents as arguments and the +//! other hard-coded the same two numbers. Both had shipped and both were correct. +//! A shared replacement that is merely *plausible* passes every test those two +//! surfaces already have, because none of them asserts a fold at a width where +//! the two implementations could disagree. +//! +//! So the last test here is not a property test at all. It carries a verbatim +//! copy of each of the two functions as 0.21.0 shipped them and runs the new one +//! against both, over inputs chosen to reach the arm that splits a word — which +//! is the arm a rewrite gets wrong and the arm no existing test reaches. The +//! copies are dead code that exists to disagree. + +use io_cli::glyphs::ASCII; +use io_cli::page::{self, Row}; +use io_cli::theme::{Theme, Tone, DARK}; +use ratatui::text::Line; + +/// The theme the sweeps draw in: the ordinary palette, ASCII glyph set, so a +/// failure message is readable in a terminal that cannot draw the Unicode rule. +fn ascii() -> Theme { + DARK.with_glyphs(ASCII) +} + +/// One rendered line as a reader sees it. +fn row(line: &Line<'_>) -> String { + line.spans.iter().map(|s| s.content.as_ref()).collect() +} + +fn drawn(lines: &[Line<'_>]) -> Vec { + lines.iter().map(row).collect() +} + +/// The texts every sweep runs over. +/// +/// Chosen for the arms they reach rather than for realism, though every one of +/// them is a string this product actually draws: a workspace path with no spaces +/// in it, a policy layer's act list, a namespaced model id, a whole sentence, and +/// a single word longer than eighty columns — which is the one that separates +/// folding from fitting. +fn texts() -> Vec { + vec![ + String::new(), + "workspace".to_string(), + "policy: ops-baseline allow read src/* allow write out/* deny read .env".to_string(), + "model: anthropic/claude-sonnet-4.5".to_string(), + "/Users/somebody/Documents/work/monorepo/services/billing/src/domain/invoice.rs" + .to_string(), + "a".repeat(200), + format!("prefix {} suffix", "x".repeat(120)), + "the total above is a floor and not a total, because four calls used a model \ + with no rate in the price table" + .to_string(), + // Multi-byte, because `folded` slices a word by byte index after counting + // its characters, and the two are the same number only in ASCII. + "café ☕ ünïcödé ✱ ".repeat(12), + ] +} + +// --------------------------------------------------------------------------- +// `folded` +// --------------------------------------------------------------------------- + +/// **No row is ever wider than the width it was given, at any width.** +/// +/// The claim the module is named for. A committed page lands in the terminal's +/// own scrollback beside every earlier turn, so a row that runs past the edge is +/// not truncated — it is wrapped by the terminal at a column nothing here chose, +/// which puts the break in the middle of a path and the continuation hard against +/// the left margin where it reads as a new fact. +/// +/// The companion claim is that nothing is lost. Folding is not fitting: the whole +/// argument for it is written on the function, and it is that a committed surface +/// owns as many rows as it needs, so there is no reason left to lose a character — +/// and the characters most likely to be lost are the tail of a path, the tail of +/// an act list and the tail of a model id, which is to say the answer. Compared on +/// the non-whitespace characters, because folding is allowed to move whitespace +/// around and is not allowed to drop anything else. +/// +/// Sabotage: replace the split arm with a truncation, or use `picker::fit` here — +/// under which every width still holds and the character count fails on the two +/// long-word fixtures. +#[test] +fn folding_never_exceeds_the_width_and_never_loses_a_character() { + for text in texts() { + // From one column, which is narrower than either indent and is the case + // the `.max(1)` in the loop exists for. + for width in 1..=120usize { + for (first, rest) in [(0usize, 0usize), (2, 4), (0, 2), (4, 2), (8, 8)] { + let rows = page::folded(&text, width, first, rest); + assert!(!rows.is_empty(), "folding produced no rows at all"); + + // The general bound. A row is its indent plus its content, and + // content is at least one character however narrow the terminal + // is — the `.max(1)` that stops a width under the indent from + // looping forever. So `indent + 1` is the floor of what a row can + // be, and below it the terminal is narrower than a row can be + // drawn at all. + let bound = width.max(first.max(rest) + 1); + for line in &rows { + assert!( + line.chars().count() <= bound, + "a row of {} characters was folded to {width}: {line:?}", + line.chars().count(), + ); + } + // At every width this product supports, the indent fits and the + // bound is the width itself. + if width > first.max(rest) { + for line in &rows { + assert!( + line.chars().count() <= width, + "a row overflowed {width} columns: {line:?}", + ); + } + } + + let before: String = text.chars().filter(|c| !c.is_whitespace()).collect(); + let after: String = rows + .concat() + .chars() + .filter(|c| !c.is_whitespace()) + .collect(); + assert_eq!( + before, after, + "folding lost or invented a character at {width} columns, indents {first}/{rest}", + ); + } + } + } +} + +/// **A word longer than the room it has is split, and a word that merely does not +/// fit *this* row is not.** +/// +/// Two rules that look like one. A word is moved to a fresh row first and only +/// split when a row of its own cannot hold it — so an ordinary sentence never has +/// a word broken across rows, and a two-hundred-character path is broken rather +/// than allowed to run past the edge. A folder that split eagerly would hyphenate +/// the middle of every long word in every sentence; one that never split would put +/// the path past the margin. +/// +/// Sabotage: delete the `if used > 0 { … continue; }` arm, which is the retry on a +/// fresh row — under which the sentence fixture starts splitting words at the end +/// of every row and this test fails while the width property above still holds. +#[test] +fn a_word_longer_than_its_room_is_split_and_one_that_merely_does_not_fit_is_not() { + // Eighty columns, two-space indent: seventy-eight characters of room. A word + // of a hundred and twenty cannot be held by any row and has to be cut. + let long = "x".repeat(120); + let rows = page::folded(&format!("prefix {long} suffix"), 80, 2, 4); + assert!( + rows.len() >= 3, + "a hundred-and-twenty-character word fitted somewhere it could not: {rows:?}", + ); + // Cut at exactly the room a continuation row has — eighty columns less the + // four-space hanging indent — rather than at the room left on the row it + // could not fit, which is what "retried whole on the fresh row" means. + assert!( + rows.iter().any(|line| line.trim() == "x".repeat(76)), + "the word was cut somewhere other than the full width of a fresh row: {rows:?}", + ); + + // The same width, and a sentence of ordinary words. Every row's content is + // whole words, so splitting `text` on whitespace and re-splitting each row + // yields the same sequence of words. + let sentence = "the total above is a floor and not a total, because four calls used a \ + model with no rate in the price table"; + let rows = page::folded(sentence, 80, 2, 4); + assert!(rows.len() > 1, "the fixture has to fold to say anything"); + let words: Vec<&str> = sentence.split_whitespace().collect(); + let folded_words: Vec<&str> = rows + .iter() + .flat_map(|line| line.split_whitespace()) + .collect(); + assert_eq!( + words, folded_words, + "a word was broken across two rows when a fresh row would have held it: {rows:?}", + ); +} + +/// **The hanging indent starts on the second row, which is what says a row is a +/// continuation without aligning anything into a column.** +/// +/// The page is deliberately not a table — the argument is written out at +/// `status::committed` and it is about eighty columns rather than about taste: a +/// column width is decided by the widest cell, and the widest cell on these pages +/// is a model id. So a continuation cannot be marked by aligning under a column +/// that does not exist, and it is marked by being indented further than the row +/// it continues. +/// +/// Sabotage: set `indent = rest` before the first row rather than after it, and +/// every fact on every page gains two spaces while this is the only test that +/// notices — the width property still holds and the characters are all still +/// there. +#[test] +fn the_hanging_indent_marks_a_continuation_from_the_second_row() { + let rows = page::folded( + "policy: ops-baseline allow read src/* allow write out/* deny read .env deny net \ + ads.example.com", + 40, + 2, + 4, + ); + assert!(rows.len() > 2, "the fixture has to fold more than once"); + + assert!( + rows[0].starts_with(" ") && !rows[0].starts_with(" "), + "the first row does not carry the first indent: {:?}", + rows[0], + ); + for line in &rows[1..] { + assert!( + line.starts_with(" ") && !line.starts_with(" "), + "a continuation row does not carry the hanging indent: {line:?}", + ); + } + + // A heading folds at zero and continues at two, which is what makes it + // distinguishable from a fact by POSITION as well as by colour — the whole of + // what a reader under `NO_COLOR` or `--plain` has to go on. + let heading = page::folded("slowest calls, of the last two hundred runs", 20, 0, 2); + assert!(heading.len() > 1); + assert!( + !heading[0].starts_with(' '), + "a heading is indented to zero" + ); + assert!(heading[1].starts_with(" ")); +} + +/// **An empty text is one indented row, not no rows at all.** +/// +/// A caller folding an empty value gets a row it can put on the page, which is +/// what keeps `Row::Fact("label", "")` from silently disappearing. The tail +/// condition is `used > 0 || rows.is_empty()`, and the second half of it is the +/// whole of this behaviour. +/// +/// Sabotage: drop `|| rows.is_empty()`, under which an empty string folds to +/// nothing and a fact with an empty value vanishes from the page rather than +/// reading as empty. +#[test] +fn an_empty_text_folds_to_one_row_rather_than_to_none() { + assert_eq!(page::folded("", 80, 2, 4), vec![" ".to_string()]); + assert_eq!(page::folded(" ", 80, 0, 0), vec![String::new()]); +} + +// --------------------------------------------------------------------------- +// `commit` +// --------------------------------------------------------------------------- + +/// **A committed page has edges, and both of them name it.** +/// +/// It lands in the terminal's own scrollback beside every earlier turn and every +/// line of transcript, so a passage with no edges is one a reader cannot tell the +/// extent of — where the page started, and whether what follows is still part of +/// it. The closing rule carries `ends` for the same reason `transcript` does: two +/// identical rules would be two openings. +/// +/// Sabotage: drop the closing line, under which a page runs into whatever the +/// next turn writes and nothing fails anywhere else. +#[test] +fn a_page_is_edged_at_both_ends_with_the_title_on_each() { + let theme = ascii(); + let rule = theme.glyphs.rule; + let lines = page::commit( + "cost", + &[Row::fact("calls", "12"), Row::note("nothing recorded")], + &theme, + 80, + ); + let rows = drawn(&lines); + + assert_eq!(rows[0], format!("{rule}{rule}{rule} cost")); + assert_eq!( + rows[rows.len() - 1], + format!("{rule}{rule}{rule} cost ends"), + "the closing rule does not say the page ended, so it reads as a second opening", + ); + assert_eq!( + rows.len(), + 4, + "two rows between two rules, and nothing else: {rows:?}", + ); +} + +/// **Every row kind reaches the page, and the two that qualify a figure do not +/// read the same as the two that state one.** +/// +/// `Row::Note` and `Row::caveat` are the same variant with different tones, and +/// the difference is not decoration: io-harness's own pricing documentation calls +/// a renderer that draws a floor and a total identically "lying by omission". A +/// heading is separated by position *and* tone for the same reason, since a reader +/// under `NO_COLOR` has only the position. +/// +/// Sabotage: make `Row::caveat` call `Row::note`, under which a total that is a +/// floor reads exactly like one that is not, and no test that reads text alone +/// can tell. +#[test] +fn every_row_kind_draws_and_a_caveat_does_not_read_like_a_note() { + let theme = ascii(); + let lines = page::commit( + "stats", + &[ + Row::heading("runs by outcome"), + Row::fact("success", "12"), + Row::Blank, + Row::note("nothing recorded"), + Row::caveat("the cost above is a floor and not a total"), + ], + &theme, + 80, + ); + let rows = drawn(&lines); + + assert_eq!(rows[1], "runs by outcome", "a heading is indented to zero"); + assert_eq!(rows[2], " success: 12"); + assert_eq!(rows[3], "", "a blank row is blank"); + assert_eq!(rows[4], " nothing recorded"); + assert_eq!(rows[5], " the cost above is a floor and not a total"); + + // The tones. Read off the theme rather than compared to a colour typed here, + // so a palette change moves both sides together and only a collapse of the + // two tones fails. + let style_of = |index: usize| lines[index].spans[0].style; + assert_eq!(style_of(1), theme.style(Tone::Accent), "the heading"); + assert_eq!(style_of(2), theme.style(Tone::Normal), "the fact"); + assert_eq!(style_of(4), theme.style(Tone::Muted), "the note"); + assert_eq!(style_of(5), theme.style(Tone::Warning), "the caveat"); + assert_ne!( + style_of(4), + style_of(5), + "a qualification and an ordinary sentence draw identically, so a floor \ + reads as a total", + ); +} + +/// **A page folds its rows rather than letting one run past the edge, at every +/// width the product supports.** +/// +/// The integration of the two halves: `commit` hands each row to `folded` with +/// the indents that row kind takes, and a page whose facts were merely `format!`ed +/// straight into `Line`s would pass every content assertion above and overflow the +/// terminal on the one row that matters — which on `/cost` and `/stats` is the row +/// carrying a namespaced model id. +/// +/// Sabotage: push `Line::from(format!(" {label}: {value}"))` in the `Fact` arm +/// instead of folding it, and this is the only test that fails. +#[test] +fn no_row_of_a_committed_page_runs_past_the_terminal() { + let theme = ascii(); + let rows = [ + Row::heading("slowest calls, of the last 200 runs"), + Row::fact( + "anthropic/claude-sonnet-4.5-with-an-absurdly-long-deployment-suffix", + "4213 ms", + ), + Row::fact( + "workspace", + "/Users/somebody/Documents/work/monorepo/services/billing", + ), + Row::caveat( + "four calls used a model with no rate in the price table, so the cost above \ + is a floor and not a total", + ), + Row::Blank, + ]; + + for width in 20..=120u16 { + let lines = page::commit("cost", &rows, &theme, width); + // The first and last rows are the rules. They carry the title rather than + // content and are deliberately not folded — a rule broken across two rows + // would stop being an edge — so the claim is about everything between them. + for line in &lines[1..lines.len() - 1] { + let drawn: usize = line + .spans + .iter() + .map(|span| span.content.chars().count()) + .sum(); + assert!( + drawn <= width as usize, + "a page row overflowed {width} columns: {line:?}", + ); + } + } +} + +// --------------------------------------------------------------------------- +// The de-duplication itself +// --------------------------------------------------------------------------- + +/// `status::folded` exactly as 0.21.0 shipped it, before the four surfaces shared +/// one. +/// +/// Verbatim, comments and all. It is here to disagree, so it is not tidied, not +/// renamed and not improved — a copy that had been cleaned up would be a third +/// implementation and would prove nothing about the second. +fn folded_0_21_0(text: &str, width: usize, first: usize, rest: usize) -> Vec { + let mut rows: Vec = Vec::new(); + let mut indent = first; + let mut row = " ".repeat(indent); + let mut used = 0usize; + for word in text.split_whitespace() { + let mut word = word; + loop { + let room = width.saturating_sub(indent).max(1); + let space = usize::from(used > 0); + let length = word.chars().count(); + if used + space + length <= room { + if space == 1 { + row.push(' '); + } + row.push_str(word); + used += space + length; + break; + } + if used > 0 { + rows.push(std::mem::take(&mut row)); + indent = rest; + row = " ".repeat(indent); + used = 0; + continue; + } + let head: String = word.chars().take(room).collect(); + word = &word[head.len()..]; + row.push_str(&head); + rows.push(std::mem::take(&mut row)); + indent = rest; + row = " ".repeat(indent); + } + } + if used > 0 || rows.is_empty() { + rows.push(row); + } + rows +} + +/// `context::wrapped` exactly as 0.21.0 shipped it. +/// +/// The same twenty lines with the two indents written in rather than taken as +/// arguments, which is the whole of what separated the two copies. Kept whole for +/// the same reason as above: a paraphrase would be testing the paraphrase. +fn wrapped_0_21_0(text: &str, width: usize) -> Vec { + let mut rows: Vec = Vec::new(); + let mut indent = 2usize; + let mut row = " ".repeat(indent); + let mut used = 0usize; + for word in text.split_whitespace() { + let mut word = word; + loop { + let room = width.saturating_sub(indent).max(1); + let space = usize::from(used > 0); + let length = word.chars().count(); + if used + space + length <= room { + if space == 1 { + row.push(' '); + } + row.push_str(word); + used += space + length; + break; + } + if used > 0 { + rows.push(std::mem::take(&mut row)); + indent = 4; + row = " ".repeat(indent); + used = 0; + continue; + } + let head: String = word.chars().take(room).collect(); + word = &word[head.len()..]; + row.push_str(&head); + rows.push(std::mem::take(&mut row)); + indent = 4; + row = " ".repeat(indent); + } + } + if used > 0 || rows.is_empty() { + rows.push(row); + } + rows +} + +/// **The shared folder is the two it replaced, row for row, at every width.** +/// +/// A de-duplication is a claim about behaviour that no user-facing test states: +/// `/status` and `/context` each have a suite full of `contains` assertions, and +/// not one of them would notice a fold that broke one column earlier — the words +/// are all still on the page, in order, in the same rows plus or minus one. The +/// only way to state "nothing moved" is to keep the thing it moved from. +/// +/// The sweep runs over the fixtures that reach the split arm as well as the ones +/// that do not, because that arm is the one a rewrite gets wrong: it is reached +/// only when a single word cannot fit a row of its own, which on a real `/status` +/// page happens at eighty columns for a deep workspace path and never in a test +/// written against a short one. +/// +/// `wrapped` is compared at its own two indents, since that is the only shape it +/// had; anything else would be comparing the new function against a version of the +/// old one that never existed. +/// +/// Sabotage: change `used + space + length <= room` to `<`, which is the classic +/// off-by-one and is invisible on any page whose longest row is not exactly full — +/// under which this test fails at a handful of widths per fixture and every other +/// test in the repository stays green. +#[test] +fn the_shared_folder_is_the_two_copies_it_replaced() { + for text in texts() { + for width in 1..=120usize { + for (first, rest) in [(0usize, 0usize), (2, 4), (0, 2), (4, 2), (8, 8)] { + assert_eq!( + page::folded(&text, width, first, rest), + folded_0_21_0(&text, width, first, rest), + "the shared folder differs from `status::folded` at {width} columns, \ + indents {first}/{rest}, on {text:?}", + ); + } + assert_eq!( + page::folded(&text, width, 2, 4), + wrapped_0_21_0(&text, width), + "the shared folder differs from `context::wrapped` at {width} columns \ + on {text:?}", + ); + } + } +} diff --git a/tests/palette.rs b/tests/palette.rs index a1d6fbe..fbbde92 100644 --- a/tests/palette.rs +++ b/tests/palette.rs @@ -304,12 +304,13 @@ fn f1_an_exact_name_outranks_a_prefix_which_outranks_a_scattered_match() { #[test] fn f1_equal_scores_keep_the_first_row_still_between_keystrokes() { - // `c` matches EIGHT rows since 0.17.0: `copy`, `copy diff`, `contain`, - // `clear`, `config`, `context` and `compact` all begin with it, and `mcp` - // merely holds one — which is the point of the arm, since a scattered match - // ranks below every prefix. `attach`, which also merely held one, stopped - // being a command in 0.13.1. `co` narrows to the six that begin with `co`: - // `clear` has no `o` after its `c`, and neither does `mcp`. + // `c` matches NINE rows since 0.22.0: `copy`, `copy diff`, `contain`, + // `clear`, `config`, `context`, `compact` and now `cost` all begin with it, + // and `mcp` merely holds one — which is the point of the arm, since a + // scattered match ranks below every prefix. `attach`, which also merely held + // one, stopped being a command in 0.13.1. `co` narrows to the seven that + // begin with `co`: `clear` has no `o` after its `c`, and neither does `mcp`. + // (`stats`, the other command 0.22.0 added, holds no `c` at all.) // The three score the same and the tie-break is the order they were handed // in. The defect: an unstable sort swaps them on a keystroke that did not // change the result, and `Enter` takes a row nobody chose. @@ -320,14 +321,14 @@ fn f1_equal_scores_keep_the_first_row_still_between_keystrokes() { // quietly stopped happening. let mut picker = palette(); type_at(&mut picker, "c"); - assert_eq!(picker.matching(), 8); + assert_eq!(picker.matching(), 9); // `clear` and not `copy` since the list became grouped: the tie-break is the // order the rows were handed in, and the session group now leads. The // PROPERTY is unchanged and is what this test is about — the marked row must // not move on a keystroke that did not change the result. assert_eq!(marked(&picker), "clear"); type_at(&mut picker, "o"); - assert_eq!(picker.matching(), 6); + assert_eq!(picker.matching(), 7); assert_eq!( marked(&picker), "contain", diff --git a/tests/prices.rs b/tests/prices.rs new file mode 100644 index 0000000..1b25cc5 --- /dev/null +++ b/tests/prices.rs @@ -0,0 +1,1164 @@ +//! Where a price comes from, what it is spelled as, and what it refuses. +//! +//! **Four separate claims live in `src/prices.rs` and only one of them is +//! arithmetic.** The date conversion is a pure function of a number and can be +//! checked against a calendar; the rest are claims about *naming* — which id a +//! rate is filed under, whose catalogue it is attributed to, and whether the row +//! survives a trip through a real configuration file and back out of io-harness. +//! A test that only exercised `Catalogue::of` in memory would pass on every one +//! of those and still ship a release that writes nothing an operator can read. +//! +//! So the end of this file is deliberately not a unit test. It takes the edits +//! the module produces, runs them through `io_cli::edit::apply` over a +//! hand-written `io.toml`, hands the result to `io_harness::Config::from_toml`, +//! and asks the resulting `PriceTable` what a call on that model costs. That +//! round trip is the property — every step before it is a step towards it, and a +//! module that got the spelling wrong is a module whose prices exist in a file +//! nothing can read. + +use io_cli::edit; +use io_cli::prices::{self, Catalogue, DEFAULT_REFERENCE_URL}; +use io_harness::pricing::{Price, PriceTable}; +use io_harness::{Config, ModelInfo, PriceSource, ProviderSpec}; + +// --------------------------------------------------------------------------- +// Fixtures +// --------------------------------------------------------------------------- + +/// A configuration naming one provider, held so a `&ProviderSpec` can be taken +/// off it. +/// +/// **Built through io-harness's own parser rather than as a struct literal**, and +/// that is not ceremony. `ProviderSpec` is `#[non_exhaustive]` and its +/// `Compatible` variant carries seven fields; a literal here would be a second +/// spelling of a shape the harness owns, and it would stop compiling the release +/// the harness widens it — which is the wrong test failing for the wrong reason. +/// A `[[provider]]` table is also exactly what an operator writes, so the fixture +/// and the field agree by construction. +fn configured(body: &str) -> Config { + Config::from_toml(body).expect("the fixture is a configuration io-harness accepts") +} + +fn openrouter() -> Config { + configured("[[provider]]\nkind = \"openrouter\"\nmodel = \"anthropic/claude-sonnet-4.5\"\n") +} + +fn anthropic() -> Config { + configured("[[provider]]\nkind = \"anthropic\"\nmodel = \"claude-sonnet-4.5\"\n") +} + +fn openai() -> Config { + configured("[[provider]]\nkind = \"openai\"\nmodel = \"gpt-4.1\"\n") +} + +/// A provider the reference catalogue has never heard of and cannot speak for. +fn local() -> Config { + configured( + "[[provider]]\nkind = \"compatible\"\nbase_url = \"http://localhost:11434/v1\"\n\ + model = \"llama3.2\"\nauth = \"none\"\n", + ) +} + +fn spec(config: &Config) -> &ProviderSpec { + config + .provider_spec() + .expect("the fixture names a provider") +} + +/// One catalogue row the vendor put a rate on. +fn priced(id: &str, input: u64, output: u64) -> ModelInfo { + ModelInfo { + id: id.to_string(), + price: Some(Price { + input, + output, + ..Price::ZERO + }), + price_source: Some(PriceSource::Vendor), + ..Default::default() + } +} + +/// One catalogue row the vendor served and put no rate on. +/// +/// **`price: None`, never `Some(Price::ZERO)`**, which is io-harness's own rule +/// stated on `ModelInfo::price` in as many words: zero is a rate and unknown is +/// not one, and a catalogue that conflated them would report a real bill as free. +fn unpriced(id: &str) -> ModelInfo { + ModelInfo { + id: id.to_string(), + price: None, + price_source: None, + ..Default::default() + } +} + +// --------------------------------------------------------------------------- +// `date` +// --------------------------------------------------------------------------- + +/// Whether `year` carries a twenty-ninth of February, by the rule rather than by +/// the shortcut. +/// +/// Divisible by four, except centuries, except centuries divisible by four +/// hundred. The third clause is the one a naive `% 4` gets wrong, and 2100 is the +/// first year the epoch can express where it matters — which is why the sweep +/// below runs past it. +fn leap(year: i64) -> bool { + year % 4 == 0 && (year % 100 != 0 || year % 400 == 0) +} + +fn days_in(year: i64, month: i64) -> i64 { + match month { + 1 | 3 | 5 | 7 | 8 | 10 | 12 => 31, + 4 | 6 | 9 | 11 => 30, + _ => { + if leap(year) { + 29 + } else { + 28 + } + } + } +} + +/// **The date is the civil calendar, checked against a second calendar rather +/// than against itself.** +/// +/// `prices::date` is Howard Hinnant's `civil_from_days`: an era of four hundred +/// years, a year-of-era, a day-of-year and two integer divisions that reconstruct +/// the month with no lookup table. It is correct and it is completely opaque, and +/// a test that asserted `date(1_772_150_400) == "2026-02-27"` would be asserting +/// that somebody once ran the function and wrote down what it said. +/// +/// So there are two halves here. The first is a handful of dates an ordinary +/// person can verify by counting — the epoch itself, a leap day with the day +/// either side of it, a year boundary with the day either side of *it*, and +/// 2100-03-01, which follows a February the century rule shortens back to +/// twenty-eight. The second is a walk: an independent day-by-day calendar, +/// written out above with the leap rule spelled in full, stepped forward from the +/// epoch for a hundred and ten years and compared against `date` at every single +/// step. The two implementations share nothing but the definition of a day. +/// +/// Sabotage: drop the `|| year % 400 == 0` clause from Hinnant's era arithmetic — +/// which is to say make 2000 an ordinary century — and the walk fails on +/// 2000-02-29 while every hand-written date before it still passes. Or drop the +/// `if month <= 2 { year + 1 }` correction, and every January and February in the +/// range moves back a year while the whole of March to December stays right. +#[test] +fn the_date_is_the_civil_calendar_and_not_a_transcription_of_the_function() { + // Counted by hand. 1970-01-01 is day zero; 2000-01-01 is thirty years of + // three hundred and sixty-five days plus the seven leap days of 1972 to + // 1996, which is 10,957 days, which is 946,684,800 seconds. + for (secs, expected) in [ + (0u64, "1970-01-01"), + // The last second of the first day. A conversion that divided wrongly + // would tip over here rather than at midnight. + (86_399, "1970-01-01"), + (86_400, "1970-01-02"), + // A year boundary, with the day either side of it. + (946_598_400, "1999-12-31"), + (946_684_800, "2000-01-01"), + // A leap day, with the day either side of it. 2000 is the century that + // *is* a leap year, which is the rule's hardest case in this range. + (951_696_000, "2000-02-28"), + (951_782_400, "2000-02-29"), + (951_868_800, "2000-03-01"), + // And the century that is not. 2100 is divisible by four and has no + // twenty-ninth of February, so the first of March follows the + // twenty-eighth directly. + (4_107_456_000, "2100-02-28"), + (4_107_542_400, "2100-03-01"), + ] { + assert_eq!( + prices::date(secs), + expected, + "{secs} seconds after the epoch is {expected}", + ); + } + + // The seconds inside a day do not reach the answer. Asserted rather than + // assumed, because the whole point of taking a `u64` of seconds is that the + // caller hands over a clock reading and not a date. + for offset in [0u64, 1, 3_600, 43_200, 86_399] { + assert_eq!( + prices::date(951_782_400 + offset), + "2000-02-29", + "a time of day moved the date", + ); + } + + // The walk. Forty thousand days is 1970-01-01 through the middle of 2079, + // which covers twenty-seven leap years, one leap century and — with the + // hand-written pair above — the century that is not one. + let (mut year, mut month, mut day) = (1970i64, 1i64, 1i64); + for index in 0..40_000u64 { + assert_eq!( + prices::date(index * 86_400), + format!("{year:04}-{month:02}-{day:02}"), + "the two calendars disagree {index} days after the epoch", + ); + day += 1; + if day > days_in(year, month) { + day = 1; + month += 1; + } + if month > 12 { + month = 1; + year += 1; + } + } +} + +// --------------------------------------------------------------------------- +// `Catalogue::of` +// --------------------------------------------------------------------------- + +/// **On OpenRouter the reference catalogue is the vendor, and saying otherwise +/// would understate it.** +/// +/// The catalogue io-cli reads *is* OpenRouter's own `/api/v1/models`, so an +/// OpenRouter operator is being quoted the provider's published rate by the +/// provider. Calling that a reference would attach a hedge to the one case where +/// no hedge is owed. The ids are also kept whole — `anthropic/claude-sonnet-4.5` +/// is how OpenRouter names the model, and it is therefore what io-harness records +/// on the provider call and what the price has to be filed under. +/// +/// Sabotage: strip the vendor prefix for OpenRouter too, which reads as a tidy-up +/// and is the one change that makes every OpenRouter row unfindable — the key in +/// the table becomes `claude-sonnet-4.5` and the `model` on every call stays +/// `anthropic/claude-sonnet-4.5`, so `Spend::unpriced_calls` counts the lot and +/// `/cost` reports a floor of zero without a single thing failing. +#[test] +fn openrouter_keeps_the_catalogues_own_spelling_and_is_quoted_as_the_vendor() { + let config = openrouter(); + let catalogue = Catalogue::of( + spec(&config), + vec![ + priced("anthropic/claude-sonnet-4.5", 3_000_000, 15_000_000), + priced("openai/gpt-4.1", 2_000_000, 8_000_000), + ], + "2026-08-27", + ); + + assert_eq!(catalogue.as_of, "2026-08-27"); + assert_eq!(catalogue.served, 2); + assert_eq!( + catalogue.source, + PriceSource::Vendor, + "the catalogue is OpenRouter's own, so for OpenRouter it is not a reference", + ); + let ids: Vec<&str> = catalogue.rows.iter().map(|(id, _)| id.as_str()).collect(); + assert_eq!( + ids, + ["anthropic/claude-sonnet-4.5", "openai/gpt-4.1"], + "an id was re-spelled, so the key will not match the model on a call", + ); +} + +/// **Anthropic and OpenAI lose the prefix and gain the hedge, and both halves +/// matter.** +/// +/// Neither vendor publishes a price on any endpoint, so the numbers come from a +/// third party's catalogue and every surface that draws money has to say so. The +/// spelling moves in the opposite direction from the attribution: the catalogue +/// namespaces the model (`anthropic/claude-sonnet-4.5`) and the operator's own +/// configuration does not (`claude-sonnet-4.5`), and it is the operator's +/// spelling that io-harness records on the call. +/// +/// The rows a prefix does not match are dropped entirely rather than kept under +/// their catalogue name. An OpenAI operator has no use for a row keyed +/// `anthropic/claude-sonnet-4.5`; it can never match a call they make, and it +/// would inflate the model count `/cost` reports and the count `too_short` +/// measures a later read against. +/// +/// Sabotage: keep the whole catalogue for these two providers rather than +/// filtering it, and this test fails on the row count while every other test in +/// this file — including the round trip — still passes. +#[test] +fn anthropic_and_openai_lose_their_prefix_and_are_quoted_as_a_reference() { + let served = || { + vec![ + priced("anthropic/claude-sonnet-4.5", 3_000_000, 15_000_000), + priced("anthropic/claude-opus-4.1", 15_000_000, 75_000_000), + // Served by this provider, and priced by nobody. It is what makes + // `served` and `rows.len()` different numbers below. + unpriced("anthropic/claude-experimental-preview"), + priced("openai/gpt-4.1", 2_000_000, 8_000_000), + priced("google/gemini-2.5-pro", 1_250_000, 10_000_000), + ] + }; + + let config = anthropic(); + let catalogue = Catalogue::of(spec(&config), served(), "2026-08-27"); + let ids: Vec<&str> = catalogue.rows.iter().map(|(id, _)| id.as_str()).collect(); + assert_eq!( + ids, + ["claude-opus-4.1", "claude-sonnet-4.5"], + "the prefix survived, or a model this provider does not serve did", + ); + assert_eq!( + catalogue.source, + PriceSource::Reference(DEFAULT_REFERENCE_URL.to_string()), + "Anthropic publishes no prices, so a rate about Anthropic is not from Anthropic", + ); + // **`served` counts what THIS provider serves, and `rows` counts what of it + // carries a rate.** The two are deliberately different numbers, and the gap is + // the sentence `fill_prices` writes: "priced two of the three models the + // reference catalogue serves". Counting the reference's whole four-hundred-row + // answer here would tell an Anthropic operator their rates cover two of four + // hundred, which is true of the catalogue and meaningless about them. + assert_eq!(catalogue.served, 3, "three of the fixture are Anthropic's"); + assert_eq!(catalogue.rows.len(), 2, "two of those three carry a rate"); + + let config = openai(); + let catalogue = Catalogue::of(spec(&config), served(), "2026-08-27"); + let ids: Vec<&str> = catalogue.rows.iter().map(|(id, _)| id.as_str()).collect(); + assert_eq!(ids, ["gpt-4.1"]); + assert_eq!( + catalogue.source, + PriceSource::Reference(DEFAULT_REFERENCE_URL.to_string()), + ); +} + +/// **A provider the reference cannot speak for gets no rows at all.** +/// +/// A `compatible` endpoint is a URL and a model slug: a proxy, a gateway, a +/// runtime on a port. The reference catalogue has never heard of it, and the only +/// thing it could offer is a row whose slug happens to collide — `llama3.2` on +/// somebody's laptop is not `meta-llama/llama-3.2-3b-instruct` on OpenRouter, and +/// pricing a local run at a hosted rate would be an invented bill. +/// +/// `served` is zero for the same reason `rows` is: the count is of what *this* +/// provider serves, and the reference cannot say that a local endpoint serves +/// anything at all. Reporting the reference's own four hundred here would be the +/// same confident wrong answer in a different column. +/// +/// Sabotage: make the `_` arm of `verify::named` return `models` unchanged, and a +/// local endpoint gains a full price table for models it does not run. +#[test] +fn a_provider_the_reference_cannot_speak_for_yields_no_rows() { + let config = local(); + let catalogue = Catalogue::of( + spec(&config), + vec![ + priced("meta-llama/llama-3.2-3b-instruct", 20_000, 20_000), + priced("llama3.2", 20_000, 20_000), + ], + "2026-08-27", + ); + + assert!( + catalogue.rows.is_empty(), + "a local endpoint was priced from a catalogue that has never seen it: {:?}", + catalogue.rows, + ); + assert_eq!( + catalogue.served, 0, + "the reference claimed to know what this endpoint serves", + ); +} + +/// **The same endpoint, with a catalogue the operator named — and this is the +/// half that was inert.** +/// +/// `app.io-cli.prices.source_url` is documented in `src/settings.rs` and in the +/// README as the only way an operator on a self-hosted or `compatible` endpoint +/// gets prices at all. It could not work: the read fetched the operator's own +/// catalogue and then put it through the same narrowing as the default one, whose +/// `_` arm discards every row for a `compatible` provider. So io-cli fetched the +/// right list and threw all of it away, and the test above — asserting the empty +/// result — was the only coverage either path had. +/// +/// The narrowing exists to cut one vendor's view of the whole field down to the +/// provider in force. An operator who set `source_url` has already answered that +/// question: they pointed io-cli at the list their own endpoint publishes, so +/// every row of it is theirs and the ids are already spelled the way their +/// provider names them. A named catalogue is therefore taken whole. +/// +/// The source is still recorded as the URL and not as the vendor, because it is +/// still not the vendor speaking — it is a list the operator chose. +/// +/// Sabotage: route `Catalogue::named` through the same filter as +/// `Catalogue::of` — under which only this test fails, and it fails by returning +/// the release to the state where the key exists, is documented, and does nothing. +#[test] +fn a_catalogue_the_operator_named_is_taken_whole_and_attributed_to_its_url() { + const MINE: &str = "http://localhost:11434/v1/models"; + let config = local(); + let catalogue = Catalogue::named( + spec(&config), + vec![ + priced("llama3.2", 20_000, 20_000), + priced("qwen3-coder", 30_000, 30_000), + unpriced("nomic-embed-text"), + ], + "2026-08-27", + MINE, + ); + + let ids: Vec<&str> = catalogue + .rows + .iter() + .map(|(model, _)| model.as_str()) + .collect(); + assert_eq!( + ids, + ["llama3.2", "qwen3-coder"], + "the operator's own catalogue was narrowed by a filter built for somebody else's", + ); + // Served counts what the endpoint offers, priced or not — the third row is a + // model it serves and put no rate on, which is a different fact from a model + // it does not serve. + assert_eq!(catalogue.served, 3); + assert_eq!( + catalogue.source, + PriceSource::Reference(MINE.to_string()), + "a list the operator chose was attributed to the vendor", + ); + assert!( + prices::source_word(&catalogue.source).contains(MINE), + "the page would not say which catalogue these rates came from", + ); + + // And it survives the write, which is the only thing that makes any of it + // reach a `/cost` page. + let written = edit::apply(FRESH, &catalogue.edits(prices::has_models_section(FRESH))) + .expect("the edits produce a file that parses"); + let table = Config::from_toml(&written) + .expect("io-harness accepts it") + .prices() + .expect("the file carries a table"); + assert_eq!( + table.price("llama3.2").map(|price| price.input), + Some(20_000) + ); + assert_eq!( + table.price("nomic-embed-text"), + None, + "a model served with no rate was entered at zero", + ); +} + +/// **A model the catalogue served with no price is ABSENT, never entered at +/// zero.** +/// +/// This is the single most consequential line in the module and it is one that +/// leaves no trace when it goes wrong. `PriceTable::price` returning `None` is +/// what makes `Spend::unpriced_calls` count the call, which is what makes +/// `Total::is_floor` true, which is what makes `/cost` say the number it just +/// drew is a floor rather than a total. A row entered as `Price::ZERO` satisfies +/// every one of those checks — the model *is* priced, at nothing — so the call is +/// counted as free, the total is reported as complete, and every surface agrees +/// with every other surface about a number that is wrong. +/// +/// io-harness says the same thing on `ModelInfo::price` itself: "never +/// `Price::ZERO` to mean unknown". This asserts io-cli honours it. +/// +/// Sabotage: `.map(|m| (m.id, m.price.unwrap_or(Price::ZERO)))` in +/// `verify::priced` — five characters that read as a tidy-up of an `Option`, and +/// under which this is the only test in the repository that fails. +#[test] +fn a_model_the_catalogue_priced_at_nothing_is_absent_rather_than_zero() { + let config = openrouter(); + let catalogue = Catalogue::of( + spec(&config), + vec![ + priced("anthropic/claude-sonnet-4.5", 3_000_000, 15_000_000), + unpriced("some-lab/experimental-preview"), + unpriced("another-lab/unreleased"), + ], + "2026-08-27", + ); + + assert_eq!(catalogue.served, 3, "all three were served"); + let ids: Vec<&str> = catalogue.rows.iter().map(|(id, _)| id.as_str()).collect(); + assert_eq!( + ids, + ["anthropic/claude-sonnet-4.5"], + "an unpriced model reached the table, where it will be counted as free", + ); + // And the proof that absence is what the table sees: the row that is missing + // is missing from the `PriceTable` a write of these rows would produce. + let table = catalogue.rows.iter().fold( + PriceTable::new(catalogue.as_of.clone()), + |table, (id, price)| table.with(id.clone(), *price), + ); + assert_eq!(table.price("some-lab/experimental-preview"), None); + assert!(table.price("anthropic/claude-sonnet-4.5").is_some()); +} + +// --------------------------------------------------------------------------- +// `Catalogue::too_short` +// --------------------------------------------------------------------------- + +/// **The refusal that stops a truncated read from shrinking a bill in silence.** +/// +/// Every other failure in this area is loud: a catalogue that cannot be reached +/// leaves the operator with no prices and a sentence saying so. A catalogue that +/// answers with four rows where four hundred were expected is the quiet one — the +/// write succeeds, the file parses, `/cost` draws a smaller number, and nothing +/// anywhere says the table lost three hundred and ninety-six models. +/// +/// The rule has three arms and each is a different judgement. A first fill has +/// nothing to lose and is never refused, whatever it holds. An empty answer +/// against a table that has rows is refused outright — a replacement that prices +/// nothing is not a price change. And an answer under half the size of what it +/// would replace is refused as a truncation, while an answer merely *smaller* is +/// allowed through, because vendors do retire models and a rule that refused +/// every shrinkage would freeze the table the first time one did. +/// +/// Sabotage: return `false` for the empty case — under which the emptiest possible +/// answer, the one a half-read socket produces, is the one answer that is always +/// accepted. +#[test] +fn a_first_fill_is_never_refused_and_a_truncated_answer_always_is() { + let config = openrouter(); + let of = |count: usize| { + Catalogue::of( + spec(&config), + (0..count) + .map(|index| priced(&format!("vendor/model-{index}"), 1_000_000, 2_000_000)) + .collect(), + "2026-08-27", + ) + }; + + // A first fill. Nothing to compare against, so nothing is refused — including + // the empty answer, which the caller handles separately by saying no prices + // were written rather than by keeping a table that does not exist. + assert!(!of(0).too_short(0), "a first fill has nothing to lose"); + assert!(!of(1).too_short(0)); + assert!(!of(400).too_short(0)); + + // An empty answer against a table with rows in it. Refused at every size, + // including one, which is the case a `> existing / 2` rule would let through. + assert!(of(0).too_short(1), "an empty answer replaced a real table"); + assert!(of(0).too_short(400)); + + // Drastically shorter: refused. Similar: not. The line sits at half, and + // both sides of it are asserted so that moving it fails this rather than + // passing quietly in whichever direction it moved. + assert!( + of(4).too_short(400), + "a truncated read replaced a full table" + ); + assert!(of(199).too_short(400), "under half is a truncation"); + assert!( + !of(200).too_short(400), + "exactly half is allowed: vendors do retire models", + ); + assert!( + !of(380).too_short(400), + "a slightly smaller answer is a normal one" + ); + assert!( + !of(420).too_short(400), + "a larger answer is never a truncation" + ); +} + +// --------------------------------------------------------------------------- +// `Catalogue::edits` — the round trip +// --------------------------------------------------------------------------- + +/// The `[prices]` section a file already carrying prices has. +const EXISTING: &str = "\ +[[provider]] +kind = \"openai\" +model = \"gpt-4.1\" + +[prices] +as_of = \"2026-01-01\" + +[prices.models] +\"gpt-4o\" = { input = 2500000, output = 10000000 } +"; + +/// A file written by the wizard: a provider, io-cli's own section, and no prices +/// at all. This is what a first fill is applied to. +const FRESH: &str = "\ +[[provider]] +kind = \"anthropic\" +model = \"claude-sonnet-4.5\" + +[app.io-cli] +theme = \"dark\" +"; + +/// **A model id with a dot in it survives the write, and the proof is that +/// io-harness prices a call with it afterwards.** +/// +/// A bare TOML key is `A-Za-z0-9_-` and nothing else. `gpt-4.1` written bare is +/// not a key with a dot in it — it is two path segments, `gpt-4` and `1`, and the +/// rate lands in a table nothing reads. Nothing fails: the file parses, the +/// section exists, `Config::prices()` returns a table, and `price("gpt-4.1")` +/// answers `None` forever. +/// +/// So the assertion deliberately does not stop at the text. It applies the edits +/// to a real file, parses the result with io-harness's own loader, and asks the +/// resulting `PriceTable` what the model costs — because the text containing +/// `"gpt-4.1"` and the table answering to `gpt-4.1` are two different claims and +/// only the second one is the feature. +/// +/// Sabotage: drop the `quoted` call in `Catalogue::edits` and spell the path with +/// `format!("prices.models.{model}")`. The file still parses, this test fails on +/// `price`, and nothing else in the repository notices. +#[test] +fn a_dotted_model_id_survives_the_write_and_prices_a_call() { + let config = openai(); + let catalogue = Catalogue::of( + spec(&config), + vec![ + priced("openai/gpt-4.1", 2_000_000, 8_000_000), + priced("openai/gpt-4.1-mini", 400_000, 1_600_000), + priced("openai/o3", 2_000_000, 8_000_000), + ], + "2026-08-27", + ); + + // The flag comes from the file rather than from a literal, because that pair + // is the feature: `has_models_section` asks the document and `edits` chooses + // its shape from the answer, and a test that hard-coded `true` here would pass + // against a reader that always said so. + assert!( + prices::has_models_section(EXISTING), + "the fixture is meant to be the refresh case", + ); + let written = edit::apply( + EXISTING, + &catalogue.edits(prices::has_models_section(EXISTING)), + ) + .expect("the edits produce a configuration file that parses"); + + // The spelling, so a failure here names the cause rather than the symptom. + assert!( + written.contains("\"gpt-4.1\""), + "the dotted id was written bare, which is two keys and not one:\n{written}", + ); + + // The property. io-harness reads the file it would really read, and the table + // it builds answers to the id io-harness would really record on a call. + let reloaded = Config::from_toml(&written).expect("io-harness accepts the written file"); + let table = reloaded.prices().expect("the file carries a price table"); + assert_eq!(table.as_of(), "2026-08-27", "the date did not move"); + assert_eq!( + table.price("gpt-4.1"), + Some(Price { + input: 2_000_000, + output: 8_000_000, + ..Price::ZERO + }), + "the dotted id reached a key io-harness cannot find:\n{written}", + ); + assert!(table.price("gpt-4.1-mini").is_some()); + assert!(table.price("o3").is_some()); + + // **A row the catalogue no longer serves is left alone**, which is the module's + // own rule and not an accident of the splice: io-harness prices a call by the + // model name recorded on it, so the old row is what prices an old run + // correctly and `/cost` reports history as well as today. + assert_eq!( + table.price("gpt-4o"), + Some(Price { + input: 2_500_000, + output: 10_000_000, + ..Price::ZERO + }), + "a model the catalogue stopped serving lost the rate that prices its history", + ); +} + +/// **The first fill: the case every new install takes, against a file that has no +/// `[prices]` section at all.** +/// +/// The test above starts from a file that already carries `[prices.models]`, so +/// every model edit finds a section to insert a key into. That is the *second* +/// fill. The first one — the wizard writes `io.toml`, the credential check +/// passes, `fill_prices` reads the catalogue and writes it — starts from a file +/// with neither `[prices]` nor `[prices.models]` in it, and it is the only path +/// an operator takes on day one. +/// +/// **Every edit in a batch is resolved against the document as it was BEFORE the +/// batch**, which is why the two cases cannot share a shape. `edit::apply` walks +/// the file's headers once and every `set` is answered from that walk — so four +/// hundred `set`s into a `[prices.models]` that does not exist yet each fall +/// through to the append arm and each emit their own `[prices.models]` header. The +/// file gains four hundred definitions of one table, TOML refuses the lot, and the +/// only thing an operator sees is "no prices were written". The first fill could +/// not have worked at all. +/// +/// So a first fill writes the section once, whole, and this test is the one that +/// says so: exactly one header, every model in it, and the whole thing readable +/// back through io-harness. The header count is asserted directly rather than left +/// to the parse, because a duplicate is only *usually* a parse error — a second +/// header carrying keys the first did not would parse on some shapes and silently +/// split the table on others. +/// +/// Sabotage: pass `true` here, or make `has_models_section` answer on the parent +/// `[prices]` rather than on the child — under which this fails with a duplicate +/// table and every other test in this file still passes. +#[test] +fn a_first_fill_writes_every_model_into_a_file_that_has_no_prices_section() { + let config = anthropic(); + let catalogue = Catalogue::of( + spec(&config), + vec![ + priced("anthropic/claude-sonnet-4.5", 3_000_000, 15_000_000), + priced("anthropic/claude-opus-4.1", 15_000_000, 75_000_000), + priced("anthropic/claude-haiku-4.5", 1_000_000, 5_000_000), + ], + "2026-08-27", + ); + assert_eq!(catalogue.rows.len(), 3, "the fixture prices three models"); + assert!( + !prices::has_models_section(FRESH), + "the fixture is meant to be the first-fill case", + ); + + let edits = catalogue.edits(prices::has_models_section(FRESH)); + assert_eq!( + edits.len(), + 2, + "a first fill is the date and the section, however many models it holds", + ); + + let written = edit::apply(FRESH, &edits).unwrap_or_else(|error| { + panic!( + "a first fill of three models produced a file that does not parse, so an operator's \ + very first `/cost` reports tokens and no money: {error}" + ) + }); + + assert_eq!( + written.matches("[prices.models]").count(), + 1, + "the fill wrote the table more than once:\n{written}", + ); + + let reloaded = Config::from_toml(&written).expect("io-harness accepts the written file"); + let table = reloaded.prices().expect("the file carries a price table"); + assert_eq!(table.as_of(), "2026-08-27"); + for (model, price) in &catalogue.rows { + assert_eq!( + table.price(model).as_ref(), + Some(price), + "`{model}` is not in the table the first fill wrote:\n{written}", + ); + } + // Nothing else in the file moved. The provider the operator just configured + // is the one thing a broken splice is most likely to take with it. + assert_eq!( + reloaded.provider_spec(), + config.provider_spec(), + "the write disturbed the provider:\n{written}", + ); + + // **And it is not a special case that only works at three.** Four hundred is + // what an OpenRouter operator's first fill really is, and it is the size at + // which the old shape produced four hundred headers rather than two. + let config = openrouter(); + let many = Catalogue::of( + spec(&config), + (0..400) + .map(|index| priced(&format!("vendor-{index}/model-4.1"), 1_000_000, 2_000_000)) + .collect(), + "2026-08-27", + ); + let written = edit::apply(FRESH, &many.edits(false)) + .expect("four hundred models are one section, not four hundred"); + assert_eq!(written.matches("[prices.models]").count(), 1); + let table = Config::from_toml(&written) + .expect("io-harness accepts it") + .prices() + .expect("the file carries a table"); + assert!( + (0..400).all(|index| table.price(&format!("vendor-{index}/model-4.1")).is_some()), + "a model was lost from a four-hundred-row first fill", + ); +} + +/// **`has_models_section` asks the file, and asks about the child table rather +/// than the parent.** +/// +/// The question `edits` needs answered is whether a `set` will find somewhere to +/// insert a key, and that is a question about `[prices.models]` and nothing else. +/// A `[prices]` carrying an `as_of` and no models is a real state — it is exactly +/// what a refresh whose catalogue priced nothing leaves behind — and answering +/// `true` for it sends the next fill down the `set` path into a section that is +/// not there. +/// +/// The two substitutes that look equivalent are both wrong, and the file is the +/// only thing that is not: io-cli's own record of how many models it last wrote +/// is zero for a `[prices]` an operator typed by hand, and asking the loaded +/// `PriceTable` whether it prices anything this catalogue serves answers `false` +/// for a real section whose models the provider has since replaced. +/// +/// Sabotage: drop the `path.len() == 2` test and match on any path beginning +/// `prices.models` — under which a file whose models are written as +/// `[prices.models."gpt-4.1"]` sub-tables answers `true`, takes the `set` path, +/// and lands its rows in a section that does not exist. +#[test] +fn the_models_section_is_looked_for_in_the_file_and_not_inferred() { + assert!(prices::has_models_section(EXISTING)); + assert!(!prices::has_models_section(FRESH)); + + // A `[prices]` with a date and no models: the state a refresh that priced + // nothing leaves, and the one an inference off the parent gets wrong. + assert!( + !prices::has_models_section("[prices]\nas_of = \"2026-01-01\"\n"), + "a `[prices]` with no models was read as a models section", + ); + + // An unparseable file answers `false`, which sends the caller down the create + // path — where `apply` refuses it with a complaint about the file rather than + // about the prices, which is the truthful one. + assert!(!prices::has_models_section("[prices\nas_of =")); +} + +/// **On a refresh it is one edit per model, because a rate is something an +/// operator is invited to correct by hand.** +/// +/// The alternative — one edit rewriting `[prices.models]` whole, which is what the +/// first fill does because it must — produces the same file and a very different +/// consequence. `Edit::section` replaces nothing and refuses a section that +/// exists, precisely so that a refresh cannot take this path: a whole-section +/// write discards every row nobody named, which on a refresh is every model the +/// catalogue stopped serving *and* every rate the operator corrected by hand. +/// io-harness prices a call by the model name recorded on it, so those old rows +/// are exactly what price an old run correctly, and `/cost` reports history as +/// well as today. +/// +/// The unpriced model is the other half: it contributes no edit at all, so a +/// model the catalogue served without a rate never reaches the file as a zero. +/// +/// Sabotage: collect the rows into one `Edit::section` on the refresh path too — +/// under which the first `/config` refresh silently deletes every hand-corrected +/// rate and every model the provider retired, and the file still parses. +#[test] +fn a_refresh_writes_one_row_per_model_and_leaves_the_rest_alone() { + let config = openrouter(); + let catalogue = Catalogue::of( + spec(&config), + vec![ + priced("a/one", 1_000_000, 2_000_000), + priced("a/two", 1_000_000, 2_000_000), + unpriced("a/three"), + ], + "2026-08-27", + ); + + let edits = catalogue.edits(true); + assert_eq!( + edits.len(), + 1 + catalogue.rows.len(), + "the date plus one row per PRICED model, and nothing for the unpriced one", + ); + assert_eq!(edits[0].path(), "prices.as_of"); + assert!( + edits[1..] + .iter() + .all(|edit| edit.path().starts_with("prices.models.")), + "a refresh edit addresses something other than one model's row", + ); + + // And `Edit::section` refuses the file that already has one, which is what + // stops the two paths being interchangeable by accident. + let refused = edit::apply(EXISTING, &catalogue.edits(false)) + .expect_err("a whole-section write over an existing section must be refused"); + assert!( + refused.contains("already in this file"), + "the refusal does not say why it refused: {refused}", + ); + + // A model with every dimension free still round-trips as a `Price` rather + // than being dropped: `{}` is the inline table that says exactly that, and a + // free model is a fact the table can state where an unpriced one is not. + let free = Catalogue::of( + spec(&config), + vec![ModelInfo { + id: "a/free".into(), + price: Some(Price::ZERO), + price_source: Some(PriceSource::Vendor), + ..Default::default() + }], + "2026-08-27", + ); + let written = edit::apply(EXISTING, &free.edits(true)).expect("a free model is writable"); + let table = Config::from_toml(&written) + .expect("io-harness accepts it") + .prices() + .expect("the file carries a table"); + assert_eq!( + table.price("a/free"), + Some(Price::ZERO), + "a model that costs nothing is priced at nothing, which is not the same as unpriced", + ); +} + +// --------------------------------------------------------------------------- +// `changes` +// --------------------------------------------------------------------------- + +/// **A refresh lists what moved and nothing else, and a model with no row yet is +/// `was: None` rather than a change from zero.** +/// +/// The distinction is the difference between two sentences an operator reads +/// differently: "this got cheaper, from nothing" is nonsense, and "this is new" +/// is the answer. A `was` defaulted to `Price::ZERO` would render every arrival +/// as a price rise from free, which on a four-hundred-model catalogue turns the +/// first refresh after an upstream addition into four hundred rows of noise with +/// the real change buried in it. +/// +/// The other half is the silence. A rate that has not moved is not a change, so a +/// refresh that found nothing says so in one line rather than reprinting the +/// whole table — and an operator who has been shown four hundred unchanged rows +/// once will not read the list the time it matters. +/// +/// Sabotage: compare on the model id rather than on the price, and every refresh +/// lists every model forever. +#[test] +fn changes_lists_only_what_moved_and_calls_an_unpriced_model_new() { + let config = openrouter(); + let fresh = Catalogue::of( + spec(&config), + vec![ + // Moved. + priced("a/dearer", 4_000_000, 16_000_000), + // Unchanged, to the micro-unit. + priced("a/steady", 1_000_000, 2_000_000), + // Not in the table at all. + priced("a/arrived", 500_000, 1_500_000), + ], + "2026-08-27", + ); + + let existing = PriceTable::new("2026-01-01") + .with( + "a/dearer", + Price { + input: 3_000_000, + output: 12_000_000, + ..Price::ZERO + }, + ) + .with( + "a/steady", + Price { + input: 1_000_000, + output: 2_000_000, + ..Price::ZERO + }, + ); + + let moved = prices::changes(Some(&existing), &fresh); + let names: Vec<&str> = moved.iter().map(|change| change.model.as_str()).collect(); + // **Alphabetical, because `Catalogue::rows` is and `changes` preserves it** — + // which is a property worth pinning rather than working around. A refresh + // report an operator reads twice should list the same rates in the same order + // both times, and catalogue order is whatever a remote server felt like. + assert_eq!( + names, + ["a/arrived", "a/dearer"], + "an unchanged rate was listed, a changed one was not, or the order is not stable", + ); + + // Found by name rather than by index, so a future ordering change fails the + // assertion above — which is about order — instead of silently moving these, + // which are about values. + let by = |model: &str| { + moved + .iter() + .find(|change| change.model == model) + .unwrap_or_else(|| panic!("`{model}` is not in {names:?}")) + }; + + let dearer = by("a/dearer"); + assert_eq!(dearer.was.map(|price| price.input), Some(3_000_000)); + assert_eq!(dearer.now.input, 4_000_000); + + let arrived = by("a/arrived"); + assert_eq!( + arrived.was, None, + "a model the table has never priced is new, not a change from zero", + ); + assert_eq!(arrived.now.output, 1_500_000); + + // Nothing moved: nothing listed. Including the case an operator hits most + // often, which is running `/config` twice in a day. + assert!( + prices::changes( + Some(&existing), + &Catalogue::of( + spec(&config), + vec![priced("a/steady", 1_000_000, 2_000_000)], + "2026-08-27" + ) + ) + .is_empty(), + "an unchanged table reported a change", + ); + + // No table at all: every row is new. The first fill, from the other side. + let first = prices::changes(None, &fresh); + assert_eq!(first.len(), 3); + assert!( + first.iter().all(|change| change.was.is_none()), + "a model was reported as having had a rate in a table that does not exist", + ); +} + +// --------------------------------------------------------------------------- +// `source_word` +// --------------------------------------------------------------------------- + +/// **The sentence under every `$` on every page, and it names the catalogue +/// rather than the connected provider.** +/// +/// An operator on Anthropic is being shown numbers about Anthropic that Anthropic +/// never published. Flattening that into "Anthropic's prices" would attribute a +/// figure to a vendor who did not state it, which is the one claim this whole +/// module exists to avoid making. So the reference form carries the URL — the +/// operator can go and read the source — and the vendor form does not, because +/// there is nothing to point at that is not the provider they are already +/// connected to. +/// +/// The `_` arm is required rather than defensive: `PriceSource` is +/// `#[non_exhaustive]`, so io-harness can add a variant this release has never +/// seen, and the honest answer for one is that io-cli does not recognise it — not +/// a name io-cli invented for it. +/// +/// Sabotage: make the `_` arm say "the provider's own catalogue", and a future +/// harness variant is silently attributed to the vendor. +#[test] +fn source_word_names_the_catalogue_and_never_the_connected_provider() { + let vendor = prices::source_word(&PriceSource::Vendor); + assert_eq!(vendor, "the provider's own catalogue"); + assert!( + !vendor.contains("http"), + "the vendor form points at a URL the operator did not need", + ); + + let reference = prices::source_word(&PriceSource::Reference(DEFAULT_REFERENCE_URL.to_string())); + assert!( + reference.contains(DEFAULT_REFERENCE_URL), + "the reference form does not say which reference: {reference}", + ); + assert!( + reference.contains("reference"), + "the hedge is the word, not the URL: {reference}", + ); + + // The URL is io-harness's own constant re-exported, not a second copy of it + // in this repository — which is what makes it move when the harness moves it. + assert_eq!( + DEFAULT_REFERENCE_URL, + io_harness::provider::catalog::DEFAULT_REFERENCE_URL, + ); +} + +// --------------------------------------------------------------------------- +// The filter, and why it is applied exactly once +// --------------------------------------------------------------------------- + +/// **`verify::named` is not idempotent, and that is why `priced` no longer +/// filters.** +/// +/// The two used to be one call each, and both filtered: `Catalogue::of` handed the +/// raw catalogue to `named` to get its count and the raw catalogue to `priced` to +/// get its rows, and `priced` filtered again internally. Applied twice to an +/// Anthropic catalogue the second pass finds no `anthropic/` prefix left to strip +/// — every id has already lost it — and `strip`'s `strip_prefix(prefix)?` discards +/// the row rather than keeping it. The result is a table with nothing in it, from +/// a catalogue that priced fifteen models, with nothing failing anywhere. +/// +/// This test states the non-idempotence directly rather than trusting the call +/// sites, because the property is what makes the call sites' shape mandatory: a +/// future edit that "tidies up" by filtering inside `priced` again is not a +/// duplicate of work already done, it is an erasure. +/// +/// The OpenRouter arm is the control. Its filter is the identity, so applying it +/// twice changes nothing — which is exactly why a test written only against +/// OpenRouter would have passed against the broken shape. +/// +/// Sabotage: put the `named` call back inside `verify::priced` — under which this +/// fails on the Anthropic arm and `Catalogue::of` starts returning an empty table +/// for both providers whose prefix it strips. +#[test] +fn the_catalogue_filter_is_applied_exactly_once_because_it_is_not_idempotent() { + let config = anthropic(); + let raw = vec![ + priced("anthropic/claude-sonnet-4.5", 3_000_000, 15_000_000), + priced("openai/gpt-4.1", 2_000_000, 8_000_000), + ]; + + let once = io_cli::verify::named(spec(&config), raw.clone()); + assert_eq!(once.len(), 1, "one of the two is Anthropic's"); + assert_eq!(once[0].id, "claude-sonnet-4.5", "the prefix is stripped"); + + let twice = io_cli::verify::named(spec(&config), once.clone()); + assert!( + twice.is_empty(), + "the filter is idempotent after all, and this test is asserting nothing: {:?}", + twice.iter().map(|model| &model.id).collect::>(), + ); + + // So `priced` takes rows that have already been named and does not filter + // again — which is the only shape under which `Catalogue::of` can both count + // and price the same set. + let rows = io_cli::verify::priced(once); + assert_eq!(rows.len(), 1, "pricing filtered a second time"); + assert_eq!(rows[0].0, "claude-sonnet-4.5"); + + // The control. OpenRouter's filter is the identity, so it survives being + // applied twice — and a test written only against it would have passed + // against the shape that lost every Anthropic row. + let config = openrouter(); + let once = io_cli::verify::named(spec(&config), raw.clone()); + let twice = io_cli::verify::named(spec(&config), once.clone()); + assert_eq!( + once.len(), + twice.len(), + "OpenRouter's filter is the identity and has to survive being applied twice", + ); + assert_eq!(twice.len(), 2); +} + +/// **`quoted` is the module's only speller, and it escapes rather than decorates.** +/// +/// Model ids carry dots, slashes and colons, and a dot in a TOML key can only be +/// spelled quoted — a bare key is `A-Za-z0-9_-` and nothing else. So this is not +/// presentation: an unquoted `gpt-4.1` is two path segments and reaches the wrong +/// place, or no place. It is spelled through the `toml` crate rather than with a +/// `format!` for the case a hand-written quote gets wrong, which is a value +/// containing a quote or a backslash — and a model id from a catalogue is +/// somebody else's string. +/// +/// Sabotage: `format!("\"{text}\"")` — which passes for every id anyone has ever +/// seen and produces a file that does not parse the first time a catalogue serves +/// one with a quote in it. +#[test] +fn a_model_id_is_spelled_by_the_toml_crate_and_not_by_a_format_string() { + assert_eq!(prices::quoted("gpt-4.1"), "\"gpt-4.1\""); + assert_eq!( + prices::quoted("anthropic/claude-sonnet-4.5"), + "\"anthropic/claude-sonnet-4.5\"" + ); + + // The two a hand-written pair of quotes gets wrong. Both round-trip through + // the parser rather than being compared to a spelling typed here, because the + // escape rules are the `toml` crate's and this file should not have a second + // opinion about them. + for id in ["a\"b", "back\\slash", "new\nline"] { + let spelled = prices::quoted(id); + let parsed = toml::from_str::(&format!("probe = {spelled}")) + .unwrap_or_else(|error| { + panic!("`{id}` was spelled as {spelled}, which is not TOML: {error}") + }); + assert_eq!( + parsed.get("probe").and_then(|value| value.as_str()), + Some(id), + "`{id}` was spelled as {spelled}, which parses back as something else", + ); + } +} diff --git a/tests/stats.rs b/tests/stats.rs new file mode 100644 index 0000000..2d00cff --- /dev/null +++ b/tests/stats.rs @@ -0,0 +1,642 @@ +//! How the work has gone: `/stats`, and the four places it refuses to average. +//! +//! **Every figure on this page is a `Store` call io-cli had never made.** The +//! aggregates have been in io-harness since 0.30.0 — `runs_by_outcome`, +//! `runs_by_day`, `first_try`, `gate_failures_by_phase`, `recovery` and +//! `store_size` — and the interface over them reported none of it. Reading them is +//! the easy half; the hard half is the four questions the harness deliberately +//! refuses to answer, because each of them is a place where a page can quietly +//! answer a different question from the one its heading asks. +//! +//! `FirstTry` is three counts and not a rate, because the denominator is a +//! choice. `gate_failures_by_phase` covers one of two gate mechanisms, whose phase +//! vocabularies do not overlap. `ttft_ms` is `None` for a call nothing measured, +//! never zero. And `StoreSize::tables` is empty on a SQLite build without +//! `dbstat`, which is a build that cannot answer rather than a database whose +//! tables are all empty. Every one of those has an obvious wrong rendering that +//! reads perfectly well, and this file is mostly about those four. +//! +//! The store is real and driven through io-harness's own recording calls. A page +//! asserted against hand-built aggregate structs would be asserting its own format +//! strings; here the numbers come out of SQL that ran. + +use io_harness::{ + CheckpointEvent, ContextEvent, GateOutcome, ProviderCall, SandboxEvent, Store, Usage, +}; + +use io_cli::glyphs::ASCII; +use io_cli::stats::{self, RECENT_RUNS}; +use io_cli::theme::{Theme, DARK}; + +// --------------------------------------------------------------------------- +// Fixtures +// --------------------------------------------------------------------------- + +/// Wide enough that nothing folds, so a row lookup finds a whole row. +const ROOMY: u16 = 200; + +fn ascii() -> Theme { + DARK.with_glyphs(ASCII) +} + +fn call(model: &str, latency_ms: u64, ttft_ms: Option) -> ProviderCall { + ProviderCall { + step: 1, + provider: "anthropic".into(), + model: Some(model.to_string()), + usage: Some(Usage { + prompt_tokens: 900, + completion_tokens: 100, + total_tokens: 1_000, + ..Default::default() + }), + latency_ms, + ttft_ms, + ..Default::default() + } +} + +/// A store with a history: runs that ended four ways, both gate mechanisms +/// having failed, all three recovery mechanisms having fired, and provider calls +/// with a mixture of measured and unmeasured first tokens. +/// +/// Everything is recorded through the call io-harness's own documentation shows, +/// so the aggregates below are the aggregates a real run would produce rather +/// than rows this file inserted a shape it invented. +fn history() -> Store { + let store = Store::memory().expect("an in-memory store"); + + // Three runs that succeeded, one of which failed a gate on the way, and one + // that stalled. `first_try` is finished AND successful AND carrying no + // `gate_phase_failed` event, so this fixture is 5 finished, 3 succeeded, + // 2 first try — three different numbers, which is what makes a page that + // picked the wrong one visible. + for outcome in ["success", "success", "stalled", "step_cap_reached"] { + let run = store + .start_run("summarise the module", "/repo") + .expect("a run"); + store.finish_run(run, outcome).expect("the run finishes"); + } + + let retried = store.start_run("port the parser", "/repo").expect("a run"); + // **The sandbox gate's vocabulary.** Four phases, none of which overlaps the + // review/command/contains set below. + store + .record_sandbox_event(&SandboxEvent::gate_phase_failed(retried, 2, "test-run")) + .expect("the phase failure is recorded"); + store + .record_sandbox_event(&SandboxEvent::gate_phase_failed( + retried, + 4, + "criterion-compile", + )) + .expect("the phase failure is recorded"); + // **The other gate's vocabulary, recorded in another table entirely.** These + // must not reach the phase list: they are a different mechanism, and merging + // them would produce a list whose categories mean two things. + for phase in ["review", "command", "contains"] { + store + .put_gate_attempt(retried, 3, phase, GateOutcome::Failed, "did not hold") + .expect("the gate attempt is recorded"); + } + store.finish_run(retried, "success").expect("it finishes"); + + // All three recovery mechanisms, one each, so a page that reported the wrong + // counter cannot pass by coincidence. + store + .record_context_event(retried, &ContextEvent::served(1, "anthropic")) + .expect("the fallback is recorded"); + store + .record_context_event(retried, &ContextEvent::replan(3, "no progress")) + .expect("the replan is recorded"); + store + .record_checkpoint_event(&CheckpointEvent::resume(retried, 4, "after a crash")) + .expect("the resume is recorded"); + + // Five calls, three of which measured a first token. The two that did not are + // the whole of the `None`-is-not-zero case. + for (model, latency, ttft) in [ + ("claude-sonnet-4.5", 1_200u64, Some(300u64)), + ("claude-sonnet-4.5", 4_213, Some(900)), + ("claude-opus-4.1", 800, Some(100)), + ("claude-opus-4.1", 2_500, None), + ("claude-haiku-4.5", 150, None), + ] { + store + .record_provider_call(retried, &call(model, latency, ttft)) + .expect("the call is recorded"); + } + + store +} + +/// The `/stats` page as a reader sees it: every row, spans concatenated. +fn page(store: &Store) -> Vec { + stats::committed(store, &ascii(), ROOMY) + .expect("the page draws") + .iter() + .map(|line| { + line.spans + .iter() + .map(|span| span.content.as_ref()) + .collect::() + }) + .collect() +} + +fn text(rows: &[String]) -> String { + rows.join("\n") +} + +/// The rows of one section: everything after its heading, up to the blank row +/// that separates it from the next. +/// +/// **Sections are the whole point of `Row::Heading` existing**, and a test that +/// searched the whole page for a word could not tell a figure under the right +/// heading from the same figure under the wrong one — which is exactly the +/// failure the two gate vocabularies would produce. +fn section<'a>(rows: &'a [String], heading: &str) -> &'a [String] { + let at = rows + .iter() + .position(|row| row == heading) + .unwrap_or_else(|| panic!("no `{heading}` heading on the page:\n{}", text(rows))); + let rest = &rows[at + 1..]; + let end = rest + .iter() + .position(|row| row.is_empty()) + .unwrap_or(rest.len()); + &rest[..end] +} + +/// The value of the row reading `label: …` inside `rows`. +fn field<'a>(rows: &'a [String], label: &str) -> &'a str { + let prefix = format!("{label}: "); + rows.iter() + .find_map(|row| row.trim_start().strip_prefix(&prefix)) + .unwrap_or_else(|| panic!("no `{label}` row here:\n{}", text(rows))) +} + +/// Every heading the page draws, in the order it draws them. +/// +/// A heading is indented to zero and every other row to two, which is what makes +/// this readable off the rendered rows rather than off an internal structure — and +/// it is also the property a reader in `--plain` depends on. +fn headings(rows: &[String]) -> Vec<&str> { + rows.iter() + .skip(1) + .take(rows.len().saturating_sub(2)) + .filter(|row| !row.is_empty() && !row.starts_with(' ')) + .map(String::as_str) + .collect() +} + +// --------------------------------------------------------------------------- +// The page +// --------------------------------------------------------------------------- + +/// **Every section draws, over a store with a real history in it.** +/// +/// Eight lists on one page, and each is a separate `Store` read that can fail on +/// its own. A page that lost one would still render, still be edged, and still +/// answer seven questions — so the assertion is on the whole list of headings in +/// order rather than on any one of them, because that is the only form that +/// notices a section quietly leaving. +/// +/// The sample bound is in the heading and not merely applied. `/cost` could not +/// bound its grouped reads honestly and says so in a comment; this page can, +/// because `Store::runs` returns ids and io-cli takes the tail itself — so the +/// sample really is the last N runs and the heading says which N. That is the +/// difference between a bound and a truncation: "the last 200 runs" is a true +/// statement about a subset where "by model" over a truncated read would be a +/// false statement about the whole. +/// +/// Sabotage: drop any one `rows.push(Row::heading(…))` — under which the figures +/// beneath it join the section above and read as answers to its question. +#[test] +fn every_section_of_the_page_draws_with_its_scope_in_the_heading() { + let store = history(); + let rows = page(&store); + + assert_eq!( + headings(&rows), + [ + "runs by outcome", + "runs by day", + "finished on the first try", + "gate failures by phase", + "recovery", + // Five runs in the store, all of them inside the bound. + "slowest calls, of the last 5 runs", + "time to first token", + "what the store holds", + ], + "a section left the page, or arrived under a name that does not say its \ + scope:\n{}", + text(&rows), + ); + + // The bound the heading names is the module's own constant, so a release that + // moves it moves both together and this stays a claim about honesty rather + // than about the number two hundred. + assert_eq!(RECENT_RUNS, 200); + + // Outcomes are the raw strings io-harness recorded, not a success/failure + // collapse: "stalled" and "step_cap_reached" are different endings and the + // distinction is the reason an operator is reading this page. + let outcomes = section(&rows, "runs by outcome"); + assert_eq!(field(outcomes, "success"), "3"); + assert_eq!(field(outcomes, "stalled"), "1"); + assert_eq!(field(outcomes, "step_cap_reached"), "1"); + + // Recovery: three counters, one each, so a page reading the wrong one cannot + // pass by coincidence. + let recovery = section(&rows, "recovery"); + assert_eq!(field(recovery, "fallbacks"), "1"); + assert_eq!(field(recovery, "replans"), "1"); + assert_eq!(field(recovery, "resumes"), "1"); + + // The slowest call is named by its model and its latency, in that order, + // because the question is which model is slow. + let slowest = section(&rows, "slowest calls, of the last 5 runs"); + assert_eq!(field(slowest, "claude-sonnet-4.5"), "4213 ms"); + assert!( + slowest.len() <= 5, + "the slow list is unbounded: {slowest:?}", + ); +} + +/// **The first-try share names its denominator in the row, rather than printing a +/// bare percentage.** +/// +/// io-harness returns three counts and deliberately not a rate, and it says why: +/// *first_try / succeeded* is "when we got there, how often first time" and +/// *first_try / runs* is "how often does this work at all", and both are +/// legitimate questions with different answers. Returning one number would be +/// picking for the reader and hiding which was picked. +/// +/// So the page picks one and says which. A row reading `73%` is a number two +/// readers reconstruct differently — one of them will read it as a share of every +/// run, including the ones still going — and they will disagree about whether the +/// agent got better this week. +/// +/// The fixture makes the three counts three different numbers on purpose. A +/// fixture where every run succeeded first time would give the same percentage +/// under either denominator, and the test would pass against the wrong one. +/// +/// Sabotage: `format!("{}%", first.first_try * 100 / first.runs)` with the label +/// left as it is — under which the row says "share of successful runs" and shows +/// a share of every run, which is the failure a bare percentage makes invisible. +#[test] +fn the_first_try_share_names_its_denominator_rather_than_printing_a_bare_percentage() { + let store = history(); + let rows = page(&store); + let first = section(&rows, "finished on the first try"); + + // The three counts io-harness gives, each under a label that says what it + // counts. Five runs finished, three of them succeeded, two of those had no + // gate failure — three different numbers, from one fixture. + assert_eq!(field(first, "runs finished"), "5"); + assert_eq!(field(first, "of those, succeeded"), "3"); + assert_eq!(field(first, "of those, with no gate failure"), "2"); + + // The share, and the label carries the denominator rather than the reader + // having to supply one. Two of three is sixty-six percent; two of five would + // be forty, which is the number the wrong denominator produces. + assert_eq!( + field(first, "first-try share of successful runs"), + "66%", + "the share is not two of the three runs that succeeded:\n{}", + text(&rows), + ); + assert!( + !first.iter().any(|row| row.trim() == "66%"), + "the percentage is drawn without a label saying what it is a share of: {first:?}", + ); + + // A store where nothing has succeeded yet has no share to draw, and draws + // none rather than dividing by zero or reporting a confident nought percent. + let empty = Store::memory().expect("an in-memory store"); + let run = empty.start_run("summarise", "/repo").expect("a run"); + empty.finish_run(run, "stalled").expect("it finishes"); + let rows = page(&empty); + let first = section(&rows, "finished on the first try"); + assert_eq!(field(first, "of those, succeeded"), "0"); + assert!( + !first.iter().any(|row| row.contains('%')), + "a share was drawn over a denominator of zero: {first:?}", + ); +} + +/// **The two gate vocabularies stay under separate headings, because they are two +/// mechanisms and not two spellings.** +/// +/// io-harness records gate failures twice, in two tables, with two phase +/// vocabularies that do not overlap. `SandboxEvent` phases are `subject-compile`, +/// `criterion-compile`, `test-run` and `subject-emptied`; `GateAttempt` phases are +/// `review`, `command` and `contains`. Merged into one list they would produce a +/// chart whose categories mean two different things, and the operator reading +/// "gate failures by phase" would be adding a compile failure to a review +/// disagreement. +/// +/// The fixture has both, failing, at the same time — which is the only way to +/// state that the page keeps them apart rather than that the page has only ever +/// seen one of them. The note is the other half: a reader who came looking for +/// their failing `review` gate has to be told where it is not, or they will read +/// its absence as "the review gate has never failed". +/// +/// Sabotage: `UNION ALL` the two tables behind `gate_failures_by_phase`, or add a +/// second `tallies` call over the gate attempts under the same heading — under +/// which the counts still add up, the page still renders, and this is the only +/// test that fails. +#[test] +fn the_two_gate_vocabularies_stay_under_separate_headings() { + let store = history(); + let rows = page(&store); + let gates = section(&rows, "gate failures by phase"); + + assert_eq!(field(gates, "test-run"), "1"); + assert_eq!(field(gates, "criterion-compile"), "1"); + + // The other mechanism's phases are not rows here. Checked as rows rather than + // as words, because the note below deliberately names all three in prose. + for phase in ["review", "command", "contains"] { + assert!( + gates + .iter() + .all(|row| !row.trim_start().starts_with(format!("{phase}: ").as_str())), + "`{phase}` is a gate attempt and is counted under the sandbox gate's \ + phases: {gates:?}", + ); + } + + // And the reader is told where it went, in the section itself rather than in + // a footnote somewhere else — because the question is asked while looking at + // this list. + let note = text(gates); + assert!( + note.contains("review, command and contains"), + "the section does not say which gate's phases these are not:\n{note}", + ); + assert!( + note.contains("not merged into this list"), + "the section does not say the other gate records separately:\n{note}", + ); +} + +/// **An unmeasured time to first token is `None`, and it is counted apart rather +/// than averaged in as a zero.** +/// +/// A zero would drag every figure on the section toward an instant nothing ever +/// took, and it would do it worst on exactly the paths that do not stream — which +/// are the paths an operator is most likely to be diagnosing when they open this +/// page. So the unmeasured calls are excluded from the statistics and the count of +/// what was measured is drawn beside them, which is what makes the median +/// interpretable at all. +/// +/// The median is computed here from the seeded latencies rather than read off the +/// page and asserted against itself. +/// +/// Sabotage: `call.ttft_ms.unwrap_or(0)` in the collector — under which the median +/// of this fixture drops from 300 ms to 100 ms, the slowest is unchanged, and +/// nothing says two of the five calls were never measured. +#[test] +fn an_unmeasured_time_to_first_token_is_counted_apart_and_never_averaged_as_zero() { + let store = history(); + let rows = page(&store); + let ttft = section(&rows, "time to first token"); + + // The three measured values, sorted, and the median io-harness's own data + // would give: the middle of what was measured, not the middle of what was + // called. + let measured = [100u64, 300, 900]; + assert_eq!( + field(ttft, "median"), + format!("{} ms", measured[measured.len() / 2]), + "the median counted the calls that measured nothing:\n{}", + text(&rows), + ); + assert_eq!( + field(ttft, "slowest"), + format!("{} ms", measured[measured.len() - 1]), + ); + assert_eq!( + field(ttft, "measured on"), + "3 of 5 calls", + "the page does not say how much of the sample it is speaking for", + ); + // **A whole figure of `0 ms`, not the substring.** `text(ttft).contains("0 ms")` + // was the first spelling of this and it is a false positive on every measured + // value that happens to end in a zero: `300 ms` and `100 ms` both contain it. + // The claim is that no *figure* is zero, so the test is on the value of a row. + assert!( + !ttft.iter().any(|row| row.trim_end().ends_with(": 0 ms")), + "an unmeasured first token reached the figures as an instant one:\n{}", + text(ttft), + ); + + // A store whose calls measured nothing says so in words rather than drawing a + // median of nought. + let store = Store::memory().expect("an in-memory store"); + let run = store.start_run("summarise", "/repo").expect("a run"); + store + .record_provider_call(run, &call("claude-sonnet-4.5", 900, None)) + .expect("recorded"); + let rows = page(&store); + let ttft = section(&rows, "time to first token"); + assert!( + text(ttft).contains("unmeasured rather than instant"), + "a sample that measured nothing drew a figure anyway: {ttft:?}", + ); + assert!( + !text(ttft).contains("median"), + "a median was drawn over nothing: {ttft:?}", + ); +} + +/// **A SQLite build that cannot break the file down by table says so, rather than +/// drawing the breakdown as zeroes or omitting it in silence.** +/// +/// `dbstat` is a virtual table over the b-tree pages and it is compiled in or it +/// is not. io-harness returns an empty table list on a build without it, and it +/// says why in a comment: that is a build that cannot answer the question, not a +/// database whose tables are all empty. The two are indistinguishable in the data +/// and completely different facts. +/// +/// **Which branch runs here is a property of the SQLite this crate happens to +/// link, not of io-cli**, so the test asserts the mapping in both directions +/// rather than picking one and pretending the other cannot happen: a breakdown +/// present means no excuse drawn, and a breakdown absent means the excuse drawn. +/// Asserting one branch would go green on the developer's machine and say nothing +/// about CI — where the SQLite is built by a different job on a different +/// platform, which is exactly where this would differ. +/// +/// The file's own figures stand either way, which is the other half of io-harness's +/// argument for not failing the whole size call over it. +/// +/// Sabotage: draw the table rows unconditionally — under which a build without +/// `dbstat` shows a `what the store holds` section with the size, the free space +/// and nothing at all where the breakdown should be, and no sentence saying which +/// of "there are no tables" and "I cannot see the tables" is true. +#[test] +fn an_absent_table_breakdown_reads_as_cannot_break_down_rather_than_as_zeroes() { + let store = history(); + let size = store.store_size().expect("the store reports its size"); + let rows = page(&store); + let held = section(&rows, "what the store holds"); + let drawn = text(held); + + // The figures that do not depend on `dbstat` are there either way. + assert_eq!(field(held, "sessions"), "0", "no session was opened"); + assert_eq!(field(held, "runs"), "5"); + assert!( + !field(held, "file").starts_with("0 "), + "a store with five runs in it reported no file at all: {drawn}", + ); + + let excuse = "cannot break the file down by table"; + if size.tables.is_empty() { + assert!( + drawn.contains(excuse), + "this build has no `dbstat` and the page does not say so:\n{drawn}", + ); + assert!( + !held.iter().any(|row| row.contains("provider_calls")), + "a breakdown was drawn from a table list that is empty: {held:?}", + ); + } else { + assert!( + !drawn.contains(excuse), + "this build has `dbstat` and the page claims it does not:\n{drawn}", + ); + // Largest first, and no more than five, so the section stays a summary. + let (largest, _) = &size.tables[0]; + assert!( + drawn.contains(largest.as_str()), + "the largest table is not in the breakdown:\n{drawn}", + ); + assert!( + size.tables + .iter() + .take(5) + .all(|(name, _)| drawn.contains(name.as_str())), + "the breakdown is shorter than the five rows it promises:\n{drawn}", + ); + } +} + +/// **Nothing on this page reclaims anything, and the free figure says so.** +/// +/// `Store::compact` is a full `VACUUM`: it needs free disk roughly equal to the +/// database file, it rewrites the whole thing, and it is not quick. A page that +/// reported free space and then reclaimed it on its own would be a page that +/// surprised somebody mid-session — and this one is opened while a turn may be in +/// flight. +/// +/// The sentence is only drawn when there is something to reclaim, because a page +/// that explained a decision about nothing every time it was opened would be a +/// page nobody finishes reading. +/// +/// Sabotage: call `Store::compact` from this page, or drop the sentence and leave +/// the figure — under which an operator reads a number with no idea whether it is +/// about to move. +#[test] +fn the_page_reports_free_space_and_does_not_offer_to_reclaim_it() { + let store = history(); + let before = store.store_size().expect("a size"); + let rows = page(&store); + let held = section(&rows, "what the store holds"); + + // **Named units, and the disjunction that used to stand here was no test at + // all**: `ends_with(..) || !is_empty()` is satisfied by the second half for + // every non-empty string, so a figure rendered as a bare byte count would have + // passed it. `stats::bytes` renders exactly four units and this asserts on + // that set, so a figure that lost its suffix fails here rather than on a + // reader's screen. + let free = field(held, "free inside it"); + assert!( + ["B", "KiB", "MiB", "GiB"] + .iter() + .any(|unit| free.ends_with(unit)), + "the free figure has no unit: {free:?} in {held:?}", + ); + if before.free_bytes > 0 { + assert!( + text(held).contains("is not done from this page"), + "free space was reported with no word about what would reclaim it: {held:?}", + ); + } + + // Reading the page changed nothing about the store. The figures a second read + // gives are the figures the first one gave. + let after = store.store_size().expect("a size"); + assert_eq!( + (before.file_bytes, before.free_bytes, before.runs), + (after.file_bytes, after.free_bytes, after.runs), + "drawing the page moved the database it was reporting on", + ); +} + +/// **An empty store draws every section and reports nothing, rather than a row of +/// zeroes.** +/// +/// The distinction every counter in this product is held to: "no run has been +/// recorded" and "every run scored zero" are different facts, and a page that +/// drew the second when the first was true would be reporting a corpus that does +/// not exist. It is also the state a brand new install is in, which is to say the +/// first thing anybody sees. +/// +/// The page still has all eight sections. An empty state that quietly dropped its +/// headings would leave a first-time reader with no idea what the page is for. +/// +/// Sabotage: return `Vec::new()` from `tallies` on an empty list rather than a +/// note — under which the headings sit above nothing at all and read as sections +/// that failed rather than as questions with no answer yet. +#[test] +fn an_empty_store_says_nothing_is_recorded_rather_than_drawing_zeroes() { + let store = Store::memory().expect("an in-memory store"); + let rows = page(&store); + let drawn = text(&rows); + + assert_eq!( + headings(&rows).len(), + 8, + "an empty store lost a section:\n{drawn}", + ); + // Zero runs, so the sample is zero runs and the heading says so in the + // singular-aware form rather than reading "the last 0 run". + assert!( + headings(&rows).contains(&"slowest calls, of the last 0 runs"), + "the sample heading does not survive an empty store: {:?}", + headings(&rows), + ); + + for heading in ["runs by outcome", "runs by day"] { + let empty = section(&rows, heading); + assert_eq!( + empty.len(), + 1, + "`{heading}` drew more than a note: {empty:?}" + ); + assert_eq!( + empty[0], " no runs recorded", + "`{heading}` drew a zero where it has nothing to report", + ); + } + assert!( + text(section(&rows, "gate failures by phase")).contains("no gate has failed"), + "an empty store reported gate failures:\n{drawn}", + ); + assert!( + text(section(&rows, "slowest calls, of the last 0 runs")) + .contains("no provider calls in those runs"), + "an empty store drew a slow list:\n{drawn}", + ); + + // The counters that genuinely are zero are still drawn as zero: nobody has + // fallen back, replanned or resumed, and that is a measured nought rather + // than an absence. + let recovery = section(&rows, "recovery"); + assert_eq!(field(recovery, "fallbacks"), "0"); + assert_eq!(field(recovery, "replans"), "0"); + assert_eq!(field(recovery, "resumes"), "0"); +} diff --git a/tests/status.rs b/tests/status.rs index 7494c1a..6947ecb 100644 --- a/tests/status.rs +++ b/tests/status.rs @@ -2015,3 +2015,320 @@ fn n3_the_queue_depth_survives_forgetting_the_run() { ); } } + +/// A session at the width and in the state the 0.21.0 live capture caught. +/// +/// Nothing here is invented for the test. The posture and the containment word +/// are the pair `Status::fields` names in its own comment as the real thing — +/// `workspace-write/macos-sandbox-exec`, thirty-four characters — and the +/// counters are the ones the footer had when the last turn ended: six steps, +/// twenty-six point nine thousand tokens, twenty-three percent of the window, +/// and the idle key hint. That row is forty-six characters, the right-hand +/// group is fifty-seven, and a hundred columns cannot hold both. +fn full_row() -> Status { + let mut status = Status::new("anthropic/claude-sonnet-4.5"); + status.policy = Some("read-only".into()); + status.containment = Some("workspace-write/macos-sandbox-exec".into()); + status.planning = true; + status.steps = Some(6); + status.tokens = Some(26_900); + status.context = Some(23); + status +} + +/// **F4 — a counts row that will not fit costs a counter, never the mode.** +/// +/// `row` fits the footer's right-hand group all or nothing, and the counts row +/// used to be handed to it whole: one character over and the posture, the +/// containment word and the planning phase all left the screen together while +/// every counter stayed. In the 0.21.0 live capture that is exactly what +/// happened — the working frame drew the group at column forty-four and the +/// finished frame, five characters wider because `esc stops` had become `/ for +/// commands`, drew no group at all. The operator finished a turn under `/plan +/// on` with nothing on screen saying io-harness would refuse the next write. +/// +/// It is the failure F4 was written against, reached from the other direction, +/// and it is a regression with a date: the same capture passes on 0.12.0 and +/// 0.13.1, whose row was `4 steps · 14.2k tok · / for commands` and left room. +/// Adding `ctx N%` to the counts row spent the last ten columns. +/// +/// Which side gives is settled by the module rather than by this test: a +/// standing mode that stops the agent writing outranks what the last turn +/// spent. So the assertion is not merely that `planning` survives — it is that +/// a counter went instead, and that the counters left are the leftmost ones. +/// +/// Sabotage: hand `counts` to `row` unnarrowed, which is the 0.21.0 code — under +/// which only this pair of tests fails, and it fails at the one width an +/// eighty-to-a-hundred-and-twenty-column terminal is most likely to be. +#[test] +fn f4_a_full_counts_row_drops_a_counter_and_not_the_planning_phase() { + let text = footed(&full_row(), 100); + + assert!( + text.contains("planning"), + "the mode that stops the agent writing left the row: {text:?}", + ); + assert!( + text.contains("workspace-write/macos-sandbox-exec") && text.contains("read-only"), + "the phase took the rest of its group with it: {text:?}", + ); + // What paid for it, and which end it was taken off. `ctx 23%` is the + // rightmost *counter* and is what a crowded row gives up first. + assert!( + !text.contains("ctx 23%"), + "nothing was dropped, so the row cannot have fitted: {text:?}", + ); + assert!( + text.contains("6 steps") && text.contains("26.9k tok"), + "counters came off the wrong end: {text:?}", + ); + // **And the key hint is not a counter and is not droppable.** It was the + // rightmost item on the row and the narrowing took it first, which meant a + // full row lost `esc stops` mid-turn — the only place the footer says how to + // interrupt what is running. It is held out of the narrowing and appended + // after it, so it survives every width the row is drawn at. + assert!( + text.contains("/ for commands"), + "the interrupt hint was dropped to make room for a counter: {text:?}", + ); +} + +/// **F4 — and it holds with one more counter on the row, at every width the +/// group can be drawn at.** +/// +/// The companion above pins the geometry that actually failed; this one pins +/// the property, because the geometry is about to move. The counts row grows +/// every release — `ctx N%` is what tipped 0.21.0 over, and the cost counter +/// this release adds sits in the same group and makes the row wider still — so +/// a test that asserted one magic width would go green on the arithmetic of the +/// week it was written and say nothing about the week after. +/// +/// `plan 2/5` stands in for that next counter: it is a real member of the same +/// row, it is eight characters plus a separator, and with it the row no longer +/// fits at any width in the sweep on the first trim. +/// +/// **The sweep starts at seventy-four, and the number is the whole of what the +/// key hint being undroppable costs.** The right-hand group is fifty-seven +/// columns; `/ for commands` is fourteen and its separator three; fifty-seven +/// plus seventeen is seventy-four, and that is the narrowest row that can carry +/// the standing mode at all now that the hint is not a counter and cannot be +/// taken to make room. Below it the group goes and the hint stays — which is the +/// right way round: a mode an operator cannot see is bad, and a running turn an +/// operator cannot find the key to stop is worse. +/// +/// Sabotage: pop from the front of `counts` instead of the back — under which +/// this test still passes at every width and the one above fails on `6 steps`, +/// which is why both are here. +#[test] +fn f4_the_planning_phase_holds_at_every_width_that_can_hold_the_group() { + let mut status = full_row(); + status.plan = Some((2, 5)); + + for width in 74..=160u16 { + let text = footed(&status, width); + assert!( + text.contains("planning"), + "the planning phase is off the footer at {width} columns: {text:?}", + ); + // The row it was fitted into is still a row. A trim that undercounted + // would put the group past the edge instead of dropping a counter. + for line in status.footer(width, &DARK) { + let drawn: usize = line + .spans + .iter() + .map(|span| span.content.chars().count()) + .sum(); + assert!( + drawn <= width as usize, + "a footer row overflowed {width} columns: {line:?}", + ); + } + } +} + +// --------------------------------------------------------------------------- +// 0.22.0 — the cost field +// --------------------------------------------------------------------------- + +/// A store with one run, and one provider call per model named. +/// +/// Real rows through io-harness's own recorder rather than a `Total` built here, +/// because `Status::note_cost_from` reads the store: the split a price needs — +/// fresh prompt against cache read against cache write against completion — lives +/// only on the `provider_calls` row, and `EventKind::Step` carries a scalar token +/// count that cannot be priced at all. A fixture that handed the figure in would +/// be asserting the format string. +fn run_calling(models: &[&str]) -> (io_harness::Store, i64) { + let store = io_harness::Store::memory().expect("an in-memory store"); + let run = store + .start_run("summarise the module", "/repo") + .expect("a run"); + for model in models { + store + .record_provider_call( + run, + &io_harness::ProviderCall { + step: 1, + provider: "anthropic".into(), + model: Some((*model).to_string()), + usage: Some(io_harness::Usage { + prompt_tokens: 10_000, + completion_tokens: 2_000, + total_tokens: 12_000, + ..Default::default() + }), + latency_ms: 1_200, + ..Default::default() + }, + ) + .expect("the call is recorded"); + } + (store, run) +} + +/// The rates in force, pricing exactly one model. +fn rates() -> io_harness::pricing::PriceTable { + io_harness::pricing::PriceTable::new("2026-08-27").with( + "anthropic/claude-sonnet-4.5", + io_harness::pricing::Price { + input: 3_000_000, + output: 15_000_000, + ..io_harness::pricing::Price::ZERO + }, + ) +} + +/// **The cost field is absent rather than zero in all three ways of having no +/// answer, and they are three different states of an install.** +/// +/// An operator with no `[prices]` section, an operator whose table does not price +/// the model this run is using, and a session that has not called anything yet. +/// The status line has no room to tell them apart and does not try — `/cost` is +/// one keystroke away and tells all three — but it must not answer any of them +/// with `$0`, because `$0` is a measured figure and none of these is measured. +/// +/// This is the same rule the rest of the line already keeps: every counter is +/// absent until there is something to count, so a session that has run nothing +/// carries an almost empty row rather than a row of zeroes. The money field is the +/// one where getting it wrong is not merely untidy — a run whose models are all +/// outside the table has a real cost this program cannot state, and stating it as +/// nothing is the invented number the whole of `/cost` is built to avoid. +/// +/// Sabotage: `self.cost = Some(total.micros)` — three characters, reads as a +/// simplification of a `then_some`, and under it every unpriced session reports +/// `$0` for a turn that cost real money. Nothing else in the repository fails. +#[test] +fn the_cost_field_is_absent_rather_than_zero_when_nothing_can_be_priced() { + // 1. No prices configured at all. An empty table is what `cost::table` hands + // back for a file with no `[prices]`, so this is the ordinary state of a + // fresh install rather than an edge case. + let (store, run) = run_calling(&["anthropic/claude-sonnet-4.5"]); + let mut status = Status::new("anthropic/claude-sonnet-4.5"); + status.note_cost_from(&store, run, &io_harness::pricing::PriceTable::new("")); + assert_eq!(status.cost, None, "an empty table produced a figure"); + for (renderer, text) in both_renderers(&status) { + assert!( + !text.contains('$'), + "{renderer} drew money with no prices configured: {text:?}", + ); + } + + // 2. Prices configured, and none of them for the model this run used. The + // table is real, the date is real, and the answer is still unavailable. + let (store, run) = run_calling(&["some-lab/experimental-preview"]); + let mut status = Status::new("some-lab/experimental-preview"); + status.note_cost_from(&store, run, &rates()); + assert_eq!( + status.cost, None, + "a model outside the table was priced at nothing", + ); + for (renderer, text) in both_renderers(&status) { + assert!( + !text.contains('$'), + "{renderer} priced a model the table does not price: {text:?}", + ); + } + + // 3. A run that has called nothing. Not zero: nothing. + let (store, run) = run_calling(&[]); + let mut status = Status::new("anthropic/claude-sonnet-4.5"); + status.note_cost_from(&store, run, &rates()); + assert_eq!(status.cost, None, "a run with no calls reported a cost"); + for (renderer, text) in both_renderers(&status) { + assert!(!text.contains('$'), "{renderer}: {text:?}"); + } + + // And a run id nothing ever recorded reads back as no calls at all — which + // is no figure, not a zero one, and is not an error either: a notice about a + // failed read of a decorative field would be worse than the field's absence, + // which is why `note_cost_from` is silent on every failure. + let mut status = Status::new("anthropic/claude-sonnet-4.5"); + status.note_cost_from(&store, 9_999, &rates()); + assert_eq!( + status.cost, None, + "a run id that does not exist invented a figure" + ); +} + +/// **When there is a figure it is on BOTH renderers, out of one method.** +/// +/// `Status` renders two ways: `line` is the one-row form, which has exactly one +/// production caller — the fallback for a terminal under seven rows — and `footer` +/// is the three-row form `Status::render` takes on everything taller, which is to +/// say on every real terminal anybody uses. **This file has shipped a field into +/// one of them twice**: 0.8.0's spend field and 0.12.0's planning phase were each +/// added to `line`, asserted against `line`, went green, and were nowhere on +/// screen in a live capture of the running binary. +/// +/// So the assertion is over the pair, and the money is one method — `cost_field` +/// — that both renderers extend from, exactly as `budgets_left` and `queued_left` +/// already are. A test that read one renderer is a test that can pass while the +/// operator has an invisible field, which is the whole of what this test is +/// against. +/// +/// The figure itself is checked against `cost::money` of what io-harness says the +/// call cost, rather than against a string, so the two surfaces are proved to be +/// drawing the same number and not merely both drawing a dollar sign. +/// +/// Sabotage: push `self.cost_field()` into `Status::fields` alone and drop the +/// `counts.extend(self.cost_field())` from the footer — under which the one-row +/// arm of this test passes and the binary shows nothing. +#[test] +fn the_cost_field_is_drawn_by_the_line_and_by_the_footer_alike() { + let (store, run) = run_calling(&["anthropic/claude-sonnet-4.5"]); + let table = rates(); + + let mut status = Status::new("anthropic/claude-sonnet-4.5"); + status.note_cost_from(&store, run, &table); + + // What the harness says the run cost, from the rows the harness stored. + let calls = store.provider_calls(run).expect("the calls are readable"); + let expected = io_cli::cost::Total::of(&calls, &table).micros; + assert!(expected > 0, "a fixture that costs nothing proves nothing"); + assert_eq!(status.cost, Some(expected)); + + let drawn = io_cli::cost::money(expected); + for (renderer, text) in both_renderers(&status) { + assert!( + text.contains(&drawn), + "{renderer} does not carry the cost {drawn}: {text:?}", + ); + } + + // **Run-scoped, and forgotten with the run.** A cost that outlived its turn + // would be a figure that stopped moving while a new turn spent money beside + // it — the same argument the module makes for clearing the step count and the + // provider, and the opposite of the one it makes for keeping the planning + // phase. + status.forget_run(); + assert_eq!( + status.cost, None, + "the cost outlived the run that incurred it" + ); + for (renderer, text) in both_renderers(&status) { + assert!( + !text.contains('$'), + "{renderer} still shows the last run's bill: {text:?}", + ); + } +}