From c640c073d4dfc742cb03e9065d569dc9b8f4b8c8 Mon Sep 17 00:00:00 2001 From: aakashpawar1999 Date: Thu, 27 Aug 2026 21:02:18 +0530 Subject: [PATCH 1/7] build: take crossterm 0.29, which 0.18.0 held and named the exit for ratatui chooses the crossterm it re-exports by feature, so the direct crate and `crossterm_0_29` move in one commit or the build carries two crossterm versions whose key events are different types. That is the whole reason 0.18.0 pinned this at 0.28 and added the dependabot ignore, and both notes said the same thing: the bump is a deliberate change of the flag and the entry together, not a bot's to make. crossterm 0.29's only breaking change is a `KeyModifiers` Display fix, and no source file in this crate formats one. `Cargo.lock` resolves a single crossterm. The `ignore` list is now empty and gone with it. Nothing replaces it: one crossterm in the tree is the two lines in `Cargo.toml` agreeing, and a future bot PR that moves the direct crate alone fails to build rather than passing quietly. --- .github/dependabot.yml | 27 ++++++++++++-------- Cargo.lock | 57 +++++------------------------------------- Cargo.toml | 19 ++++++++------ 3 files changed, 33 insertions(+), 70 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index ca961f1..eef6eef 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -21,7 +21,9 @@ updates: # so a patch bump moves Cargo.lock and nothing anybody resolves against. patch-updates: update-types: ["patch"] - ignore: + # **There is no `ignore` list any more, and the two entries that were here are + # worth their notes.** + # # The `ratatui` entry that stood here is GONE as of 0.18.0, and the note it # carried is worth keeping because of how it ended. It said the composer was # `tui-textarea` 0.7 built against exactly ratatui 0.29 and crossterm 0.28, @@ -61,16 +63,19 @@ updates: # `tests/dependencies.rs::n1_the_composer_is_free_and_no_lru_is_in_the_advisorys_range` # fails the build if any `lru` in the lockfile falls inside the range. # - # `crossterm` stays pinned, and now for a reason of our own choosing rather - # than a dependency's: ratatui 0.30 is taken with the `crossterm_0_28` - # feature, so a Dependabot bump of the direct `crossterm` to 0.29 would put - # two versions in the tree again. Moving to crossterm 0.29 is a deliberate - # change of that feature flag and this entry together, not a bot's to make. - # It is pre-1.0, so the minor bump is the breaking one — patches still come - # through. - - dependency-name: crossterm - update-types: - ["version-update:semver-minor", "version-update:semver-major"] + # **The `crossterm` entry that stood here is GONE as of 0.22.0**, and it went + # the way it said it would. It held the direct `crossterm` at 0.28 because + # ratatui 0.30 was taken with the `crossterm_0_28` feature and a bot bumping + # one without the other would put two versions in the tree — and it named the + # exit: "moving to crossterm 0.29 is a deliberate change of that feature flag + # and this entry together, not a bot's to make." 0.22.0 made it. `Cargo.toml` + # now carries `crossterm_0_29` and `crossterm = "0.29"`, changed in one + # commit, and there is nothing left for this entry to hold back. + # + # Nothing replaces it. The property that entry was protecting — one crossterm + # in the tree — is not a judgement a list of ignores can keep: it is the two + # lines in `Cargo.toml` agreeing, and a future bot PR that moves the direct + # crate alone fails to build rather than passing quietly. - package-ecosystem: github-actions directory: "/" diff --git a/Cargo.lock b/Cargo.lock index be28b56..c84f445 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -758,22 +758,6 @@ version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" -[[package]] -name = "crossterm" -version = "0.28.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" -dependencies = [ - "bitflags 2.13.1", - "crossterm_winapi", - "mio", - "parking_lot", - "rustix 0.38.44", - "signal-hook", - "signal-hook-mio", - "winapi", -] - [[package]] name = "crossterm" version = "0.29.0" @@ -786,7 +770,7 @@ dependencies = [ "document-features", "mio", "parking_lot", - "rustix 1.1.4", + "rustix", "signal-hook", "signal-hook-mio", "winapi", @@ -1869,7 +1853,7 @@ name = "io-cli" version = "0.21.0" dependencies = [ "clap", - "crossterm 0.28.1", + "crossterm", "image", "io-harness", "ratatui", @@ -2150,12 +2134,6 @@ dependencies = [ "bitflags 2.13.1", ] -[[package]] -name = "linux-raw-sys" -version = "0.4.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" - [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -3193,8 +3171,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567584a3b0e6a8203c23de40b4861497266725eb5363dbfd18a1edd603cca9f0" dependencies = [ "cfg-if", - "crossterm 0.28.1", - "crossterm 0.29.0", + "crossterm", "instability", "ratatui-core", ] @@ -3505,19 +3482,6 @@ dependencies = [ "semver", ] -[[package]] -name = "rustix" -version = "0.38.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" -dependencies = [ - "bitflags 2.13.1", - "errno", - "libc", - "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", -] - [[package]] name = "rustix" version = "1.1.4" @@ -3527,7 +3491,7 @@ dependencies = [ "bitflags 2.13.1", "errno", "libc", - "linux-raw-sys 0.12.1", + "linux-raw-sys", "windows-sys 0.61.2", ] @@ -4086,7 +4050,7 @@ dependencies = [ "fastrand", "getrandom 0.4.3", "once_cell", - "rustix 1.1.4", + "rustix", "windows-sys 0.61.2", ] @@ -4098,7 +4062,7 @@ checksum = "9048a889effe34a5cddee0af7f53285198b16dca3be510858d38dfdb3e62a04e" dependencies = [ "bitflags 2.13.1", "parking_lot", - "rustix 1.1.4", + "rustix", "signal-hook", "windows-sys 0.61.2", ] @@ -5074,15 +5038,6 @@ dependencies = [ "windows-targets", ] -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets", -] - [[package]] name = "windows-sys" version = "0.61.2" diff --git a/Cargo.toml b/Cargo.toml index 1f5e156..bada99e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -75,12 +75,15 @@ io-harness = { version = "0.69", features = ["media", "browser", "documents"] } # go. io-cli owns its editing model now (`src/editor.rs`), and `lru` is out of the # tree entirely. # -# `default-features = false` with `crossterm_0_28` rather than the default -# `crossterm`, which resolves to 0.29: that bump is a separate decision from this -# one, and taking both at once would put two crossterm versions in the tree for as -# long as anything disagreed. Of the remaining defaults, `all-widgets` is only the -# calendar and `macros` is only the declarative builders — this crate uses -# neither, and the base widgets are not behind either flag. +# `default-features = false` with `crossterm_0_29` rather than the bare default +# `crossterm`, and the two crates move together or not at all. ratatui chooses the +# crossterm it re-exports by feature, so the direct `crossterm` line below and this +# flag name the same version by construction; the release that let them disagree +# would build two crossterm versions whose key events are different types. 0.18.0 +# held this at 0.28 and said so here and in `.github/dependabot.yml`; 0.22.0 is the +# deliberate change both notes were waiting for. Of the remaining defaults, +# `all-widgets` is only the calendar and `macros` is only the declarative builders +# — this crate uses neither, and the base widgets are not behind either flag. # # `unstable-rendered-line-info` is enabled for one method: `Paragraph::line_count`, # which answers how many rows a wrapped paragraph will occupy. `insert_before` @@ -88,12 +91,12 @@ io-harness = { version = "0.69", features = ["media", "browser", "documents"] } # exactly the content this product exists to keep. It is a feature of a dependency # already taken, not a dependency. ratatui = { version = "0.30", default-features = false, features = [ - "crossterm_0_28", + "crossterm_0_29", "layout-cache", "underline-color", "unstable-rendered-line-info", ] } -crossterm = "0.28" +crossterm = "0.29" clap = { version = "4", features = ["derive"] } tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync", "time"] } serde = { version = "1", features = ["derive"] } From 8665b2f8357cec42b1e42102891caafafbadd322 Mon Sep 17 00:00:00 2001 From: aakashpawar1999 Date: Thu, 27 Aug 2026 21:18:53 +0530 Subject: [PATCH 2/7] fix: the four defects 0.21.0 found, named, and did not fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All four were real, and one of them is a precondition for this release rather than a tidy-up: a price row is addressed as `prices.models."gpt-4.1"`, and `edit::split_path` split a dotted path on `.` with no notion of a quoted segment while `regions` stripped quotes after the same naive split. The two sides normalised differently, so any key containing a dot reached the wrong place or no place — a duplicate table, caught only by the read-back, reported to the operator as "the edit would have produced a file that does not parse" for a perfectly legal TOML key. A dot in a key can only be spelled quoted, so this was every model id with a version number in it. One quote-aware splitter now serves both sides, decoding through `toml` rather than by hand: a basic string takes the full escape set, a literal string takes none, and whitespace around a dot is legal. The two sites that put a segment back into a document re-spell it through the inverse. `providers` and `servers` are unaffected and were checked rather than assumed — both address entries by numeric index and put names in the entry body, already escaped. The other three: Removing an entry spliced to the next header's first byte, so it carried away the comment block documenting the *next* section. `move_entry` did the same. Both now stop at the trailing comment run. `move_entry` into the last position of a file with no trailing newline concatenated the header onto the final value — usually a parse refusal, but silent corruption when the file ends on a comment, which swallows the header and lands the moved entry's keys inside the previous table. It now guards the newline in both directions, as `Kind::Append` already did. `~/.claude/skills/CONVENTIONS.md` is a legal loose skill file that `skill_files` itself produces, and `import::plan` matched it on basename and planned it as *instructions* — appending a skill's whole body into the standing instructions loaded on every turn, forever. The skills-root guard now runs ahead of the name match. The comment asserting that arm was "exactly the skill files" was false in the direction that loses skills, and is corrected with it. And a codex `[mcp_servers."a.b"]` server was being dropped silently, because `sections()` reported three segments where there were two. Fixed by the splitter; the filter that looked wrong was starving, not wrong. --- src/edit.rs | 215 ++++++++++++++++++++++++++++++++++++++++---- src/import.rs | 52 +++++++++-- tests/edit.rs | 232 ++++++++++++++++++++++++++++++++++++++++++++++++ tests/import.rs | 152 +++++++++++++++++++++++++++++++ 4 files changed, 629 insertions(+), 22 deletions(-) diff --git a/src/edit.rs b/src/edit.rs index 712e2e5..1f74337 100644 --- a/src/edit.rs +++ b/src/edit.rs @@ -210,7 +210,7 @@ pub fn apply(text: &str, edits: &[Edit]) -> Result { .and_then(|(a, b)| Some((a.parse::().ok()?, b.parse::().ok()?))) .ok_or_else(|| format!("`{}` has no positions to move between", edit.path))?; - let names = vec![edit.path.clone()]; + let names = segments(&edit.path)?; let at = |index: usize| { regions .iter() @@ -221,7 +221,9 @@ pub fn apply(text: &str, edits: &[Edit]) -> Result { continue; } let source = at(from)?; - let block = text[source.start..source.body.end].to_string(); + // Stops before any comment run trailing the entry, which belongs + // to the section below it. See [`removal_end`]. + let taken = source.start..removal_end(text, &source.body); // Both splices are computed against the ORIGINAL text and applied // right to left, which is what makes a move one pass rather than @@ -234,7 +236,25 @@ pub fn apply(text: &str, edits: &[Edit]) -> Result { // Moving up: land where that entry's header begins. destination.start }; - splices.push((source.start..source.body.end, String::new())); + + // **The guard [`Kind::Append`] has, in both directions.** The + // last region's `body.end` is the length of the file, so a move + // into last place against a file that does not end in a newline + // splices the header onto whatever the final line was — and when + // that line is a comment, the comment swallows the header, the + // moved entry's keys join the table above it, and the result + // parses cleanly as a different configuration. The mirror is a + // moved block that ends without a newline, which swallows the + // destination's header the same way. + let mut block = text[taken.clone()].to_string(); + if !block.ends_with('\n') { + block.push('\n'); + } + if insert > 0 && !text[..insert].ends_with('\n') { + block.insert(0, '\n'); + } + + splices.push((taken, String::new())); splices.push((insert..insert, block)); continue; } @@ -263,7 +283,13 @@ pub fn apply(text: &str, edits: &[Edit]) -> Result { .iter() .find(|r| r.path == names && r.index == index) .ok_or_else(|| format!("there is no `{}` to remove in this file", edit.path))?; - splices.push((region.start..region.body.end, String::new())); + // Not `region.body.end`: that is the first byte of the next + // header, and the lines just above it document the next section + // rather than this one. See [`removal_end`]. + splices.push(( + region.start..removal_end(text, ®ion.body), + String::new(), + )); continue; } Kind::Set => {} @@ -300,7 +326,11 @@ pub fn apply(text: &str, edits: &[Edit]) -> Result { // end of this section rather than the end of the file, // where it would silently join whatever section is last. let at = insertion_point(text, ®ion.body); - splices.push((at..at, format!("{key} = {}\n", edit.value))); + // `key` is DECODED, so it is spelled back rather than + // pasted: a name with a dot or a space in it written bare + // is a dotted key or a parse error, never the key asked + // for. + splices.push((at..at, format!("{} = {}\n", spell(&key), edit.value))); } } } @@ -313,12 +343,21 @@ pub fn apply(text: &str, edits: &[Edit]) -> Result { if resolve(&parsed, &table_path.names, &key).is_some() { return Err(dotted_refusal(&edit.path)); } - let header = table_path.names.join("."); + let header = table_path + .names + .iter() + .map(|name| spell(name)) + .collect::>() + .join("."); let mut block = String::new(); if !text.is_empty() && !text.ends_with('\n') { block.push('\n'); } - block.push_str(&format!("\n[{header}]\n{key} = {}\n", edit.value)); + block.push_str(&format!( + "\n[{header}]\n{} = {}\n", + spell(&key), + edit.value + )); splices.push((text.len()..text.len(), block)); } } @@ -494,8 +533,8 @@ fn split_path(path: &str) -> Result<(TablePath, String), String> { let mut names: Vec = Vec::new(); let mut index = 0usize; - let segments: Vec<&str> = path.split('.').collect(); - let (key, table) = segments + let parts = segments(path)?; + let (key, table) = parts .split_last() .ok_or_else(|| format!("`{path}` names no key"))?; @@ -511,14 +550,123 @@ fn split_path(path: &str) -> Result<(TablePath, String), String> { .map_err(|_| format!("`{path}` has an index that is not a number"))?; names.push(name.to_string()); } else { - names.push(segment.to_string()); + names.push(segment.clone()); } } if key.is_empty() { return Err(format!("`{path}` names no key")); } - Ok((TablePath { names, index }, (*key).to_string())) + Ok((TablePath { names, index }, key.clone())) +} + +/// Cut a dotted TOML path into its segments, **decoded**. +/// +/// The one splitter, used by both halves of this module, and that is the point +/// of it. A dot inside a quoted key is not a separator — TOML spells a bare key +/// out of `A-Za-z0-9_-` and nothing else, so a key carrying a dot can only ever +/// be written quoted (`"gpt-4.1"`, `"github.com/x"`) and `path.split('.')` cuts +/// it in half. [`split_path`] and [`regions`] each did exactly that and then +/// normalised what came out DIFFERENTLY, so a caller's path and the file's own +/// header never matched: the read half answered `None` for any dotted key and +/// the write half fell through to the append arm and emitted a second copy of a +/// table that was already there, caught only by the read-back in [`apply`] and +/// only as "would have produced a file that does not parse". +/// +/// The segments come back decoded, which is the half `trim_matches('"')` cannot +/// do: a basic string takes the full escape set, so `"a\"b"` is a legal key +/// whose name is `a"b`, and trimming strips repeated quotes off both ends and +/// resolves no escape at all. A literal string `'...'` takes no escapes, so a +/// `'` can never appear inside one and finding its end is a search. The decoding +/// is `toml`'s own for the reason [`array`] spells through `toml` rather than a +/// format string — the escape rules belong to the format, not to this file. +/// [`spell`] is the inverse, for the two places a segment goes back into a +/// document. +/// +/// Whitespace around a dot is legal (`[ a . b ]`), so an unquoted segment is +/// trimmed and a quoted one is taken exactly as the quotes deliver it. +fn segments(path: &str) -> Result, String> { + let bytes = path.as_bytes(); + let mut out: Vec = Vec::new(); + let mut i = 0usize; + + loop { + while matches!(bytes.get(i), Some(b' ' | b'\t')) { + i += 1; + } + match bytes.get(i).copied() { + Some(quote) if quote == b'"' || quote == b'\'' => { + // Find the closing quote. A backslash in a basic string escapes + // whatever follows it, including the quote itself; a literal + // string has no escapes, so the first `'` ends it. + let mut j = i + 1; + let end = loop { + match bytes.get(j).copied() { + None => { + return Err(format!( + "`{path}` has a quoted segment that is never closed" + )) + } + Some(b'\\') if quote == b'"' => j += 2, + Some(c) if c == quote => break j, + Some(_) => j += 1, + } + }; + // `toml` decodes it, because `toml` is what wrote the rules. + let literal = &path[i..=end]; + let decoded = toml::from_str::(&format!("probe = {literal}")) + .ok() + .and_then(|table| Some(table.get("probe")?.as_str()?.to_string())) + .ok_or_else(|| { + format!("`{path}` has a quoted segment that is not a TOML string") + })?; + out.push(decoded); + i = end + 1; + while matches!(bytes.get(i), Some(b' ' | b'\t')) { + i += 1; + } + if !matches!(bytes.get(i), None | Some(b'.')) { + return Err(format!( + "`{path}` has more than a quoted string in one of its segments" + )); + } + } + _ => { + let end = path[i..].find('.').map_or(bytes.len(), |n| i + n); + out.push(path[i..end].trim().to_string()); + i = end; + } + } + if i >= bytes.len() { + return Ok(out); + } + i += 1; // The dot. + } +} + +/// Spell one decoded segment the way a document has to carry it. +/// +/// The inverse of the decoding [`segments`] does, and it exists because that +/// decoding created the need: a segment in hand is now a NAME, and a name is not +/// always spellable as itself. A bare key is `A-Za-z0-9_-` and nothing else, so +/// anything holding a dot, a space, a slash or a quote has to go back into the +/// file quoted — and writing `[prices.models.gpt-4.1]` instead of +/// `[prices.models."gpt-4.1"]` names a different table two levels deeper, which +/// still parses. +/// +/// The quoting goes through `toml`'s own serializer for the same reason +/// [`array`] does: a format string cannot escape a `"` or a backslash correctly +/// and would produce either a parse error or a different name. +#[must_use] +pub fn spell(segment: &str) -> String { + let bare = !segment.is_empty() + && segment + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-'); + if bare { + return segment.to_string(); + } + toml::Value::String(segment.to_string()).to_string() } /// Whether the parsed document already holds this path, in any shape. @@ -550,6 +698,43 @@ fn insertion_point(text: &str, body: &Range) -> usize { } } +/// Where a region's bytes end, for the purpose of taking the region away. +/// +/// **A region's `body.end` is the first byte of the NEXT header**, which is what +/// makes it right for parsing the section and wrong for deleting it: everything +/// between this section's last key and that header is inside the range, and what +/// sits there is almost always the comment block an operator wrote to explain the +/// section BELOW. Splicing `region.start..region.body.end` away took somebody +/// else's sentence with it, and a moved entry carried it to a place it was no +/// longer true. +/// +/// So the trailing lines are walked backwards over blank and comment lines, and +/// the answer is the first line of that trailing comment run — `body.end` when +/// there is no comment in it. The blank lines ABOVE the run stay inside the range +/// on purpose: they belonged to the section being taken away, and leaving them +/// would make every removal add an empty line to the file. +fn removal_end(text: &str, body: &Range) -> usize { + let slice = &text[body.clone()]; + let mut starts: Vec = vec![0]; + starts.extend(slice.match_indices('\n').map(|(at, _)| at + 1)); + let mut answer = body.end; + for &start in starts.iter().rev() { + // The empty remainder after a final newline is not a line. + if start >= slice.len() { + continue; + } + let line = slice[start..].split('\n').next().unwrap_or_default().trim(); + if line.is_empty() { + continue; + } + if !line.starts_with('#') { + break; + } + answer = body.start + start; + } + answer +} + /// Cut the document into regions at every table header. /// /// The scan is a character state machine rather than a line filter, because a @@ -663,10 +848,10 @@ fn regions(text: &str) -> Result, String> { .or_else(|| header.strip_prefix('[').and_then(|h| h.strip_suffix(']'))) .ok_or_else(|| format!("`{header}` is not a section header this editor understands"))?; - let path: Vec = inner - .split('.') - .map(|s| s.trim().trim_matches('"').trim_matches('\'').to_string()) - .collect(); + // The same splitter the caller's path goes through, which is the whole + // reason there is only one: two scans that agree about what a segment is + // are the only way a header and a path can ever be compared. + let path = segments(inner)?; let index = seen.entry(path.clone()).or_insert(0); let body_end = header_starts.get(n + 1).copied().unwrap_or(bytes.len()); diff --git a/src/import.rs b/src/import.rs index 5e52bac..bccaa41 100644 --- a/src/import.rs +++ b/src/import.rs @@ -494,6 +494,30 @@ pub fn plan(found: &[Found], home: &Path, scope: Scope) -> Vec { .file_name() .map(|name| name.to_string_lossy().to_string()) .unwrap_or_default(); + + // **Where a file came from decides what it is, and the basename + // match below cannot see that.** [`skill_files`] counts any loose + // `*.md` at the top of a skills root, so `~/.claude/skills/ + // CONVENTIONS.md` — or `CLAUDE.md`, or `MEMORY.md` — arrives here as + // a `Source::Claude` path wearing an anchor file's name, and the arm + // below matched it as standing instructions: the whole body appended + // into the file that is loaded on EVERY turn, forever, instead of a + // named skill read on demand. + // + // Pairing each anchor name with its owning `Source` is the smaller + // diff and it does not close this: `~/.claude/skills/CLAUDE.md` is + // genuinely `Source::Claude` and genuinely named `CLAUDE.md`, and it + // is still a skill. So the guard asks where the file is, ahead of + // asking what it is called. + let rooted = path.parent().and_then(Path::file_name).is_some_and(|dir| { + dir.to_str() + .is_some_and(|dir| dir == "skills" || dir == "plugins") + }); + if name == SKILL_FILE || rooted { + skills.push((one.source, path.clone())); + continue; + } + match (one.source, name.as_str()) { ( _, @@ -509,10 +533,13 @@ pub fn plan(found: &[Found], home: &Path, scope: Scope) -> Vec { (Source::Gemini, "mcp_config.json") => { items.extend(foreign_mcp(one.source, path, scope)); } - // Everything [`files`] can produce is named above except the - // skill files, so this arm is exactly those. Keep it that way: - // a new anchor file added to `files` without a case here would - // silently arrive as a skill. + // Every skill file has already been taken by the guard above, so + // what reaches here is a file [`files`] produces and no arm + // claims. It is treated as a skill, which is the safe end of the + // mistake — a skill is read on demand and an instructions file is + // read on every turn. Keep the guard ahead of this: a new anchor + // file added to `files` without a case above still arrives as a + // skill, and that is a plan an operator can see and decline. _ => skills.push((one.source, path.clone())), } } @@ -669,17 +696,25 @@ fn codex_config(path: &Path, scope: Scope) -> Vec { items.push(model_item(Source::Codex, path, &model)); } + // **Two segments exactly**, which is the server's own table and not its + // `[mcp_servers..env]` child. That was already the right filter and it + // was throwing servers away, because `sections` split `[mcp_servers. + // "github.com/x"]` on the dot inside the quoted name and reported three + // segments — a name that can ONLY be spelled quoted was the one shape it + // could not see. `edit::segments` decides that now, and the name arrives + // decoded, so every path built back out of it goes through `edit::spell`. for name in crate::edit::sections(&text) .into_iter() .filter(|section| section.len() == 2 && section[0] == "mcp_servers") .map(|section| section[1].clone()) { - let Some(command) = crate::edit::value_at(&text, &format!("mcp_servers.{name}.command")) + let quoted = crate::edit::spell(&name); + let Some(command) = crate::edit::value_at(&text, &format!("mcp_servers.{quoted}.command")) .map(|v| unquoted(&v)) else { continue; }; - let args = crate::edit::value_at(&text, &format!("mcp_servers.{name}.args")) + let args = crate::edit::value_at(&text, &format!("mcp_servers.{quoted}.args")) .map(|value| strings(&value)) .unwrap_or_default(); let names = env_names(&text, &name); @@ -710,7 +745,10 @@ fn codex_config(path: &Path, scope: Scope) -> Vec { // config.toml with a `"""` block above its env table is not a thing that exists; // if one turns up, expose `regions` from `edit` and use it here. fn env_names(text: &str, server: &str) -> Vec { - let header = format!("[mcp_servers.{server}.env]"); + // `server` is a decoded name, so it is spelled back before it is compared + // against a line of the file — a name with a dot in it is written quoted in + // the header and matching the raw name would find no table at all. + let header = format!("[mcp_servers.{}.env]", crate::edit::spell(server)); let mut names = Vec::new(); let mut inside = false; for line in text.lines() { diff --git a/tests/edit.rs b/tests/edit.rs index c98f360..68924e7 100644 --- a/tests/edit.rs +++ b/tests/edit.rs @@ -385,3 +385,235 @@ max_steps = 30 "a move is not reversible" ); } + +/// **A dot inside a quoted key is not a path separator, and the two sides of this +/// module used to disagree about that.** +/// +/// `split_path` split the caller's path on every `.` and normalised nothing; +/// `regions` split the file's header on every `.` and then stripped quotes with +/// `trim_matches`. Two different wrong answers, so `prices.models."gpt-4.1"` never +/// matched itself: the read half answered `None` and the write half fell through +/// to the append arm and emitted a SECOND `[prices.models."gpt-4.1"]` table, which +/// only the read-back caught, and only as "the edit would have produced a file +/// that does not parse" — for a perfectly legal TOML key. +/// +/// Sabotage: put `path.split('.')` back in `split_path`, or +/// `.trim_matches('"').trim_matches('\'')` back in `regions`, or drop either +/// `spell` call from the two spelling sites, and this test fails. +#[test] +fn f1_a_dot_inside_a_quoted_key_is_one_segment_on_both_sides() { + // Two quoted keys, and the second is the one `trim_matches` cannot ever get + // right: a basic string takes the full escape set, so `"a\"b"` is a legal key + // whose NAME is `a"b`. Trimming quotes off both ends of that yields `a\"b`, + // which is a key no file holds. + const QUOTED: &str = "\ +[prices.models.\"gpt-4.1\"] +input = 3.0 +output = 12.0 + +[prices.models.\"a\\\"b\"] +input = 1.0 +"; + + // 1. The header is cut into segments, decoded, and the escape is resolved. + let sections = edit::sections(QUOTED); + assert!( + sections.contains(&vec![ + "prices".to_string(), + "models".to_string(), + "gpt-4.1".to_string() + ]), + "the quoted header was split on the dot inside it: {sections:?}" + ); + assert!( + sections.contains(&vec![ + "prices".to_string(), + "models".to_string(), + "a\"b".to_string() + ]), + "the escaped quote was not decoded: {sections:?}" + ); + + // 2. A caller's path is cut the same way, so the two sides meet. + assert_eq!( + edit::value_at(QUOTED, "prices.models.\"gpt-4.1\".input").as_deref(), + Some("3.0"), + "a dotted key showed a file with no value in it" + ); + // A literal string spells the same key and takes no escapes at all. + assert_eq!( + edit::value_at(QUOTED, "prices.models.'gpt-4.1'.output").as_deref(), + Some("12.0"), + "a literal-string segment did not reach the key a basic string reaches" + ); + assert_eq!( + edit::value_at(QUOTED, "prices.models.\"a\\\"b\".input").as_deref(), + Some("1.0"), + "the escape was not resolved on the caller's side" + ); + // Whitespace around a dot is legal TOML and an unquoted segment is trimmed. + assert_eq!( + edit::value_at(QUOTED, "prices . models . \"gpt-4.1\" . input").as_deref(), + Some("3.0"), + "whitespace around a dot became part of a segment name" + ); + + // 3. The write half reaches the same section, and reaches it exactly once. + let after = edit::apply( + QUOTED, + &[Edit::set("prices.models.\"gpt-4.1\".input", "4.0")], + ) + .unwrap(); + assert_eq!( + assert_only_span_changed(QUOTED, &after, "prices.models.\"gpt-4.1\".input"), + "4.0" + ); + assert_eq!( + after.matches("[prices.models.\"gpt-4.1\"]").count(), + 1, + "the write fell through to the append arm and duplicated the table:\n{after}" + ); + + // 4. Because a segment is DECODED now, the two sites that spell one back into + // a document have to re-quote anything that is not a bare key. A bare key is + // `A-Za-z0-9_-` and nothing else, so both of these need quotes. + let key = edit::apply( + QUOTED, + &[Edit::set( + "prices.models.\"gpt-4.1\".\"cached input\"", + "1.5", + )], + ) + .unwrap(); + assert!( + key.contains("\"cached input\" = 1.5"), + "the new key was spelled bare and the space in it would not parse:\n{key}" + ); + + let header = edit::apply( + QUOTED, + &[Edit::set("prices.models.\"gpt-4.5\".input", "5.0")], + ) + .unwrap(); + assert!( + header.contains("[prices.models.\"gpt-4.5\"]"), + "the new header was spelled bare and the dot in it would name a third table:\n{header}" + ); + // And the proof that both spellings are right is that the document still says + // what it was asked to say when it is read back as TOML. + let parsed: toml::Value = toml::from_str(&header).expect("the appended header parses"); + assert_eq!( + parsed["prices"]["models"]["gpt-4.5"]["input"].as_float(), + Some(5.0), + "the appended section landed under a different key than the one asked for" + ); +} + +/// **A section's trailing comment run belongs to the section BELOW it.** +/// +/// A region's `body.end` is the first byte of the NEXT header, so splicing +/// `region.start..region.body.end` away deletes everything between this entry's +/// last key and that header — including the comment block an operator wrote to +/// document the next section. `Kind::Move` carried the same bytes away with it. +/// +/// Sabotage: use `region.body.end` at either splice instead of `removal_end`, and +/// the comment naming the entry that survives is deleted along with the one that +/// was asked for. +#[test] +fn f1_removing_or_moving_an_entry_leaves_the_next_one_s_comment_behind() { + const DOCUMENTED: &str = "\ +[[mcp]] +id = \"docs\" + +# search is the one the team actually uses; do not remove it +[[mcp]] +id = \"search\" +"; + + let removed = edit::apply(DOCUMENTED, &[Edit::remove("mcp[0]")]).unwrap(); + assert!( + removed.contains("# search is the one the team actually uses"), + "removing the first entry deleted the second entry's comment:\n{removed}" + ); + assert!(!removed.contains("docs"), "the wrong entry survived"); + // The blank line that separated the removed entry from that comment goes with + // it, so repeated removals do not leave a growing stack of empty lines. + assert!( + removed.starts_with("# search"), + "removal accumulated whitespace above the comment:\n{removed:?}" + ); + + // The same bytes, carried away by a move rather than a removal. + let moved = edit::apply(DOCUMENTED, &[Edit::move_entry("mcp", 0, 1)]).unwrap(); + assert!( + moved.contains("# search is the one the team actually uses"), + "the moved entry took the next entry's comment with it:\n{moved}" + ); + let comment = moved.find("# search").unwrap(); + let docs = moved.find("docs").unwrap(); + assert!( + comment < docs, + "the comment moved down with the entry it does not describe:\n{moved}" + ); +} + +/// **A move has no newline guard, and `Kind::Append` right beside it does.** +/// +/// The last region's `body.end` is the length of the file, so moving an entry +/// into last place splices its header straight onto whatever the final line was. +/// Usually that is a parse refusal; when the final line is a COMMENT with no +/// newline after it the comment swallows the header, the moved entry's keys land +/// inside the previous table, and the file still parses — as something else. +/// +/// Sabotage: delete either half of the guard in `Kind::Move`. The down arm loses +/// the `\n` in front of the block and the up arm loses the one behind it. +#[test] +fn f1_a_move_against_a_file_with_no_trailing_newline_still_parses() { + // No newline after the last line, and the last line is a comment. + const COMMENTED: &str = "\ +[[provider]] +kind = \"openrouter\" +model = \"a\" + +[[provider]] +kind = \"compatible\" +preset = \"groq\" +model = \"b\" +# groq answers last, and there is no newline after this line"; + + let down = edit::apply(COMMENTED, &[Edit::move_entry("provider", 0, 1)]).unwrap(); + let config = io_harness::Config::from_toml(&down).expect("the moved file loads"); + assert_eq!( + config.fallback_specs().len(), + 1, + "the moved entry's keys were absorbed by the entry above it:\n{down}" + ); + assert!( + down.find("groq").unwrap() < down.find("openrouter").unwrap(), + "the move did not reorder the chain:\n{down}" + ); + assert!( + down.contains("# groq answers last"), + "the trailing comment was overwritten:\n{down}" + ); + + // The mirror: the block itself ends without a newline, and the destination's + // header is spliced onto the end of it. + const TAIL: &str = "\ +[[provider]] +kind = \"openrouter\" +model = \"a\" + +[[provider]] +kind = \"compatible\" +preset = \"groq\" +model = \"b\""; + + let up = edit::apply(TAIL, &[Edit::move_entry("provider", 1, 0)]).unwrap(); + let config = io_harness::Config::from_toml(&up).expect("the moved file loads"); + assert_eq!(config.fallback_specs().len(), 1, "{up}"); + assert!( + up.find("groq").unwrap() < up.find("openrouter").unwrap(), + "the move did not reorder the chain:\n{up}" + ); +} diff --git a/tests/import.rs b/tests/import.rs index 15ab26d..f1f273b 100644 --- a/tests/import.rs +++ b/tests/import.rs @@ -349,6 +349,70 @@ fn a_codex_mcp_table_round_trips_into_an_mcp_server() { assert!(env.is_empty()); } +/// **A codex server whose name has a dot in it can only be spelled quoted, and +/// io-cli used to drop it in silence.** +/// +/// `[mcp_servers."github.com/x"]` is how every marketplace-style server id is +/// written, and `crate::edit::sections` reported it as THREE segments because the +/// header was split on every `.`. The `section.len() == 2` filter here then threw +/// it away — no item, no refusal, no line in the report. The two `value_at` paths +/// built from the name have the mirror problem: a decoded name spliced raw into a +/// dotted path names a table nobody has. +/// +/// Sabotage: put `path.split('.')` back in `edit::split_path` or `edit::regions`, +/// or drop `edit::spell` from the paths `codex_config` and `env_names` build. +#[test] +fn a_codex_server_whose_name_must_be_quoted_still_arrives() { + let home_root = tempfile::tempdir().expect("a fake home"); + let workspace = tempfile::tempdir().expect("a workspace"); + let home = tempfile::tempdir().expect("io's home"); + + put( + &home_root.path().join(".codex/config.toml"), + &format!( + "[mcp_servers.\"docs.v2\"]\n\ + command = \"docs-mcp\"\n\ + args = [\"--port\", \"8080\"]\n\ + \n\ + [mcp_servers.\"docs.v2\".env]\n\ + {UNSET} = \"{SECRET}\"\n" + ), + ); + + let found = detect(home_root.path(), workspace.path()); + let items: Vec<_> = plan(&found, home.path(), Scope::Project) + .into_iter() + .filter(|item| item.kind == Kind::Mcp) + .collect(); + assert_eq!( + items.len(), + 1, + "a server whose name needed quoting was dropped without a word: {items:?}", + ); + + let report = apply(&items, workspace.path()); + assert!(report.refused.is_empty(), "{report:?}"); + + let servers = written_servers(&workspace.path().join("io.toml")); + assert_eq!(servers.len(), 1); + assert_eq!( + servers[0].id, "docs.v2", + "the id arrived with quotes still on it or split in half", + ); + let (command, args, env) = stdio(&servers[0]); + assert_eq!(command, "docs-mcp"); + assert_eq!(args, vec!["--port", "8080"]); + assert_eq!( + env, + vec![UNSET], + "the env table of a quoted server was looked for under an unquoted header", + ); + assert!( + holds(workspace.path(), SECRET).is_none(), + "the secret was copied", + ); +} + // --------------------------------------------------------------------------- // D. Credentials // --------------------------------------------------------------------------- @@ -740,6 +804,94 @@ fn a_skill_name_that_is_already_taken_is_refused_rather_than_overwritten() { ); } +/// **A loose file in a skills root is a skill, whatever it happens to be called.** +/// +/// `skill_files` counts any loose `*.md` at the top of `~/.claude/skills` and +/// `~/.claude/plugins`, so a `CONVENTIONS.md` sitting there is produced as a +/// `Source::Claude` path. `plan`'s first arm matched on the BASENAME with `_` for +/// the source, so it was planned as `Kind::Instructions` — the whole body appended +/// into the standing instructions file that is loaded on every turn, forever, +/// rather than a named skill copied under `/skills` and read on demand. +/// `~/.claude/skills/CLAUDE.md` is the case that proves pairing each anchor name +/// with its owning source would not have closed this: that file genuinely IS +/// `Source::Claude` and genuinely IS named `CLAUDE.md`, and it is still a skill. +/// +/// Sabotage: delete the skills-root guard at the top of `plan`'s loop. +#[test] +fn a_loose_file_in_a_skills_root_is_a_skill_and_never_standing_instructions() { + let home_root = tempfile::tempdir().expect("a fake home"); + let workspace = tempfile::tempdir().expect("a workspace"); + let home = tempfile::tempdir().expect("io's home"); + std::fs::create_dir_all(home.path().join("skills")).expect("a skills directory"); + + // Three loose skills wearing three anchor-file names, and the one real anchor + // file a directory above them. + put( + &home_root.path().join(".claude/skills/CONVENTIONS.md"), + "# conventions\n\nA skill about conventions.\n", + ); + put( + &home_root.path().join(".claude/skills/CLAUDE.md"), + "# claude\n\nA skill that happens to be named like the anchor file.\n", + ); + put( + &home_root.path().join(".claude/plugins/MEMORY.md"), + "# memory\n\nA plugin's loose skill.\n", + ); + put( + &home_root.path().join(".claude/CLAUDE.md"), + "# standing\n\nThe operator's own standing guidance.\n", + ); + + let found = detect(home_root.path(), workspace.path()); + let items = plan(&found, home.path(), Scope::Project); + + let instructions: Vec<_> = items + .iter() + .filter(|item| item.kind == Kind::Instructions) + .collect(); + assert_eq!( + instructions.len(), + 1, + "a skill was planned as standing instructions: {instructions:?}", + ); + assert_eq!( + instructions[0].from, + home_root.path().join(".claude/CLAUDE.md"), + "the anchor file that is instructions is the one OUTSIDE the skills roots", + ); + + let mut names: Vec = items + .iter() + .filter(|item| item.kind == Kind::Skill) + .map(|item| match &item.to { + Destination::File(path) => path + .parent() + .and_then(|parent| parent.file_name()) + .map(|name| name.to_string_lossy().to_string()) + .unwrap_or_default(), + other => panic!("a loose skill file has nowhere to go: {other:?}"), + }) + .collect(); + names.sort(); + assert_eq!(names, vec!["CLAUDE", "CONVENTIONS", "MEMORY"]); + + let report = apply(&items, workspace.path()); + assert!(report.refused.is_empty(), "{report:?}"); + let landed = home.path().join("skills/CONVENTIONS/SKILL.md"); + assert_eq!( + std::fs::read_to_string(&landed).expect("the skill was copied"), + "# conventions\n\nA skill about conventions.\n", + ); + assert!( + io_harness::Skills::discover(home.path().join("skills")) + .expect("it discovers") + .get("CONVENTIONS") + .is_some(), + "the loose file did not become a skill io-harness can find on demand", + ); +} + #[test] fn sixty_three_present_and_three_incoming_is_reported_and_nothing_is_written() { let home_root = tempfile::tempdir().expect("a fake home"); From 5dfb68f81c1168219ba20197c536e37a69576737 Mon Sep 17 00:00:00 2001 From: aakashpawar1999 Date: Thu, 27 Aug 2026 21:19:07 +0530 Subject: [PATCH 3/7] feat: read the money that has been in the store since 0.18.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit io-harness has recorded a row per provider call for four releases — the model, the whole token split, the latency, the time to first token — and io-cli read none of them, while `src/status.rs` explained in a doc comment that a figure with a currency in front of it "would be one this interface invented". That sentence was true when it was written and stopped being true in io-harness 0.18.0. `/cost` commits what has been spent: this run, this session, by model and by day, with cache reads and writes shown as the breakdown of the prompt they are rather than added on top of it. `/stats` commits how the work has gone: outcomes, days, the first-try counts with their denominator named, gate failures by phase with the two vocabularies kept apart, the slowest calls, time to first token, and what the store holds. `/usage` is an alias for `/cost` and earns no row; it answered `/status` before, under an argument that a page whose whole subject is spending supersedes. Prices are not compiled in. `verify::catalogue` has fetched the model catalogue since 0.1.0 and mapped every row down to its id, discarding the price on it; `src/prices.rs` keeps it, filtered to what the connected provider serves and spelled the way that provider names it. An operator with no provider connected has no prices, sees tokens and no currency, and nothing claims a cost. For three of the four providers the rates necessarily come from the reference catalogue rather than the vendor — OpenAI, Anthropic and Google publish none — and the page says which. Three honesty rules are held as hard as the arithmetic: a call the provider reported no usage for is unknown and never free, a model with no rate makes the total a floor and the page renders that count rather than hiding it, and every surface that draws money names the catalogue and the date it was read. Two status-line defects are fixed because this release would have made both worse. The footer fitted its right-hand group all-or-nothing, so a counts row that grew past the line took the policy layer, the containment mode and the planning phase off the screen together — the failure the footer's own comment says it exists to prevent, arriving from the other side, and the reason the live planning capture has failed since 0.14.0 added `ctx N%`. Counters now yield. And `fields` pushed `planning` to the right of the counters, so the narrow line dropped it before the token count, inverting the priority written five lines above it. This release adds another counter to that row. `Inspect` is re-split to hold the two new commands. `/image`, `/copy` and `/copy diff` move to `Turn`, where they belonged — all three act on the turn that just finished and none of them asks the store a question. The fold that `status` and `context` each carried a copy of is now `src/page.rs`, which is what both new pages are built from. --- src/commands.rs | 87 +++++++++-- src/configure.rs | 8 + src/context.rs | 49 +----- src/cost.rs | 399 +++++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 4 + src/main.rs | 54 +++++++ src/page.rs | 190 ++++++++++++++++++++++ src/prices.rs | 285 +++++++++++++++++++++++++++++++++ src/settings.rs | 54 +++++++ src/stats.rs | 243 +++++++++++++++++++++++++++++ src/status.rs | 244 +++++++++++++++++++---------- src/verify.rs | 86 ++++++++-- tests/docs.rs | 5 + tests/status.rs | 117 ++++++++++++++ 14 files changed, 1678 insertions(+), 147 deletions(-) create mode 100644 src/cost.rs create mode 100644 src/page.rs create mode 100644 src/prices.rs create mode 100644 src/stats.rs diff --git a/src/commands.rs b/src/commands.rs index f8c6b1a..958bd9b 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -232,6 +232,18 @@ pub const COMMANDS: &[(&str, &str)] = &[ "/clear", "start a new conversation; this one stays in /resume", ), + // The two halves of one question, and they are two commands because they are + // two questions. `/cost` says what the work cost; `/stats` says whether it + // worked. Every agent that has both keeps them apart, and a single screen + // carrying thirteen sections would be one nobody reads to the end. + ( + "/cost", + "commit what this run, this session and this install have spent", + ), + ( + "/stats", + "commit how the runs have gone: outcomes, first-try, gates, latency", + ), ]; /// What an operator is doing when they reach for a command. @@ -306,10 +318,33 @@ pub const GROUPS: &[(Group, &[&str])] = &[ Group::Session, &["/clear", "/resume", "/fork", "/setup", "/exit"], ), + // **`/image`, `/copy` and `/copy diff` moved here in 0.22.0, and it is a + // correction rather than a way of making room** — the same sentence 0.19.0 + // wrote when it moved `/mcp` and `/provider`, and it is worth being able to + // say it twice. All three act on the turn that just finished: `/image` draws + // an image attached to it, `/copy` puts its answer or its diff on the + // clipboard. None of them asks the store a question, which is what `Inspect` + // means. They were filed there because they *show* something, and showing is + // not the same as inspecting. + // + // `/cost` and `/stats` are what made the misfiling worth correcting: `Inspect` + // stood at nine of a bound of ten and both new commands belong in it, so the + // choice was between re-filing three commands that were in the wrong group and + // filing two more that would have been. The bound exists to stop the grouped + // menu becoming the flat list it replaced, and answering it by weakening it + // would have given up the thing it was protecting. ( Group::Turn, &[ - "/model", "/contain", "/plan", "/profile", "/steer", "/compact", + "/model", + "/contain", + "/plan", + "/profile", + "/steer", + "/compact", + "/image", + "/copy", + "/copy diff", ], ), ( @@ -321,9 +356,8 @@ pub const GROUPS: &[(Group, &[&str])] = &[ "/expand", "/fleet", "/skills", - "/image", - "/copy", - "/copy diff", + "/cost", + "/stats", ], ), // **`/mcp` and `/provider` moved here in 0.19.0, and it is a correction rather @@ -815,6 +849,24 @@ pub enum Action { /// budgets in force, the context fill, the servers that came up — so what is /// committed is the state io-harness is in and not io-cli's account of it. Status, + /// Commit what has been spent into the scrollback. + /// + /// The sibling of [`Action::Status`] and drawn the same way, for the same + /// reason: this is a page a reader wants to keep beside the turns it accounts + /// for, and the terminal's own scrollback is where it keeps things. What + /// separates the two is the question. `/status` says what the session *is* + /// — the policy, the caps, the budgets in force; this says what it has *cost*, + /// which is a different fact about the same run and one that only ever climbs. + Cost, + /// Commit how the runs have gone into the scrollback. + /// + /// Deliberately not folded into [`Action::Cost`] under one name. What a run + /// cost and whether it worked are two questions, and the answer to the second + /// — outcomes, the first-try share, gate failures by phase, latency — is + /// eight sections that have no money in them at all. One page carrying both + /// would be thirteen sections deep and a reader looking for either would + /// scroll past the other. + Stats, /// Send what is queued into the turn that is already running. /// /// io-harness delivers it at the next step boundary, so the step in flight @@ -1558,12 +1610,27 @@ pub fn parse(input: &str, keys: &Keys, theme: &Theme) -> Action { // second screen: both open the one surface. "import" | "migrate" => Action::Import, // **An alias earns no row of its own.** `/usage` is what an operator - // coming from another agent types for "what is this costing me", and the - // answer is `/status` — which already commits the spend, the budgets and - // what is left of them. A second row for one screen reads as a second - // screen, so this is answered and never listed: it is absent from - // `COMMANDS`, from the palette and from every group. - "usage" => Action::Status, + // coming from another agent types for "what is this costing me", and a + // second row for one screen reads as a second screen — so this is + // answered and never listed: it is absent from `COMMANDS`, from the + // palette and from every group. + // + // **It answered `/status` until 0.22.0, and the argument for that was + // sound right up until it was not.** The note here read: the answer is + // `/status`, which already commits the spend, the budgets and what is left + // of them. That was the closest thing this program had to a spending + // surface, and it was a token draw against a ceiling rather than a cost. + // The moment a page exists whose whole subject is what has been spent, an + // operator typing the field's word for that page and landing on the + // session's configuration is being answered a question they did not ask. + // + // It stays an alias rather than becoming a third screen. `/usage` means + // plan and rate-limit status everywhere else it exists, and this product + // has no plan and no rate limit of its own to report — so a distinct + // `/usage` here would be a screen with nothing on it. + "usage" => Action::Cost, + "cost" => Action::Cost, + "stats" => Action::Stats, "config" | "settings" => { let mut rest = input.split_whitespace().skip(1); match rest.next() { diff --git a/src/configure.rs b/src/configure.rs index b9d3713..4ebf715 100644 --- a/src/configure.rs +++ b/src/configure.rs @@ -137,6 +137,14 @@ pub const CATALOGUE: &[&str] = &[ "app.io-cli.max_parallel_reads", "app.io-cli.spawn_background_after_secs", "app.io-cli.detached_spawns", + // What a turn costs, split across two sections because one of them is not + // ours. `prices.as_of` is io-harness's and dates the whole table; it is + // written by a fetch rather than typed, and it is listed here because a date + // an operator cannot see is a claim with no expiry they cannot check. + // `[prices.models]` is deliberately **not** a row: it is a list rather than a + // setting, and it is reached the way `/provider` and `/mcp` reach a list. + "prices.as_of", + "app.io-cli.prices.source_url", ]; /// Whether a key's value is a credential and must never be shown in full. diff --git a/src/context.rs b/src/context.rs index be60724..c50bf01 100644 --- a/src/context.rs +++ b/src/context.rs @@ -483,48 +483,11 @@ pub fn committed( /// `text` as rows no wider than `width`, indented two then four. /// -/// **Folded and never fitted**, for the reason `crate::status`'s own helper is: a -/// committed surface owns as many rows as it needs, and the characters a fitter -/// would drop here are tool names — which is to say the answer. A word longer -/// than its row is split rather than allowed to overflow, so eighty columns holds. +/// **The body moved to [`crate::page::folded`] in 0.22.0.** This was the second of +/// two copies of the same twenty lines — `crate::status` had the other, under the +/// name `folded`, taking as arguments the two indents this hard-coded — and +/// `/cost` and `/stats` would have made four. The two numbers stay here because +/// they are this surface's decision; the folding is not. fn wrapped(text: &str, width: usize) -> Vec { - let mut rows: Vec = Vec::new(); - let mut indent = 2usize; - let mut row = " ".repeat(indent); - let mut used = 0usize; - for word in text.split_whitespace() { - let mut word = word; - loop { - // At least one cell, so a terminal narrower than the indent still - // makes progress instead of looping forever. - let room = width.saturating_sub(indent).max(1); - let space = usize::from(used > 0); - let length = word.chars().count(); - if used + space + length <= room { - if space == 1 { - row.push(' '); - } - row.push_str(word); - used += space + length; - break; - } - if used > 0 { - rows.push(std::mem::take(&mut row)); - indent = 4; - row = " ".repeat(indent); - used = 0; - continue; - } - let head: String = word.chars().take(room).collect(); - word = &word[head.len()..]; - row.push_str(&head); - rows.push(std::mem::take(&mut row)); - indent = 4; - row = " ".repeat(indent); - } - } - if used > 0 || rows.is_empty() { - rows.push(row); - } - rows + crate::page::folded(text, width, 2, 4) } diff --git a/src/cost.rs b/src/cost.rs new file mode 100644 index 0000000..056a4f8 --- /dev/null +++ b/src/cost.rs @@ -0,0 +1,399 @@ +//! What has been spent: `/cost`, and `/usage` under the name the field uses. +//! +//! **Every number here has been in `runs.db` since io-harness 0.18.0 and nothing +//! has ever read one.** The harness records a row per provider call — the model, +//! the prompt, completion, cache-read, cache-write and reasoning token split, the +//! latency and the time to first token — and io-cli reported token counts and +//! called the money question unanswerable. It was answerable four releases ago. +//! +//! # Nothing is estimated +//! +//! Every figure is a row already in the store, or that row multiplied by a rate +//! the operator can point at. There is no sampling, no extrapolation from a +//! partial window and no model of what a token "usually" costs. A call the +//! provider reported no usage for is **unknown**, never zero; a model with no rate +//! in the table is **unpriced**, and a total containing one is a floor rather than +//! a total, which this page says out loud. io-harness's own pricing documentation +//! calls a renderer that hides that count "lying by omission", and it is right. +//! +//! # The three honesty rules, and why each is a rule +//! +//! **Cache reads and writes are a breakdown of the prompt, not an addition to +//! it.** `Usage::cache_read_tokens` and `cache_write_tokens` are already inside +//! `prompt_tokens`; adding them would over-report every cached turn, which is most +//! of them. So this page shows the prompt total and the three parts under it, and +//! the parts sum to the total rather than past it. +//! +//! **Reasoning tokens are a breakdown of completion, likewise** — a vendor that +//! reports reasoning separately still bills it as output, and on Anthropic the +//! count is zero because thinking is billed inside `output_tokens` and never +//! split out. +//! +//! **A `None` usage is not a zero usage.** io-harness stores `total_tokens` as +//! SQL `NULL` for a call the provider said nothing about, and reads it back as +//! `None` for exactly that reason. Summing it as zero would report a turn that +//! cost something as a turn that cost nothing. +//! +//! # The unit +//! +//! io-harness prices in micro-units and does not name a currency, because it is +//! not the layer that knows one. io-cli does: every catalogue it can read quotes +//! USD, and every provider it can connect to bills in it. So `$` is written, and +//! the page names the catalogue the rates came from and the date they were read, +//! which is the claim behind the symbol. + +use io_harness::pricing::{PriceTable, Spend}; +use io_harness::{ProviderCall, Store, Usage}; + +use crate::page::{self, Row}; +use crate::theme::Theme; + +// # What this page costs to draw, stated rather than discovered +// +// The `this run` and `this session` sections are bounded by construction: one +// run's calls, or one conversation's turns. The `by model` and `by day` sections +// are not. `Store::spend_by_model` and `Store::spend_by_day` each read **every** +// `provider_calls` row in the store and group in Rust — pricing is a Rust value +// SQLite cannot see, so it cannot be a `GROUP BY` — which makes both linear in the +// whole store and growing for the life of the install. +// +// **io-cli does not bound them, and the reason is that it cannot bound them +// honestly.** A `LIMIT` would have to be applied to rows rather than to groups, so +// a bounded read would report the most recent N calls under headings that say +// "by model" and "by day" without qualification — a partial total presented as a +// whole one, which is the exact failure the rest of this module is built to +// avoid. There is no public API to ask io-harness for the last N days grouped, and +// `ProviderCall` carries no timestamp of its own for io-cli to group by instead. +// +// So the release measures the cost against a realistically sized store, records +// the number, and leaves the read whole. The bound belongs upstream, beside the +// grouping, and is reported there. + +/// What one set of calls came to. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct Total { + /// Calls made, failed attempts included — a retry costs what it costs. + pub calls: u64, + /// Calls the provider reported no usage for. **Unknown, not zero.** + pub unknown: u64, + /// Calls whose model the table does not price. + pub unpriced: u64, + /// The token split, summed. Cache and reasoning are breakdowns of the prompt + /// and completion figures beside them, never additions to them. + pub usage: Usage, + /// Micro-units. A floor rather than a total whenever `unpriced` is non-zero. + pub micros: u64, +} + +impl Total { + /// Sum a run's calls, priced by `table`. + /// + /// `table` is never `Option`: an operator with no prices gets an empty table, + /// under which every call is unpriced and every token figure is still exact. + /// One code path rather than two, and the no-price case is the same page with + /// the money left off rather than a different page. + pub fn of(calls: &[ProviderCall], table: &PriceTable) -> Self { + let mut total = Self { + calls: calls.len() as u64, + ..Self::default() + }; + for call in calls { + let Some(usage) = call.usage else { + total.unknown += 1; + continue; + }; + add(&mut total.usage, &usage); + let priced = call + .model + .as_deref() + .and_then(|model| table.cost_micros(model, &usage)); + match priced { + Some(micros) => total.micros = total.micros.saturating_add(micros), + None => total.unpriced += 1, + } + } + total + } + + /// Whether the money figure is a floor rather than a total. + pub fn is_floor(&self) -> bool { + self.unpriced > 0 || self.unknown > 0 + } +} + +/// Add `from` into `into`, field by field. +/// +/// Written out rather than derived: `Usage` is `#[non_exhaustive]`-adjacent in +/// spirit — io-harness adds a dimension when a vendor starts billing one — and a +/// field added upstream that this function silently dropped would under-report a +/// bill. Spelling every field means the compiler names the new one. +fn add(into: &mut Usage, from: &Usage) { + into.prompt_tokens += from.prompt_tokens; + into.completion_tokens += from.completion_tokens; + into.total_tokens += from.total_tokens; + into.cache_read_tokens += from.cache_read_tokens; + into.cache_write_tokens += from.cache_write_tokens; + into.reasoning_tokens += from.reasoning_tokens; + into.server_tool_requests += from.server_tool_requests; +} + +/// Where the prices came from, for the line at the foot of the page. +/// +/// **The claim behind every `$` on the page**, and it is carried rather than +/// re-derived so that one sentence cannot drift from the figures above it. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Provenance { + /// The table's own `as_of`, or `None` when there is no table at all. + pub as_of: Option, + /// The catalogue, in words. `None` when nothing has been read yet. + pub source: Option, + /// How many models the table prices, as the last read recorded it. + pub models: usize, +} + +impl Provenance { + /// What the configuration in force says about its prices. + /// + /// The date comes from io-harness's own `[prices]` section, because that is + /// the table actually pricing the calls; the source and the count come from + /// `[app.io-cli.prices]`, because io-harness models neither. A file carrying + /// prices but no io-cli section — hand-written, or written by an older + /// release — still gets its date, and the page says the source was not + /// recorded rather than inventing one. + pub fn of(config: &io_harness::Config, stored: Option<&crate::settings::CliSettings>) -> Self { + let prices = stored.and_then(|settings| settings.prices.as_ref()); + Self { + as_of: config.prices().map(|table| table.as_of().to_string()), + source: prices.and_then(|p| p.source.clone()), + models: prices.and_then(|p| p.models).unwrap_or(0), + } + } +} + +/// The price table in force, or an empty one dated to nothing. +/// +/// **One code path rather than two.** An operator with no prices gets a table +/// that prices nothing, under which every call is unpriced and every token figure +/// is still exact — so the page is the same page with the money left off, rather +/// than a second page for the unconfigured case that would go stale unread. +pub fn table(config: &io_harness::Config) -> PriceTable { + config.prices().unwrap_or_else(|| PriceTable::new("")) +} + +/// The `/cost` page. +/// +/// `run` is the turn in flight or the last one; `session` is every turn of this +/// conversation. Both are `None` before anything has run, and the page says so +/// rather than drawing a row of zeroes — the same distinction the status line +/// holds every one of its counters to. +pub fn committed( + store: &Store, + table: &PriceTable, + provenance: &Provenance, + run: Option, + session: Option, + theme: &Theme, + width: u16, +) -> Result>, String> { + let mut rows: Vec = Vec::new(); + + if provenance.as_of.is_none() { + rows.push(Row::caveat( + "no prices are configured, so this page reports tokens and no money. \ + Prices arrive from the catalogue your provider serves when you connect \ + one, and `/config` refreshes them." + .to_string(), + )); + rows.push(Row::Blank); + } + + rows.push(Row::heading("this run".to_string())); + match run { + Some(id) => { + let calls = store.provider_calls(id).map_err(|e| e.to_string())?; + rows.extend(section(&Total::of(&calls, table))); + } + None => rows.push(Row::note("nothing has run in this session yet")), + } + + rows.push(Row::Blank); + rows.push(Row::heading("this session".to_string())); + match session { + Some(id) => { + let turns = store.session_turns(id).map_err(|e| e.to_string())?; + let mut calls: Vec = Vec::new(); + for turn in &turns { + calls.extend(store.provider_calls(turn.run_id).map_err(|e| e.to_string())?); + } + // **Walked rather than grouped, because io-harness offers no session + // grouping.** `spend_by_run` keys on a run and `spend_by_day` on a + // date; a session is neither, and reporting one run's cost under the + // heading of a whole conversation would be the wrong number under the + // right word. + rows.extend(section(&Total::of(&calls, table))); + rows.push(Row::note(format!( + "across {} turn{}", + turns.len(), + if turns.len() == 1 { "" } else { "s" } + ))); + } + None => rows.push(Row::note("this session has no turns yet")), + } + + rows.push(Row::Blank); + rows.push(Row::heading("by model".to_string())); + rows.extend(grouped( + store.spend_by_model(table).map_err(|e| e.to_string())?, + )); + + rows.push(Row::Blank); + rows.push(Row::heading("by day".to_string())); + rows.extend(grouped(store.spend_by_day(table).map_err(|e| e.to_string())?)); + + rows.push(Row::Blank); + rows.extend(footer(provenance)); + + Ok(page::commit("cost", &rows, theme, width)) +} + +/// The rows for one [`Total`]. +fn section(total: &Total) -> Vec { + if total.calls == 0 { + return vec![Row::note("no provider calls")]; + } + let usage = &total.usage; + let mut rows = vec![ + Row::fact("calls", total.calls.to_string()), + Row::fact("cost", money(total.micros)), + Row::fact("prompt", tokens(usage.prompt_tokens)), + ]; + // The two cache figures are indented under the prompt they are part of, by + // being named as parts of it rather than by being drawn further right — this + // page has no columns to indent into, and a reader in `--plain` has only the + // words. + if usage.cache_read_tokens > 0 || usage.cache_write_tokens > 0 { + rows.push(Row::fact( + " of which cache read", + tokens(usage.cache_read_tokens), + )); + rows.push(Row::fact( + " of which cache written", + tokens(usage.cache_write_tokens), + )); + rows.push(Row::fact( + " of which fresh", + tokens( + usage + .prompt_tokens + .saturating_sub(usage.cache_read_tokens) + .saturating_sub(usage.cache_write_tokens), + ), + )); + } + rows.push(Row::fact("completion", tokens(usage.completion_tokens))); + if usage.reasoning_tokens > 0 { + rows.push(Row::fact( + " of which reasoning", + tokens(usage.reasoning_tokens), + )); + } + if usage.server_tool_requests > 0 { + rows.push(Row::fact( + "provider tool requests", + usage.server_tool_requests.to_string(), + )); + } + rows.extend(caveats(total.unknown, total.unpriced)); + rows +} + +/// The rows for a set of io-harness's own [`Spend`] groups. +fn grouped(spend: Vec) -> Vec { + if spend.is_empty() { + return vec![Row::note("nothing recorded")]; + } + let mut rows: Vec = Vec::new(); + for group in spend { + let mut value = format!( + "{} · {} call{}", + money(group.cost_micros), + group.calls, + if group.calls == 1 { "" } else { "s" } + ); + if group.usage.total_tokens > 0 { + value.push_str(&format!(" · {} tok", tokens(group.usage.total_tokens))); + } + if group.unpriced_calls > 0 { + // **Never hidden, and on the row rather than in a note below it.** A + // reader scanning for the largest figure has to be able to see which + // of them is incomplete without reading past the list. + value.push_str(&format!(" · {} unpriced", group.unpriced_calls)); + } + rows.push(Row::fact(group.key, value)); + } + rows +} + +/// The sentences that qualify a figure, if any qualify it. +fn caveats(unknown: u64, unpriced: u64) -> Vec { + let mut rows: Vec = Vec::new(); + if unpriced > 0 { + rows.push(Row::caveat(format!( + "{unpriced} call{} used a model with no rate in the price table, so \ + the cost above is a floor and not a total", + if unpriced == 1 { "" } else { "s" } + ))); + } + if unknown > 0 { + rows.push(Row::caveat(format!( + "{unknown} call{} reported no usage at all, which is unknown rather \ + than free — neither its tokens nor its cost is in the figures above", + if unknown == 1 { " " } else { "s " } + ))); + } + rows +} + +/// Where the prices came from and when, at the foot of the page. +fn footer(provenance: &Provenance) -> Vec { + match (&provenance.as_of, &provenance.source) { + (Some(as_of), Some(source)) => vec![Row::note(format!( + "prices: {} model{} read from {source} on {as_of}", + provenance.models, + if provenance.models == 1 { "" } else { "s" } + ))], + (Some(as_of), None) => vec![Row::note(format!( + "prices: {} model{}, dated {as_of}, from a source this install did not record", + provenance.models, + if provenance.models == 1 { "" } else { "s" } + ))], + _ => vec![Row::note( + "prices: none configured".to_string(), + )], + } +} + +/// Micro-units as money. +/// +/// Four decimals below a unit and two above it: a turn costs a fraction of a cent +/// and a month costs dollars, and one precision cannot show both without either +/// rounding the turn to nothing or padding the month with noise. Integer +/// arithmetic throughout — a bill is not a floating-point quantity, and +/// `cost_micros` is exact. +pub fn money(micros: u64) -> String { + if micros == 0 { + return "$0".to_string(); + } + let units = micros / 1_000_000; + let rest = micros % 1_000_000; + if units == 0 { + format!("$0.{:04}", rest / 100) + } else { + format!("${units}.{:02}", rest / 10_000) + } +} + +/// A token count, in the spelling the status line already uses. +fn tokens(count: u64) -> String { + crate::status::format_tokens(count) +} diff --git a/src/lib.rs b/src/lib.rs index 1717955..7aa00d8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -41,6 +41,7 @@ pub mod composer; pub mod configure; pub mod context; pub mod contract; +pub mod cost; pub mod diff; pub mod edit; pub mod editor; @@ -57,10 +58,12 @@ pub mod intent; pub mod keys; pub mod markdown; pub mod memory; +pub mod page; pub mod picker; pub mod picture; pub mod plan; pub mod pluginview; +pub mod prices; pub mod provider; pub mod providers; pub mod queue; @@ -74,6 +77,7 @@ pub mod shell; pub mod skills; pub mod skillview; pub mod splash; +pub mod stats; pub mod status; pub mod stdin; pub mod term; diff --git a/src/main.rs b/src/main.rs index 0becfde..b4103d1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2822,6 +2822,38 @@ async fn loop_over Result>( ); screen.commit(&lines).map_err(|error| error.to_string())?; } + // The sibling of `Action::Status` above and wired the same way, + // with one difference worth stating: this arm holds no decision + // at all. Every sentence, every figure and every caveat is + // `io_cli::cost::committed`, because nothing under `tests/` can + // link this file — so a format string here would be a format + // string no test could ever read. + Action::Cost => { + // Asked of the configuration again rather than threaded down + // from the driver's own binding, for the reason stated where + // `settings_in_force` is: `settings::stored` is a pure + // function of the config, so a second call cannot disagree + // with the first, and a threaded copy could go stale behind a + // `/config` write that reloaded one and not the other. + let (settings, _) = settings::stored(&config); + let table = io_cli::cost::table(&config); + let provenance = io_cli::cost::Provenance::of(&config, settings.as_ref()); + let lines = io_cli::cost::committed( + &store, + &table, + &provenance, + last_run(&session, &store).map(|turn| turn.run_id), + Some(session.id()), + &app.theme, + screen.width(), + )?; + screen.commit(&lines).map_err(|error| error.to_string())?; + } + Action::Stats => { + let lines = + io_cli::stats::committed(&store, &app.theme, screen.width())?; + screen.commit(&lines).map_err(|error| error.to_string())?; + } // Reached only at an idle prompt: while a turn runs the driver's // own key handler answers `/steer` before `parse` is ever called, // because that is where the inbox lives. So this arm is the @@ -3519,6 +3551,7 @@ async fn turn( // same turn. The trace is the fallback for the window between a // step landing and the first completion call being seen. note_context(app, store, &event, seen, &contract); + note_cost(app, store, config, event.run_id); note_fleet(app, store, &event, &contract); commit_viewed(screen, app, &root, policy, &event)?; commit_fold(app, store, &event, &mut folding); @@ -3854,6 +3887,10 @@ async fn turn( // for: the last step of a turn is exactly the one whose event the select // loop loses to the turn's own return. note_context(app, store, &event, seen, &contract); + // And on the drain too, for the same race: the last step of a turn is the + // one whose event the select loop loses, and it is also the step that + // makes the largest single difference to what the turn cost. + note_cost(app, store, config, event.run_id); note_fleet(app, store, &event, &contract); // And the picture, for the same reason and the same race: a `view_image` // on the turn's last step is exactly the one the drain would otherwise @@ -4082,6 +4119,23 @@ fn note_context( } } +/// Set the status line's cost field from what this run has actually called. +/// +/// Beside [`note_context`] and on the same events, because the two are the same +/// shape of fact: both are read from the store rather than accumulated off the +/// stream, both change only when a step lands, and both are decorative enough +/// that a failed read is a missing field rather than a notice. +/// +/// **The table is asked of the configuration on every call rather than held.** A +/// `/config` write or a price refresh mid-session changes what a turn costs, and a +/// table captured when the session opened would go on reporting the old rate for +/// the rest of it — which is the one thing a figure with a currency in front of it +/// must not do. +fn note_cost(app: &mut App, store: &Store, config: &io_harness::Config, run_id: i64) { + let table = io_cli::cost::table(config); + app.status.note_cost_from(store, run_id, &table); +} + /// Report a fold that was asked for — once, and only from the event that /// announces it. /// diff --git a/src/page.rs b/src/page.rs new file mode 100644 index 0000000..86ebfae --- /dev/null +++ b/src/page.rs @@ -0,0 +1,190 @@ +//! The shape of a page committed into the scrollback. +//! +//! **Four surfaces draw one, and until 0.22.0 two of them each carried their own +//! copy of the folding.** `crate::status::committed` and +//! `crate::context::committed` had the same twenty lines under two names, +//! `folded` and `wrapped`, differing only in that one took its indents as +//! arguments and the other hard-coded the same two numbers. `/cost` and `/stats` +//! would have been the third and fourth copies, so the folding lives here and +//! all four call it. +//! +//! # What a committed page is +//! +//! Three rule glyphs and a title, one fact per row, three rule glyphs and the +//! title again with `ends`. It lands in the terminal's own scrollback beside every +//! earlier turn, which is why it has edges at all: a passage with no edges is one +//! a reader cannot tell the extent of. +//! +//! **It is not a table, and that is a decision about eighty columns rather than +//! about taste** — the argument is written out in full at +//! `crate::status::committed` and it holds identically here. A table has a column +//! width, a column width is decided by the widest cell, and on these two pages the +//! widest cell is a model id. One fact per row, `label: value`, nothing padded +//! into a column and nothing aligned across rows, so there is no width for +//! anything to be squeezed out of. A row too long for the terminal is **folded**, +//! never cut. +//! +//! # Sections, which the older two pages did not need +//! +//! `/status` and `/context` are each one list. `/cost` and `/stats` are seven and +//! eight lists respectively — this run, this session, by model, by day — and a +//! reader has to be able to tell which figure belongs to which question. So a +//! page here is a sequence of [`Row`]s rather than a flat vector of pairs, and a +//! [`Row::Heading`] is what separates them. It is drawn in the accent tone and +//! indented to zero, so it reads as a break rather than as another fact. + +use ratatui::text::{Line, Span}; + +use crate::theme::{Theme, Tone}; + +/// One row of a committed page. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Row { + /// `label: value`, folded at two columns and continued at four. + Fact(String, String), + /// A section break within the page. + Heading(String), + /// A whole sentence, in a tone of its own. + /// + /// What a page says when it has no figure to give: that the price table is + /// empty, that a total is a floor because some calls were unpriced, that a + /// provider reported no usage for a call. **These are the rows that make the + /// figures trustworthy**, so they are rows rather than a footnote somewhere + /// else, and they carry a tone because "this total is incomplete" is not the + /// same kind of statement as "this total is 3,412 tokens". + Note(String, Tone), + /// A blank row, for separating a heading from what came before it. + Blank, +} + +impl Row { + /// `label: value`, taking anything that can spell itself. + pub fn fact(label: impl Into, value: impl Into) -> Self { + Self::Fact(label.into(), value.into()) + } + + /// A section break. + pub fn heading(text: impl Into) -> Self { + Self::Heading(text.into()) + } + + /// A sentence in the ordinary tone. + pub fn note(text: impl Into) -> Self { + Self::Note(text.into(), Tone::Muted) + } + + /// A sentence that qualifies a figure above it. + /// + /// `Warning` rather than `Muted`, because a total that is a floor and a total + /// that is a total must not read the same at a glance. This is the tone the + /// "lying by omission" rule in io-harness's own pricing documentation exists + /// to prevent a renderer from skipping. + pub fn caveat(text: impl Into) -> Self { + Self::Note(text.into(), Tone::Warning) + } +} + +/// A whole page, edged and folded, ready for `Screen::commit`. +/// +/// `title` appears in both rules, the closing one suffixed with `ends`, which is +/// the edge `crate::status::committed` and `crate::transcript` already give a +/// committed passage. +pub fn commit(title: &str, rows: &[Row], theme: &Theme, width: u16) -> Vec> { + let rule = theme.glyphs.rule; + let room = width as usize; + let mut lines = vec![Line::from(Span::styled( + format!("{rule}{rule}{rule} {title}"), + theme.style(Tone::Accent), + ))]; + for row in rows { + match row { + Row::Fact(label, value) => { + for text in folded(&format!("{label}: {value}"), room, 2, 4) { + lines.push(Line::from(Span::styled(text, theme.style(Tone::Normal)))); + } + } + // Indented to zero rather than to two, so a heading is distinguishable + // from a fact by position as well as by colour — which is the whole of + // what a reader in `--plain` or under `NO_COLOR` has to go on. + Row::Heading(text) => { + for text in folded(text, room, 0, 2) { + lines.push(Line::from(Span::styled(text, theme.style(Tone::Accent)))); + } + } + Row::Note(text, tone) => { + for text in folded(text, room, 2, 4) { + lines.push(Line::from(Span::styled(text, theme.style(*tone)))); + } + } + Row::Blank => lines.push(Line::from(String::new())), + } + } + lines.push(Line::from(Span::styled( + format!("{rule}{rule}{rule} {title} ends"), + theme.style(Tone::Accent), + ))); + lines +} + +/// `text` as rows no wider than `width`, indented `first` and then `rest`. +/// +/// **Folded and never fitted**, which is the one thing this differs in from every +/// other width-aware helper in this crate. [`crate::picker::fit`] and +/// [`crate::status::Status::line`] shorten, because a picker row and a status line +/// each own exactly one row of a viewport that cannot grow. A committed surface +/// owns as many rows as it needs, so there is no reason left to lose a character — +/// and the characters most likely to be lost are the tail of a workspace path, the +/// tail of a policy layer's act list and the tail of a model id, which is to say +/// the answer. +/// +/// Broken at spaces, with a hanging indent that says a row is a continuation +/// without aligning anything into a column. A word longer than the room it has — +/// a deep path, an absurd model name — is **split** rather than allowed to +/// overflow: eighty columns is a supported size, and a row that runs past it gets +/// wrapped by the terminal at a place nothing here chose. +pub fn folded(text: &str, width: usize, first: usize, rest: usize) -> Vec { + let mut rows: Vec = Vec::new(); + let mut indent = first; + let mut row = " ".repeat(indent); + // Content characters on this row, which is its width less the indent. Counted + // rather than measured off `row` so the arithmetic is in one unit — the same + // reason `Status::fits` counts characters and not bytes. + let mut used = 0usize; + for word in text.split_whitespace() { + let mut word = word; + loop { + // At least one cell, so a terminal narrower than the indent still + // makes progress instead of looping forever. + let room = width.saturating_sub(indent).max(1); + let space = usize::from(used > 0); + let length = word.chars().count(); + if used + space + length <= room { + if space == 1 { + row.push(' '); + } + row.push_str(word); + used += space + length; + break; + } + if used > 0 { + rows.push(std::mem::take(&mut row)); + indent = rest; + row = " ".repeat(indent); + used = 0; + // Retried whole on the fresh row: a word is only ever split when + // a row of its own cannot hold it. + continue; + } + let head: String = word.chars().take(room).collect(); + word = &word[head.len()..]; + row.push_str(&head); + rows.push(std::mem::take(&mut row)); + indent = rest; + row = " ".repeat(indent); + } + } + if used > 0 || rows.is_empty() { + rows.push(row); + } + rows +} diff --git a/src/prices.rs b/src/prices.rs new file mode 100644 index 0000000..7f30652 --- /dev/null +++ b/src/prices.rs @@ -0,0 +1,285 @@ +//! Where a price comes from, what it claims, and when it was read. +//! +//! **io-cli compiles no prices in, and this module is the reason it does not have +//! to.** io-harness prices a call from a [`PriceTable`] the operator's +//! configuration carries, and that table has to be filled from somewhere. Filling +//! it from a list baked into the binary would be a promise the binary cannot keep: +//! providers move prices without announcing it, a release cadence is not a pricing +//! cadence, and an operator reading a confident wrong number is worse off than one +//! reading no number at all. So the table is filled from the catalogue the +//! operator's own provider serves — which io-cli was already fetching and already +//! throwing away. +//! +//! # The call was already there +//! +//! [`crate::verify::catalogue`] has read the model catalogue since 0.1.0, to offer +//! the wizard a list of models. It mapped every [`ModelInfo`] down to its `id` and +//! discarded the `price`, `price_tiers` and `price_source` on the same row. This +//! module keeps them. No JSON is parsed here and no dependency is added: io-harness +//! did the parsing and the unit conversion, and `ModelInfo::price` arrives already +//! normalised into [`Price`]'s micro-units per million tokens. +//! +//! # Whose price it is +//! +//! For three of the four providers io-cli knows, the answer is not "the vendor's". +//! OpenAI, Anthropic and Google publish no prices on any endpoint — their model +//! endpoints carry capabilities and limits and no cost field, and their cost APIs +//! report what was *spent* rather than what a token *costs*. The reference +//! catalogue does carry rows for their models, which is why +//! [`crate::verify::catalogue`] strips the `anthropic/` and `openai/` prefixes off +//! them to build its list. +//! +//! So an operator on Anthropic gets prices *about* Anthropic *from* the reference +//! catalogue, and every surface that draws money says which. io-harness models the +//! distinction already — [`PriceSource::Vendor`] against +//! [`PriceSource::Reference`] — and this module carries it through rather than +//! flattening it into the connected provider's name. On OpenRouter the two +//! coincide, which [`crate::verify::catalogue`] already says in as many words: the +//! reference catalogue is OpenRouter's own, so for OpenRouter it is not a +//! reference at all. +//! +//! # Two files, because one of them is not ours +//! +//! The table itself lives under `[prices]`, which io-harness owns and reads +//! through `Config::prices`. That section is `deny_unknown_fields` and carries +//! exactly `as_of` and `models` — so a key of io-cli's own put beside them would +//! not be ignored, it would make the operator's whole configuration file +//! unreadable. Anything this module needs to remember that io-harness does not +//! model therefore lives under `[app.io-cli.prices]`, which is the section +//! io-harness deliberately does not validate. +//! +//! # Nothing here reads a clock +//! +//! `tests/timing.rs` permits `SystemTime::now` in the driver and nowhere else, so +//! the date a table is stamped with is handed in rather than taken. [`date`] is +//! the conversion, and it is a pure function of a number for that reason as much +//! as for testability. + +use io_harness::pricing::Price; +use io_harness::{ModelInfo, PriceSource, ProviderSpec}; + +use crate::edit::Edit; + +/// The catalogue io-harness reads when nothing names another. +/// +/// Re-exported rather than re-spelled: a second copy of this URL in this +/// repository would be a second thing to move when io-harness moves it. +pub use io_harness::provider::catalog::DEFAULT_REFERENCE_URL; + +/// A catalogue read: what it priced, where it came from, and when. +/// +/// `rows` is already filtered to the models the connected provider actually +/// serves and spelled the way that provider names them — the same filter and the +/// same stripping [`crate::verify::catalogue`] applies, because the key a price is +/// stored under has to match the `model` io-harness records on a provider call, +/// and that is the name the operator configured. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Catalogue { + /// The date this was read, `YYYY-MM-DD`. Not a date any provider supplied: + /// every timestamp on offer anywhere is a model *release* date, and no + /// provider publishes when it last changed a price. + pub as_of: String, + /// Where the rows came from, in io-harness's own terms. + pub source: PriceSource, + /// One row per priced model. A model the catalogue served with no price is + /// **absent**, never entered at zero, so `Spend::unpriced_calls` counts it + /// and the page that draws it can say the total is a floor. + pub rows: Vec<(String, Price)>, + /// How many models the catalogue served in total, priced or not. Kept + /// separately from `rows.len()` because a catalogue that answered with plenty + /// of models and no prices is a different failure from one that answered with + /// nothing, and the operator is owed the difference. + pub served: usize, +} + +impl Catalogue { + /// The rows a catalogue read yields, filtered and named for `spec`. + /// + /// `models` is what the catalogue served; `as_of` is the date the caller + /// read it. Separated from the fetch so the whole of this decision is + /// testable without a socket — the fetch itself is one io-harness call and + /// has nothing in it worth a test double. + pub fn of(spec: &ProviderSpec, models: Vec, as_of: impl Into) -> Self { + let served = models.len(); + let vendor = matches!(spec, ProviderSpec::OpenRouter { .. }); + let rows = crate::verify::priced(spec, models); + Self { + as_of: as_of.into(), + // **The one place the two coincide.** The reference catalogue is + // OpenRouter's own, so for an OpenRouter operator it is the vendor + // speaking for itself and calling it a reference would understate it. + // For everyone else it is a third party and saying otherwise would + // attribute a number to a vendor that never published one. + source: if vendor { + PriceSource::Vendor + } else { + PriceSource::Reference(DEFAULT_REFERENCE_URL.to_string()) + }, + rows, + served, + } + } + + /// Whether this read is too short to replace `existing` rows. + /// + /// **The one failure in this release that loses money quietly.** A truncated + /// or partial catalogue response that replaced a full table with a handful of + /// rows would turn most of an operator's spending into "unpriced" and shrink + /// their reported bill without anything failing. So a replacement that comes + /// back empty, or far shorter than what it would replace, is refused and the + /// old table kept. + /// + /// A first fill has nothing to compare against and is never refused: an + /// operator with no table has nothing to lose. + pub fn too_short(&self, existing: usize) -> bool { + if existing == 0 { + return false; + } + self.rows.is_empty() || self.rows.len() * 2 < existing + } + + /// The edits that write this catalogue into a configuration file. + /// + /// One `set` for the date and one per model, rather than one edit rewriting + /// the section whole. That is not a smaller diff for its own sake: a rate is + /// something the operator is invited to correct by hand, and a row per line is + /// what makes correcting one a one-line change they can find again. + /// + /// **Rows for models the catalogue no longer serves are left alone.** They are + /// not stale — io-harness prices a call by the model name on it, so an old row + /// is what prices an old run correctly, and `/cost` reports history as well as + /// today. + pub fn edits(&self) -> Vec { + let mut edits = vec![Edit::set("prices.as_of", quoted(&self.as_of))]; + for (model, price) in &self.rows { + edits.push(Edit::set( + format!("prices.models.{}", quoted(model)), + inline(price), + )); + } + edits + } +} + +/// What a refresh would change about one model's rate. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Change { + /// The model, spelled as the price table keys it. + pub model: String, + /// What the table in force says now, or `None` for a model it does not price. + pub was: Option, + /// What the catalogue served. + pub now: Price, +} + +/// Every rate a refresh would move, against the table in force. +/// +/// **Shown before anything is written, which is why this is separate from +/// [`Catalogue::edits`].** io-cli cannot tell a rate the operator corrected by +/// hand from one an older catalogue served — the file records a number and not +/// where it came from — so it does not guess. It shows what would move and lets +/// the operator decline the lot, which is the shape `/import` established in +/// 0.21.0: everything shown before anything is written. +/// +/// A model the table does not price yet is a change with `was: None`. A rate that +/// has not moved is not a change and is not listed, so a refresh that found +/// nothing new says so in one line rather than in four hundred. +pub fn changes(existing: Option<&io_harness::pricing::PriceTable>, fresh: &Catalogue) -> Vec { + fresh + .rows + .iter() + .filter_map(|(model, now)| { + let was = existing.and_then(|table| table.price(model)); + (was != Some(*now)).then(|| Change { + model: model.clone(), + was, + now: *now, + }) + }) + .collect() +} + +/// A `Price` as a one-line TOML inline table, naming only what is charged for. +/// +/// A dimension the vendor does not charge for is left out rather than written as +/// a zero, because `Price` defaults every field and a file that spells three +/// zeros to say "nothing" is three lines an operator has to read to learn nothing. +/// `..Price::ZERO` is io-harness's own advice for constructing one, and this is +/// the same decision spelled in TOML. +fn inline(price: &Price) -> String { + let mut parts: Vec = Vec::new(); + for (name, value) in [ + ("input", price.input), + ("output", price.output), + ("cache_read", price.cache_read), + ("cache_write", price.cache_write), + ("per_server_tool_request", price.per_server_tool_request), + ] { + if value != 0 { + parts.push(format!("{name} = {value}")); + } + } + if parts.is_empty() { + // A model every dimension of which is free still has to round-trip as a + // `Price`, and `{}` is the inline table that means exactly that. + return "{}".to_string(); + } + format!("{{ {} }}", parts.join(", ")) +} + +/// A string as TOML source, escaped by the TOML crate rather than by hand. +/// +/// Model ids carry dots, slashes and colons, and a dot in a key can only be +/// spelled quoted — bare keys are letters, digits, `_` and `-` and nothing else. +/// So this is not decoration: an unquoted `gpt-4.1` is two path segments and +/// reaches the wrong place, or no place. `edit::array` sets the precedent of +/// spelling TOML through `toml` rather than with `format!`. +fn quoted(text: &str) -> String { + toml::Value::String(text.to_string()).to_string() +} + +/// `YYYY-MM-DD`, UTC, from a count of seconds since the Unix epoch. +/// +/// **A pure function of a number, and that is a gate rather than a preference.** +/// `tests/timing.rs` permits `SystemTime::now` in the driver and refuses it in +/// every other file under `src/`, so the driver reads the clock and hands the +/// number here. It also happens to be the only shape in which a date conversion +/// can be tested without owning the machine's clock. +/// +/// Howard Hinnant's `civil_from_days`, whose era begins on 0000-03-01 so that the +/// leap day falls at the end of a year and the month arithmetic needs no table. +/// Correct for every date the epoch can express; io-cli has no use for one before +/// 1970 and does not pretend to handle one. +/// +/// ``` +/// assert_eq!(io_cli::prices::date(0), "1970-01-01"); +/// assert_eq!(io_cli::prices::date(1_772_150_400), "2026-02-27"); +/// ``` +pub fn date(secs: u64) -> String { + let days = (secs / 86_400) as i64; + let z = days + 719_468; + let era = if z >= 0 { z } else { z - 146_096 } / 146_097; + let doe = (z - era * 146_097) as i64; + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let year = yoe + era * 400; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let day = doy - (153 * mp + 2) / 5 + 1; + let month = if mp < 10 { mp + 3 } else { mp - 9 }; + let year = if month <= 2 { year + 1 } else { year }; + format!("{year:04}-{month:02}-{day:02}") +} + +/// How a price source reads on a surface that draws money. +/// +/// `PriceSource` is `#[non_exhaustive]`, so the `_` arm is required rather than +/// defensive — and it says "a catalogue" rather than guessing, because a variant +/// io-harness gains and this release has not seen is one whose name io-cli would +/// be inventing. +pub fn source_word(source: &PriceSource) -> String { + match source { + PriceSource::Vendor => "the provider's own catalogue".to_string(), + PriceSource::Reference(url) => format!("the reference catalogue at {url}"), + _ => "a catalogue this release does not recognise".to_string(), + } +} diff --git a/src/settings.rs b/src/settings.rs index 3dba157..c1606d5 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -126,6 +126,53 @@ pub struct CliSettings { /// child's whole life in it, which is what a detached child gives up. #[serde(default, skip_serializing_if = "Option::is_none")] pub detached_spawns: Option, + + /// What io-cli remembers about prices: `[app.io-cli.prices]`. + /// + /// **The table itself is not here, and that is not a filing decision.** The + /// prices live under `[prices]`, which io-harness owns and reads through + /// `Config::prices`. That section is `deny_unknown_fields` and carries exactly + /// `as_of` and `models`, so a key of io-cli's own put beside them would not be + /// ignored — it would make the operator's whole configuration file fail to + /// parse, taking the policy, the providers and the run budgets down with it. + /// `[app.io-cli]` is the one section io-harness deliberately does not + /// validate, so anything io-cli needs to remember that the harness does not + /// model belongs here and nowhere else. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub prices: Option, +} + +/// Where prices come from, and what the last read was. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct PriceSettings { + /// The catalogue to read: `source_url = "..."`. + /// + /// Absent means io-harness's own default. This key is the only way an + /// operator on a self-hosted or `compatible` endpoint gets prices at all: the + /// reference catalogue cannot speak for a server it has never heard of, and + /// `Reference::at` takes any URL serving the shape. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_url: Option, + /// What the last read was, in words, for the surfaces that draw money. + /// + /// Written by a fetch rather than by hand. It records whether the rates came + /// from the provider speaking for itself or from a third party's catalogue, + /// which for three of the four providers io-cli knows is the second — OpenAI, + /// Anthropic and Google publish no prices on any endpoint. A page that drew a + /// figure without saying which would be attributing a number to a vendor that + /// never published one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source: Option, + /// How many models the last read priced. + /// + /// **Bookkeeping io-harness does not model, which is what this section is + /// for.** `PriceTable` has no length and no iterator — it answers `price(model)` + /// and nothing else — so io-cli cannot count what it holds by asking it. The + /// count is needed twice: to say on `/cost` how many models the rates cover, + /// and to refuse a refetch that comes back far shorter than what it would + /// replace, which is the one failure in this area that loses money quietly. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub models: Option, } /// The caps this session runs its turns under, if any. @@ -447,6 +494,13 @@ pub fn render( max_parallel_reads: None, spawn_background_after_secs: None, detached_spawns: None, + // Left out because the wizard has not read a catalogue yet when + // it renders this. Prices arrive from the read that follows the + // credential check, as their own edits against the file this + // wrote — which is also what keeps the shape identical for an + // operator who runs the wizard and one who adds a provider from + // `/provider` later. + prices: None, }, }, }; diff --git a/src/stats.rs b/src/stats.rs new file mode 100644 index 0000000..ec5f0ae --- /dev/null +++ b/src/stats.rs @@ -0,0 +1,243 @@ +//! How the work has gone: `/stats`. +//! +//! **The other half of the question `/cost` asks.** `/cost` says what the work +//! cost; this says whether it worked. They are the field's own two names and they +//! answer different questions, which is why every agent that has both keeps them +//! apart — and why `/usage` is an alias for `/cost` here rather than a third +//! screen, since `/usage` means plan and rate-limit status everywhere it exists +//! and this product has no plan to report. +//! +//! Every figure is a `Store` call io-cli has never made: `runs_by_outcome`, +//! `runs_by_day`, `first_try`, `gate_failures_by_phase`, `recovery` and +//! `store_size`, plus the latencies read off the provider calls themselves. +//! +//! # First-try is io-harness's definition, not one derived here +//! +//! It is tempting to count successful runs and call it a first-try rate. That +//! would count a run which failed three gates and then passed as a first-try +//! success. io-harness's `first_try` is finished **and** successful **and** +//! carrying no `gate_phase_failed` event, which is the question an operator is +//! actually asking, and it is a `Store` method precisely so nobody re-derives it +//! wrongly. +//! +//! `FirstTry` is three counts and deliberately not a rate, because the denominator +//! is a choice: first-try over every run counts runs that are still going, and +//! first-try over successful runs answers a different question again. This page +//! picks one and **names it in the row**, rather than printing a percentage whose +//! meaning a reader has to guess. +//! +//! # The two gate vocabularies stay apart +//! +//! io-harness records gate failures twice, in two tables, with two phase +//! vocabularies that do not overlap. `SandboxEvent` phases are `subject-compile`, +//! `criterion-compile`, `test-run` and `subject-emptied`; `GateAttempt` phases are +//! `review`, `command` and `contains`. They are different mechanisms and merging +//! them into one list would produce a chart whose categories mean two things. So +//! they get two headings. +//! +//! # Nothing here compacts anything +//! +//! `Store::compact` is a full `VACUUM`: it needs free disk roughly equal to the +//! database file, rewrites the whole thing, and is not quick. A page that reported +//! free space and then reclaimed it on its own would be a page that surprised +//! somebody mid-session. The free figure is reported and the reclaiming is not +//! offered here. + +use io_harness::Store; + +use crate::page::{self, Row}; +use crate::theme::Theme; + +/// How many of the most recent runs the latency sections read. +/// +/// **A bound this module can actually keep, unlike the one `/cost` could not.** +/// `Store::runs` returns run ids and io-cli takes the tail of that list itself, so +/// the sample is genuinely the last `N` runs and the page says so in the heading. +/// That is the difference between a bound and a truncation: a heading reading +/// "slowest calls of the last 200 runs" is a true statement about a subset, where +/// "by model" over a truncated read would have been a false statement about the +/// whole. +/// +/// The same shape and the same reason as `crate::sessions::MAX_RUNS_SCANNED`. +pub const RECENT_RUNS: usize = 200; + +/// How many slow calls to name. +const SLOWEST: usize = 5; + +/// The `/stats` page. +pub fn committed( + store: &Store, + theme: &Theme, + width: u16, +) -> Result>, String> { + let mut rows: Vec = Vec::new(); + + rows.push(Row::heading("runs by outcome".to_string())); + rows.extend(tallies( + store.runs_by_outcome().map_err(|e| e.to_string())?, + "no runs recorded", + )); + + rows.push(Row::Blank); + rows.push(Row::heading("runs by day".to_string())); + rows.extend(tallies( + store.runs_by_day().map_err(|e| e.to_string())?, + "no runs recorded", + )); + + rows.push(Row::Blank); + rows.push(Row::heading("finished on the first try".to_string())); + let first = store.first_try().map_err(|e| e.to_string())?; + rows.push(Row::fact("runs finished", first.runs.to_string())); + rows.push(Row::fact("of those, succeeded", first.succeeded.to_string())); + rows.push(Row::fact( + "of those, with no gate failure", + first.first_try.to_string(), + )); + // **The denominator is named in the row, not implied by a percent sign.** + // io-harness gives three counts and refuses to pick one for exactly this + // reason; a bare `73%` would be a number whose meaning the reader has to + // reconstruct, and two readers would reconstruct it differently. + if first.succeeded > 0 { + rows.push(Row::fact( + "first-try share of successful runs", + format!("{}%", first.first_try * 100 / first.succeeded), + )); + } + + rows.push(Row::Blank); + rows.push(Row::heading("gate failures by phase".to_string())); + rows.extend(tallies( + store.gate_failures_by_phase().map_err(|e| e.to_string())?, + "no gate has failed", + )); + rows.push(Row::note( + "these are the sandbox gate's phases; the review, command and contains \ + gates record separately and are not merged into this list" + .to_string(), + )); + + rows.push(Row::Blank); + rows.push(Row::heading("recovery".to_string())); + let recovery = store.recovery().map_err(|e| e.to_string())?; + rows.push(Row::fact("fallbacks", recovery.fallbacks.to_string())); + rows.push(Row::fact("replans", recovery.replans.to_string())); + rows.push(Row::fact("resumes", recovery.resumes.to_string())); + + let ids = store.runs().map_err(|e| e.to_string())?; + let scanned = ids.len().min(RECENT_RUNS); + let recent = &ids[ids.len() - scanned..]; + let mut calls = Vec::new(); + for id in recent { + calls.extend(store.provider_calls(*id).map_err(|e| e.to_string())?); + } + + rows.push(Row::Blank); + rows.push(Row::heading(format!( + "slowest calls, of the last {scanned} run{}", + if scanned == 1 { "" } else { "s" } + ))); + let mut slow: Vec<(u64, String)> = calls + .iter() + .map(|call| { + ( + call.latency_ms, + call.model.clone().unwrap_or_else(|| call.provider.clone()), + ) + }) + .collect(); + slow.sort_by(|a, b| b.0.cmp(&a.0)); + if slow.is_empty() { + rows.push(Row::note("no provider calls in those runs")); + } else { + for (ms, model) in slow.iter().take(SLOWEST) { + rows.push(Row::fact(model.clone(), format!("{ms} ms"))); + } + } + + rows.push(Row::Blank); + rows.push(Row::heading("time to first token".to_string())); + // **An unmeasured time to first token is `None`, never 0**, and it is counted + // separately rather than averaged in as a zero — which would drag every mean + // toward an instant nothing ever took. + let measured: Vec = calls.iter().filter_map(|call| call.ttft_ms).collect(); + if measured.is_empty() { + rows.push(Row::note( + "no call in those runs reported one, which is unmeasured rather than instant", + )); + } else { + let mut sorted = measured.clone(); + sorted.sort_unstable(); + rows.push(Row::fact( + "median", + format!("{} ms", sorted[sorted.len() / 2]), + )); + rows.push(Row::fact("slowest", format!("{} ms", sorted[sorted.len() - 1]))); + rows.push(Row::fact( + "measured on", + format!("{} of {} calls", measured.len(), calls.len()), + )); + } + + rows.push(Row::Blank); + rows.push(Row::heading("what the store holds".to_string())); + let size = store.store_size().map_err(|e| e.to_string())?; + rows.push(Row::fact("file", bytes(size.file_bytes))); + rows.push(Row::fact("free inside it", bytes(size.free_bytes))); + rows.push(Row::fact("sessions", size.sessions.to_string())); + rows.push(Row::fact("runs", size.runs.to_string())); + // **Absent rather than zeroed.** io-harness returns an empty table list when + // the SQLite build lacks `dbstat`, which is a build that cannot answer the + // question — not a database whose tables are all empty. + if size.tables.is_empty() { + rows.push(Row::note( + "this SQLite build cannot break the file down by table", + )); + } else { + for (name, count) in size.tables.iter().take(5) { + rows.push(Row::fact(name.clone(), bytes(*count))); + } + } + if size.free_bytes > 0 { + rows.push(Row::note(format!( + "{} could be returned to the filesystem, which is a full rewrite of the \ + database and is not done from this page", + bytes(size.free_bytes) + ))); + } + + Ok(page::commit("stats", &rows, theme, width)) +} + +/// Rows for a set of io-harness's own `Tally` groups. +fn tallies(tallies: Vec, empty: &str) -> Vec { + if tallies.is_empty() { + return vec![Row::note(empty.to_string())]; + } + tallies + .into_iter() + .map(|tally| Row::fact(tally.key, tally.count.to_string())) + .collect() +} + +/// A byte count in the largest unit that leaves a whole number in front. +/// +/// Powers of 1024 and named as such: this is a file on a disk, and a reader +/// comparing it against what `ls` said should not have to convert. +fn bytes(count: u64) -> String { + const UNITS: [(&str, u64); 4] = [ + ("GiB", 1024 * 1024 * 1024), + ("MiB", 1024 * 1024), + ("KiB", 1024), + ("B", 1), + ]; + for (name, size) in UNITS { + if count >= size { + if size == 1 { + return format!("{count} B"); + } + return format!("{}.{} {name}", count / size, (count % size) * 10 / size); + } + } + "0 B".to_string() +} diff --git a/src/status.rs b/src/status.rs index e7dd019..26f028b 100644 --- a/src/status.rs +++ b/src/status.rs @@ -341,10 +341,33 @@ pub struct Status { /// None`, and rendering that as `0` would report a full ceiling as an /// exhausted one. /// - /// Tokens, never money. `Containment::max_total_cost` is documented inert - /// because the crate has no price telemetry, so a figure with a currency in - /// front of it would be one this interface invented. + /// Tokens, never money — and the money is [`Status::cost`], one field down. + /// + /// This one is the tree's draw against the tree's ceiling, which io-harness + /// expresses in tokens and enforces in tokens. A currency figure here would be + /// a conversion of a ceiling nobody set in currency. + /// + /// **The sentence that stood here until 0.22.0 said something stronger and it + /// had stopped being true.** It read: a figure with a currency in front of it + /// would be one this interface invented, because the crate has no price + /// telemetry. The crate has had price telemetry since io-harness 0.18.0 — + /// `provider_calls` records the whole token split per call and `pricing` + /// derives money from it — and io-cli read none of it for four releases while + /// this comment explained why it could not. pub spend: Option<(u64, Option)>, + /// What this run has cost, in micro-units, or `None` for no answer. + /// + /// **Absent rather than zero, and there are three different ways to have no + /// answer** — no price table configured, a table that prices none of the + /// models this run used, or a run that has made no provider call yet. All + /// three draw nothing, because a `$0` on this line would say the run was free, + /// and none of the three means that. `/cost` is where the three are told + /// apart, because there is room there to say which. + /// + /// The run's own, not the session's, unlike [`Status::tokens`] beside it. What + /// an operator watches while a turn runs is what the turn is costing; the + /// session total is a keystroke away on `/cost`, which carries both. + pub cost: Option, /// How many background shells this run has started and not yet finished. /// /// A `shell_start` outlives the step that launched it, which is the whole @@ -473,6 +496,7 @@ impl Status { policy: None, tokens: None, run_tokens: None, + cost: None, context: None, containment: None, spend: None, @@ -561,6 +585,46 @@ impl Status { pub fn start_run(&mut self) { self.elapsed = Duration::ZERO; self.run_tokens = None; + // Back to "no answer" rather than to zero, and it is the same distinction + // `run_tokens` above is held to: a turn that has not called a provider yet + // has not cost zero, it has not cost anything anybody can report. + self.cost = None; + } + + /// Set [`Status::cost`] from what this run has actually called, priced by + /// `table`. + /// + /// **Read from the store rather than accumulated off the event stream, and + /// that is forced rather than chosen.** `EventKind::Step` carries a scalar + /// token count, and a price needs the split — fresh prompt against cache read + /// against cache write against completion — which lives only on the + /// `provider_calls` row. Counting the scalar at an input rate would over-report + /// every cached turn and every reasoning turn, which is to say most of them. + /// + /// The cost shape, stated: one indexed read of one run's calls, on the events + /// that can change the answer and no others — the same place and the same + /// cadence as [`Status::note_context_from`], which has read the store per step + /// since 0.16.0. + /// + /// Silent on every failure. A store that cannot be read is a status line with + /// no cost field, which is what an operator with no prices already sees; a + /// notice about a failed read of a decorative field would be worse than the + /// field's absence. + pub fn note_cost_from( + &mut self, + store: &io_harness::Store, + run_id: i64, + table: &io_harness::pricing::PriceTable, + ) { + let Ok(calls) = store.provider_calls(run_id) else { + return; + }; + let total = crate::cost::Total::of(&calls, table); + // **Nothing priced is not zero priced.** A run whose models are all + // outside the table has a real cost that this program cannot state, and + // stating it as `$0` would be the invented number the whole of `/cost` is + // built to avoid. + self.cost = (total.micros > 0).then_some(total.micros); } /// Set [`Status::context`] from an observation section of `est_tokens`. @@ -796,6 +860,22 @@ impl Status { (self.queued_prompts > 0).then(|| format!("queued {}", self.queued_prompts)) } + /// This run's cost, drawn, or nothing at all. + /// + /// **One method, reached by both renderers**, which is the shape + /// [`Status::budgets_left`] and [`Status::queued_left`] above already have and + /// for the reason written where they are: this file has shipped a field into + /// one renderer twice — 0.8.0's spend, 0.12.0's planning — and both times the + /// field was invisible on the row the binary actually draws. + /// + /// `None` covers all three ways of having no answer and does not distinguish + /// them, because this line has no room to. `/cost` is the surface that tells + /// an empty price table from an unpriced model from a run that has called + /// nothing, and it is one keystroke away. + pub fn cost_field(&self) -> Option { + self.cost.map(crate::cost::money) + } + /// The fields, most important first. pub fn fields(&self, theme: &Theme) -> Vec { // The WORD is the state, and the animation is only beside it. A spinner @@ -882,6 +962,32 @@ impl Status { }; fields.push(Field::new(text, tone)); } + // **Left of the containment word, which is where the design put it and + // where a live run proved it has to be.** Drafted to the right of it, the + // field never appeared: a real containment word is `workspace-write/ + // macos-sandbox-exec`, thirty-three characters, and at a hundred columns + // beside the model, the posture, the state, the clock and the token count + // there was nothing left — so the one field this release exists to fill + // was the first one dropped, on the first terminal it was run in. + // + // **Left of the numbers, because it is not one** — and as of 0.22.0 the + // code is finally where this sentence has always said it was. A standing + // mode that stops the agent writing outranks what the last turn spent: if + // a narrow terminal can hold one of them, it should hold the one that + // explains why nothing is happening. It was pushed *after* `steps`, `tok` + // and `ctx` for four releases, and since `fits` drops from the right that + // meant `line` gave up the planning phase before it gave up the token + // count — the exact inversion of the rule written directly above it. The + // footer had the same inversion by a different mechanism, dropping its + // whole right-hand group to keep every counter, and both are corrected in + // the release that adds another counter to that row. + // + // `Normal` rather than `Muted` for the same reason the background-job + // count is — it is a fact about what the agent may do, not a footnote + // about what it did. + if self.planning { + fields.push(Field::new("planning".to_string(), Tone::Normal)); + } // Beside the token count rather than anywhere else, because the two are // the same kind of fact — what this run has spent — and they used to sit // together in the `Finished` row this release removed. @@ -900,26 +1006,18 @@ impl Status { if let Some(context) = self.context { fields.push(Field::new(format!("ctx {context}%"), Tone::Muted)); } - // **Left of the containment word, which is where the design put it and - // where a live run proved it has to be.** Drafted to the right of it, the - // field never appeared: a real containment word is `workspace-write/ - // macos-sandbox-exec`, thirty-three characters, and at a hundred columns - // beside the model, the posture, the state, the clock and the token count - // there was nothing left — so the one field this release exists to fill - // was the first one dropped, on the first terminal it was run in. + // **Right of the token count it is derived from**, so the two read in the + // order they are computed in: the tokens are what happened and the money + // is what they came to. `Muted` like the counters beside it, because it is + // one — a fact about what the last turn spent, which is exactly the class + // this row's own priority rule says yields to a standing mode. // - // What it outranks is the honest ordering too: the containment word says - // how commands are sandboxed and does not change during a turn; this says - // what the fan-out is spending, and changes every step. - // **Left of the numbers, because it is not one.** A standing mode that - // stops the agent writing outranks what the last turn spent: if a narrow - // terminal can hold one of them, it should hold the one that explains why - // nothing is happening. `Normal` rather than `Muted` for the same reason - // the background-job count is — it is a fact about what the agent may do, - // not a footnote about what it did. - if self.planning { - fields.push(Field::new("planning".to_string(), Tone::Normal)); - } + // From the same method as the footer's, four lines of which are the whole + // lesson of 0.8.0's spend field and 0.12.0's planning field: a field + // rendered in one of the two renderers is a field the operator never sees, + // because the binary draws the footer on every terminal seven rows or + // taller. + fields.extend(self.cost_field().map(|text| Field::new(text, Tone::Muted))); // **Right of the counters they bound and left of the tree's own spend.** // A budget is read against the number beside it — `3 steps` and `left // 17/20 steps` are one fact split in two — so the two travel together and @@ -1075,6 +1173,12 @@ impl Status { if let Some(context) = self.context { counts.push(format!("ctx {context}%")); } + // Here as well as on `Status::line`, out of the same method, for the + // reason the budgets below are — this is the row the binary draws at an + // ordinary prompt, so a cost added to `line` alone would be a cost no + // operator ever saw. 0.12.0's planning field, again, and 0.8.0's spend + // field before it. + counts.extend(self.cost_field()); // **Here as well as on `Status::line`, from the same method, and that is // deliberate rather than tidy.** This is the row the binary draws at an // ordinary prompt — `Status::render` takes the footer on any terminal @@ -1149,6 +1253,31 @@ impl Status { } allowed.push(Span::styled("planning", muted)); } + // **When the two groups cannot both fit, the counters yield — not the + // group.** `row` fits its right-hand group all or nothing, so a counts + // row one character too wide took the posture, the containment word AND + // the planning phase off the screen together and kept every counter. + // That is the failure the comment directly above records, arriving from + // the other direction: 0.13.1's row was `4 steps · 14.2k tok · / for + // commands` and the group fit beside it at a hundred columns; `ctx N%` + // made the row ten characters wider, and in the 0.21.0 live capture the + // finished turn's row carries no right group at all. + // + // Which side gives is not a judgement call — this module states it + // where `planning` is ordered on `Status::fields`: a standing mode that + // stops the agent writing outranks what the last turn spent. So + // counters come off the right of `counts`, rightmost first, which is + // the order `fits` drops fields in and the order they are pushed in + // — least load-bearing last — until the group fits. The + // leftmost counter is never dropped: a row that had given up every + // number would be reporting the mode by erasing the session. + let wanted: usize = allowed.iter().map(|s| s.content.chars().count()).sum(); + while !allowed.is_empty() + && counts.len() > 1 + && counts.join(separator).chars().count() + wanted >= room + { + counts.pop(); + } let counted = row( vec![Span::styled(counts.join(separator), muted)], allowed, @@ -1275,6 +1404,11 @@ fn fits(fields: &[Field], width: usize, theme: &Theme) -> usize { /// reader can find elsewhere: the posture is on the wizard's own screen and in /// the configuration, and the clock is beside the word `working` on the row that /// says a turn is running at all. +/// +/// **That last-resort rule is not the counts row's policy**, and a caller who +/// has a droppable left group is expected to narrow it before calling: the +/// counts row trims counters off its own right until the group fits, because +/// the group it would otherwise lose carries the planning phase. fn row(left: Vec>, right: Vec>, width: usize) -> Line<'static> { let measure = |spans: &[Span<'static>]| -> usize { spans.iter().map(|span| span.content.chars().count()).sum() @@ -1658,62 +1792,10 @@ pub fn committed( /// `text` as rows no wider than `width`, indented `first` and then `rest`. /// -/// **Folded and never fitted**, which is the one thing this differs in from every -/// other width-aware helper in this crate. `crate::picker::fit` and -/// [`Status::line`] shorten, because a picker row and a status line each own -/// exactly one row of a viewport that cannot grow. A committed surface owns as -/// many rows as it needs, so there is no reason left to lose a character — and -/// the characters most likely to be lost here are the tail of a workspace path -/// and the tail of a policy layer's act list, which is to say the answer. -/// -/// Broken at spaces, with a hanging indent that says a row is a continuation -/// without aligning anything into a column. A word longer than the room it has — -/// a deep path, an absurd model name — is **split** rather than allowed to -/// overflow: eighty columns is a supported size, and a row that runs past it gets -/// wrapped by the terminal at a place nothing here chose. -fn folded(text: &str, width: usize, first: usize, rest: usize) -> Vec { - let mut rows: Vec = Vec::new(); - let mut indent = first; - let mut row = " ".repeat(indent); - // Content characters on this row, which is its width less the indent. Counted - // rather than measured off `row` so the arithmetic is in one unit — the same - // reason `fits` counts characters and not bytes. - let mut used = 0usize; - for word in text.split_whitespace() { - let mut word = word; - loop { - // At least one cell, so a terminal narrower than the indent still - // makes progress instead of looping forever. - let room = width.saturating_sub(indent).max(1); - let space = usize::from(used > 0); - let length = word.chars().count(); - if used + space + length <= room { - if space == 1 { - row.push(' '); - } - row.push_str(word); - used += space + length; - break; - } - if used > 0 { - rows.push(std::mem::take(&mut row)); - indent = rest; - row = " ".repeat(indent); - used = 0; - // Retried whole on the fresh row: a word is only ever split when - // a row of its own cannot hold it. - continue; - } - let head: String = word.chars().take(room).collect(); - word = &word[head.len()..]; - row.push_str(&head); - rows.push(std::mem::take(&mut row)); - indent = rest; - row = " ".repeat(indent); - } - } - if used > 0 || rows.is_empty() { - rows.push(row); - } - rows -} +/// **The body moved to [`crate::page::folded`] in 0.22.0 and this is the name it +/// answered to here.** `crate::context` carried the same twenty lines under the +/// name `wrapped`, differing only in hard-coding the two indents this signature +/// takes as arguments, and `/cost` and `/stats` would have been the third and +/// fourth copies. The argument for folding rather than fitting is written out +/// where the code now lives. +use crate::page::folded; diff --git a/src/verify.rs b/src/verify.rs index c4ef093..4ed36e9 100644 --- a/src/verify.rs +++ b/src/verify.rs @@ -69,35 +69,95 @@ pub async fn credential(spec: &ProviderSpec) -> Result<(), String> { } } -/// The provider's model catalogue, as ids. +/// The catalogue as this provider serves it, whole rows and not just ids. /// -/// io-harness reads and prices catalogues already; this filters the reference -/// catalogue down to the models the chosen provider actually serves. An error is -/// not fatal — the wizard offers the provider's usual default instead, because a -/// catalogue that cannot be read is a reason to make the user type a model, not a -/// reason to stop. -pub async fn catalogue(spec: &ProviderSpec) -> Vec { - let Ok(models) = Reference::new().models().await else { +/// **The rows carry prices, and until 0.22.0 this function threw them away.** It +/// read the same catalogue, mapped every [`io_harness::ModelInfo`] down to its +/// `id`, and dropped the `price`, `price_tiers` and `price_source` on the same +/// row — while the interface over it reported token counts and called the money +/// question unanswerable. [`crate::prices`] is what keeps them; this is the read +/// they both come from, so there is one filter and one spelling rather than two +/// that can drift. +/// +/// `source` names the catalogue to read, for the operator on a self-hosted or +/// `compatible` endpoint the reference catalogue has never heard of. `None` is +/// io-harness's own default. +/// +/// An error is not fatal and comes back as an empty vector: a catalogue that +/// cannot be read is a reason to make the user type a model, not a reason to stop. +pub async fn served(spec: &ProviderSpec, source: Option<&str>) -> Vec { + let reference = match source { + Some(url) if !url.is_empty() => Reference::at(url), + _ => Reference::new(), + }; + let Ok(models) = reference.models().await else { return Vec::new(); }; - let mut ids: Vec = match spec { + named(spec, models) +} + +/// The catalogue filtered to what `spec` serves, spelled the way `spec` names it. +/// +/// Separated from the fetch so the whole of this decision is testable without a +/// socket. The **spelling matters beyond the wizard's list**: the id here is the +/// key a price is stored under, and it has to match the `model` io-harness records +/// on a provider call — which is the name the operator configured, not the +/// catalogue's namespaced one. +pub fn named(spec: &ProviderSpec, models: Vec) -> Vec { + match spec { // The reference catalogue is OpenRouter's own, so for OpenRouter it is not // a reference at all — it is the provider speaking for itself. - ProviderSpec::OpenRouter { .. } => models.into_iter().map(|model| model.id).collect(), + ProviderSpec::OpenRouter { .. } => models, ProviderSpec::Anthropic { .. } => strip(models, "anthropic/"), ProviderSpec::OpenAi { .. } => strip(models, "openai/"), // Anything else serves whatever it serves and the reference cannot say. _ => Vec::new(), - }; + } +} + +/// The models this provider serves, as ids, sorted and deduplicated. +/// +/// What the wizard puts in front of the user. Unchanged in behaviour since 0.1.0; +/// it is now one `map` over [`served`] rather than its own read. +pub async fn catalogue(spec: &ProviderSpec) -> Vec { + let mut ids: Vec = served(spec, None) + .await + .into_iter() + .map(|model| model.id) + .collect(); ids.sort(); ids.dedup(); ids } -fn strip(models: Vec, prefix: &str) -> Vec { +/// The priced rows of a catalogue, sorted by model. +/// +/// **A model the catalogue served with no price is absent, never entered at +/// zero.** io-harness's `PriceTable::price` returning `None` is what makes +/// `Spend::unpriced_calls` count that call, which is what lets `/cost` say its +/// total is a floor rather than reporting a partial sum as a total. A zero here +/// would silently claim the model is free. +pub fn priced( + spec: &ProviderSpec, + models: Vec, +) -> Vec<(String, io_harness::pricing::Price)> { + let mut rows: Vec<(String, io_harness::pricing::Price)> = named(spec, models) + .into_iter() + .filter_map(|model| model.price.map(|price| (model.id, price))) + .collect(); + rows.sort_by(|a, b| a.0.cmp(&b.0)); + rows.dedup_by(|a, b| a.0 == b.0); + rows +} + +fn strip(models: Vec, prefix: &str) -> Vec { models .into_iter() - .filter_map(|model| model.id.strip_prefix(prefix).map(str::to_string)) + .filter_map(|mut model| { + let id = model.id.strip_prefix(prefix)?.to_string(); + model.id = id; + Some(model) + }) .collect() } diff --git a/tests/docs.rs b/tests/docs.rs index 058dc20..bb70725 100644 --- a/tests/docs.rs +++ b/tests/docs.rs @@ -240,6 +240,11 @@ fn the_readme_documents_every_key_of_the_io_cli_section() { max_parallel_reads: Some(16), spawn_background_after_secs: Some(120), detached_spawns: Some(true), + prices: Some(io_cli::settings::PriceSettings { + source_url: Some("https://example.invalid/models".into()), + source: Some("the reference catalogue".into()), + models: Some(417), + }), }; let value = serde_json::to_value(&every).expect("[app.io-cli] serializes"); let keys = value.as_object().expect("a table"); diff --git a/tests/status.rs b/tests/status.rs index 7494c1a..f00dfb3 100644 --- a/tests/status.rs +++ b/tests/status.rs @@ -2015,3 +2015,120 @@ fn n3_the_queue_depth_survives_forgetting_the_run() { ); } } + +/// A session at the width and in the state the 0.21.0 live capture caught. +/// +/// Nothing here is invented for the test. The posture and the containment word +/// are the pair `Status::fields` names in its own comment as the real thing — +/// `workspace-write/macos-sandbox-exec`, thirty-four characters — and the +/// counters are the ones the footer had when the last turn ended: six steps, +/// twenty-six point nine thousand tokens, twenty-three percent of the window, +/// and the idle key hint. That row is forty-six characters, the right-hand +/// group is fifty-seven, and a hundred columns cannot hold both. +fn full_row() -> Status { + let mut status = Status::new("anthropic/claude-sonnet-4.5"); + status.policy = Some("read-only".into()); + status.containment = Some("workspace-write/macos-sandbox-exec".into()); + status.planning = true; + status.steps = Some(6); + status.tokens = Some(26_900); + status.context = Some(23); + status +} + +/// **F4 — a counts row that will not fit costs a counter, never the mode.** +/// +/// `row` fits the footer's right-hand group all or nothing, and the counts row +/// used to be handed to it whole: one character over and the posture, the +/// containment word and the planning phase all left the screen together while +/// every counter stayed. In the 0.21.0 live capture that is exactly what +/// happened — the working frame drew the group at column forty-four and the +/// finished frame, five characters wider because `esc stops` had become `/ for +/// commands`, drew no group at all. The operator finished a turn under `/plan +/// on` with nothing on screen saying io-harness would refuse the next write. +/// +/// It is the failure F4 was written against, reached from the other direction, +/// and it is a regression with a date: the same capture passes on 0.12.0 and +/// 0.13.1, whose row was `4 steps · 14.2k tok · / for commands` and left room. +/// Adding `ctx N%` to the counts row spent the last ten columns. +/// +/// Which side gives is settled by the module rather than by this test: a +/// standing mode that stops the agent writing outranks what the last turn +/// spent. So the assertion is not merely that `planning` survives — it is that +/// a counter went instead, and that the counters left are the leftmost ones. +/// +/// Sabotage: hand `counts` to `row` unnarrowed, which is the 0.21.0 code — under +/// which only this pair of tests fails, and it fails at the one width an +/// eighty-to-a-hundred-and-twenty-column terminal is most likely to be. +#[test] +fn f4_a_full_counts_row_drops_a_counter_and_not_the_planning_phase() { + let text = footed(&full_row(), 100); + + assert!( + text.contains("planning"), + "the mode that stops the agent writing left the row: {text:?}", + ); + assert!( + text.contains("workspace-write/macos-sandbox-exec") && text.contains("read-only"), + "the phase took the rest of its group with it: {text:?}", + ); + // What paid for it, and which end it was taken off. The key hint is the + // rightmost counter and the least of them — `/help` is the help screen — + // and the fields left are the ones the row is ordered to keep. + assert!( + !text.contains("/ for commands"), + "nothing was dropped, so the row cannot have fitted: {text:?}", + ); + assert!( + text.contains("6 steps") && text.contains("26.9k tok") && text.contains("ctx 23%"), + "counters came off the wrong end: {text:?}", + ); +} + +/// **F4 — and it holds with one more counter on the row, at every width the +/// group can be drawn at.** +/// +/// The companion above pins the geometry that actually failed; this one pins +/// the property, because the geometry is about to move. The counts row grows +/// every release — `ctx N%` is what tipped 0.21.0 over, and the cost counter +/// this release adds sits in the same group and makes the row wider still — so +/// a test that asserted one magic width would go green on the arithmetic of the +/// week it was written and say nothing about the week after. +/// +/// `plan 2/5` stands in for that next counter: it is a real member of the same +/// row, it is eight characters plus a separator, and with it the row no longer +/// fits at any width in the sweep on the first trim. The sweep starts at seventy +/// because a right-hand group of fifty-seven plus the leftmost counter is +/// sixty-four columns, and below that there is no row that holds both — the +/// trim stops at one counter deliberately, since a row that had given up every +/// number would be reporting the mode by erasing the session. +/// +/// Sabotage: pop from the front of `counts` instead of the back — under which +/// this test still passes at every width and the one above fails on `6 steps`, +/// which is why both are here. +#[test] +fn f4_the_planning_phase_holds_at_every_width_that_can_hold_the_group() { + let mut status = full_row(); + status.plan = Some((2, 5)); + + for width in 70..=160u16 { + let text = footed(&status, width); + assert!( + text.contains("planning"), + "the planning phase is off the footer at {width} columns: {text:?}", + ); + // The row it was fitted into is still a row. A trim that undercounted + // would put the group past the edge instead of dropping a counter. + for line in status.footer(width, &DARK) { + let drawn: usize = line + .spans + .iter() + .map(|span| span.content.chars().count()) + .sum(); + assert!( + drawn <= width as usize, + "a footer row overflowed {width} columns: {line:?}", + ); + } + } +} From af54c4fb398f101e13fb8b1eee467081a29c0e74 Mon Sep 17 00:00:00 2001 From: aakashpawar1999 Date: Thu, 27 Aug 2026 21:27:53 +0530 Subject: [PATCH 4/7] feat: fill the price table from the provider the operator connected MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The table ships empty and nothing is compiled in. It is filled from the model catalogue that provider serves — the fetch io-cli already made to offer a model list, whose prices it discarded — written into io-harness's own `[prices]` as a date and one line per model, and re-read from a row on `/config`. The refresh shows every rate that would move before a byte lands, and the reason is sharper than courtesy: the file records what a rate is and not where it came from, so io-cli cannot tell a correction the operator typed from a value an older catalogue served. It does not guess. It shows the lot and lets the operator decline the lot, which is the shape `/import` established. A refetch that comes back far shorter than the table it would replace is refused and the old table kept. A truncated read that silently turned most of an operator's spending into "unpriced" would read on `/cost` as a bill that suddenly fell, which is the one failure in this area that loses money quietly. `[prices]` is io-harness's section and `deny_unknown_fields`, so a key of io-cli's own beside `as_of` and `models` would not be ignored — it would make the whole configuration file unreadable. Everything io-cli needs to remember that io-harness does not model lives under `[app.io-cli.prices]`: which catalogue to read, what the last read was in words, and how many models it priced. The clock is read in the driver because the driver is the only place `tests/timing.rs` permits it; the conversion to a date is a pure function of a number, which is also what makes it testable. Two of the new tests were failing for reasons that were not production bugs and are fixed as tests: the span helper's common-suffix scan swallowed a shared `.0`, and a test asserting a server arrives cannot use the variable another test deliberately leaves unset — an unresolvable `${env:}` is refused and rolled back, correctly. --- Cargo.lock | 2 +- Cargo.toml | 2 +- README.md | 168 +++++++++++++++++++++++--- docs/config.example.toml | 84 ++++++++++++- src/configure.rs | 37 ++++++ src/main.rs | 248 ++++++++++++++++++++++++++++++++++++--- src/prices.rs | 84 +++++++++++++ tests/commands.rs | 20 +++- tests/edit.rs | 10 +- tests/import.rs | 15 ++- 10 files changed, 626 insertions(+), 44 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c84f445..0723a64 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1850,7 +1850,7 @@ dependencies = [ [[package]] name = "io-cli" -version = "0.21.0" +version = "0.22.0" dependencies = [ "clap", "crossterm", diff --git a/Cargo.toml b/Cargo.toml index bada99e..cc25182 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "io-cli" -version = "0.21.0" +version = "0.22.0" edition = "2021" # io-harness declares 1.95, set by `libsqlite3-sys` calling the std `cfg_select!` # macro from its build script. Nothing here can be lower than what the harness diff --git a/README.md b/README.md index f5004ba..35d8904 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ reimplemented here. A test asserts that: `tests/dependencies.rs` fails the build if this crate ever grows an HTTP client, a TLS stack, a database or a sandbox. - [Install](#install) · [First run](#first-run) · [Bringing your setup across](#bringing-your-setup-across) · [While it works](#while-it-works) -- [Keys](#keys) · [Commands](#commands) · [Configuration](#configuration) +- [Keys](#keys) · [Commands](#commands) · [What it costs](#what-it-costs) · [Configuration](#configuration) - [Capability bundles](#capability-bundles) · [Hooks](#hooks) · [The fleet](#the-fleet) - [Pictures](#pictures) · [Documents](#documents) · [Background jobs](#background-jobs) - [Reading it without seeing it](#reading-it-without-seeing-it) · [Headless](#headless) @@ -47,6 +47,7 @@ on one row and the keys and the posture on the next.](docs/screenshot.png) | **Markdown, rendered** | Headings, bullets, code and emphasis drawn as themselves rather than printed as notation | | **Documents, read and written** | Spreadsheets, Word, slide decks, PDFs and barcodes through io-harness's own tools — twelve of them, six of which write, every one under the same gate as any other read or write | | **Your setup, brought across** | `/import` finds the agent tools already on this machine and offers their instructions, MCP servers, skills and model, item by item, with the whole plan shown before a byte is written and no credential read at any point | +| **What it spent, and whether it worked** | `/cost` commits the money and the token split — this run, this session, by model, by day — and `/stats` commits the outcomes, the first-try counts, the gate failures and the latencies. Nothing is estimated: an unpriced model makes a total a floor and the page says so | ## It never takes your terminal @@ -224,6 +225,24 @@ path until this release, so what an operator met under a half-finished answer wa `error: step_cap_reached`. All four are successful calls in io-harness and always were. The word stays the harness's; what changed is the weight it is said in. +**From 0.22.0 the status line also carries what the run cost**, immediately right +of the token count it is derived from, so the two read in the order they are +computed in: the tokens are what happened and the money is what they came to. It +is **absent** — not `$0` — where there is no price table, where the table prices +none of the models this run called, or where nothing has run yet. Those three +things are different and none of them means free; `/cost` is where they are told +apart. See [What it costs](#what-it-costs). + +**Two things about that line were corrected in the same release, both of them +priority inversions.** The footer used to drop its whole right-hand group — the +policy layer, the containment mode, the planning phase — to keep every counter, +so a narrow terminal gave up the standing modes that explain *why nothing is +happening* in order to keep a number about what already happened; the counters +yield now. And `planning` was pushed right of the counters, so the narrow line +dropped it before the token count, inverting the rule written five lines above it +in the code. A standing mode that stops the agent writing outranks what the last +turn spent, on both rows. + What lands in the scrollback is designed rather than defaulted. A tool call reads as a verb and a workspace-relative path — `Read src/lib.rs`, not `read_file` and an absolute path — and a tool this release has never seen keeps the name @@ -438,6 +457,9 @@ above a row that ranked there for reasons having nothing to do with it. | `/profile` | switch to a named profile from the configuration, for this session | | `/steer` | send what is queued into the turn that is already running | | `/compact` | fold this conversation into a summary, at the next step | +| `/image` | draw an attached image again: /image 1 | +| `/copy` | put the last answer on the system clipboard | +| `/copy diff` | put the whole run's patch on the system clipboard | **inspect** @@ -449,9 +471,8 @@ above a row that ranked there for reasons having nothing to do with it. | `/expand` | commit the last step's full detail into the scrollback | | `/fleet` | show the children this turn has spawned | | `/skills` | every skill, shipped or yours: what it is for, whether it is on, and its file | -| `/image` | draw an attached image again: /image 1 | -| `/copy` | put the last answer on the system clipboard | -| `/copy diff` | put the whole run's patch on the system clipboard | +| `/cost` | commit what this run, this session and this install have spent | +| `/stats` | commit how the runs have gone: outcomes, first-try, gates, latency | **configure** @@ -476,6 +497,21 @@ here you use once: the others are returned to for the life of the install. It writes files, which is the whole reason it is under **configure**. See [Bringing your setup across](#bringing-your-setup-across). +`/cost` and `/stats` are new in 0.22.0 and are two commands rather than one +because they are two questions: `/cost` says what the work cost, `/stats` says +whether it worked. See [What it costs](#what-it-costs). + +`/image`, `/copy` and `/copy diff` sit under **this turn** from 0.22.0, and it is +a correction rather than a way of making room — the same sentence 0.19.0 wrote +about `/mcp` and `/provider`, and it is worth being able to say twice. All three +act on the turn that just finished: `/image` draws an image attached to it, +`/copy` puts its answer or its diff on the clipboard. None of them asks the store +a question, which is what **inspect** means. They were filed there because they +*show* something, and showing is not the same as inspecting. `/cost` and `/stats` +are what made it worth correcting: no group may hold more than ten, **inspect** +stood at nine, and the choice was between re-filing three commands that were in +the wrong group and filing two more that would have been. + `/mcp` and `/provider` sit under **configure** from 0.19.0, and it is a correction rather than a promotion: both open with a list, and both go on from that list to add, edit and remove entries in the configuration file, which is the @@ -483,12 +519,14 @@ one thing **inspect** promises it never does. That second half was a promise rather than a fact until 0.21.0 — the writers existed and nothing called them, so both panels could only read. They write now. `/steer` and `/compact` are listed under **this turn**, which is the group the code has -always filed them in — the table above said otherwise until this release, and +always filed them in — the table above said otherwise until 0.21.0, and nothing but a reader was misled by it. -`/usage` answers what `/status` answers and is deliberately not listed above: an -alias earns no row of its own, because a second row for one screen reads as a -second screen. +`/usage` opens `/cost` and is deliberately not listed above: an alias earns no +row of its own, because a second row for one screen reads as a second screen. It +resolved to `/status` until 0.22.0, which was the closest thing there was to an +answer and was not one — `/usage` means spend everywhere else it exists, and this +product now has a spend to report. In the palette each row carries a mark saying what it is — `:` runs a command, `+` fills the prompt from a configured template, `*` names one of the agent's own @@ -541,6 +579,94 @@ refuse a large payload without saying so. `/theme` and `/model` change this session only and say so. Making a choice permanent is `io setup`. +## What it costs + +**`/cost` commits what has been spent, and `/stats` commits how the runs have +gone.** Two commands rather than one, because they are two questions — what the +work cost, and whether it worked — and a single screen carrying thirteen sections +is one nobody reads to the end. `/usage` opens `/cost`. + +`/cost` has four sections: **this run**, **this session**, **by model** and **by +day**, each with the calls, the money, and the token split under it. Every figure +is a row io-harness has been writing into `runs.db` since its 0.18.0 and this +interface has never read — the model, the prompt, completion, cache-read, +cache-write and reasoning split, the latency and the time to first token, one row +per provider call. + +**Nothing is estimated.** There is no sampling, no extrapolation from a partial +window, and no model of what a token "usually" costs. Three rules follow from +that and each is on the page rather than in this file: + +- **Cache reads and cache writes are the breakdown of the prompt they are part + of, never an addition to it.** They are already inside `prompt_tokens`, so the + page shows the prompt total and the parts under it — read, written, fresh — and + the parts sum to the total rather than past it. Reasoning is the same + arrangement under completion: a vendor that reports it separately still bills it + as output. +- **A call the provider reported no usage for is *unknown*, never free.** + io-harness stores that as SQL `NULL` and reads it back as `None` for exactly + this reason; summing it as zero would report a turn that cost something as a + turn that cost nothing. The count of them is a line on the page. +- **A model with no rate in the table makes the total a *floor*, and the page + says so** — in that word, with the number of calls it applies to, and on the + row itself in the grouped sections so a reader scanning for the largest figure + can see which of them is incomplete without reading past the list. + +`/stats` is the other half: runs by outcome, runs by day, the first-try counts, +gate failures by phase, recovery, the slowest calls and the time to first token +over the last 200 runs, and what the store holds on disk. **First-try is +io-harness's own definition** — finished *and* successful *and* carrying no gate +failure — and it arrives as three counts rather than as a rate, because the +denominator is a choice. Where a share is shown, the row names what it is a share +*of*. The sandbox gate's phases and the review, command and contains gates use +two vocabularies that do not overlap, so they are two lists and are never merged +into one. Nothing on the page compacts anything: `Store::compact` is a full +`VACUUM` that needs free disk roughly equal to the file, so the free figure is +reported and the reclaiming is not offered here. + +### Where a price comes from + +**io-cli compiles no prices in.** A rate baked into a binary is a promise the +binary cannot keep: providers move prices without announcing it, a release cadence +is not a pricing cadence, and an operator reading a confident wrong number is +worse off than one reading no number at all. So the table ships empty, and **an +install that has connected nothing sees tokens and no currency** — which is the +honest answer, not a bug. + +It is filled from **the model catalogue your connected provider already serves** — +the same fetch `io setup` has made since 0.1.0 to offer you a list of models, +whose prices it read and threw away on the same row. The rates land in `[prices]`, +which is io-harness's own section: `as_of` for the date, and one line per model +under `[prices.models]`. + +**Refreshing is a row on `/config`, and it shows every rate that would move before +it writes anything** — what each was and what it would become — and you can +decline the lot. That is not courtesy. The file records what a rate *is* and never +where it came from, so io-cli cannot tell a correction you made by hand from a +value an older catalogue served; it does not guess, it shows you. A refetch that +comes back empty, or far shorter than the table it would replace, is **refused** +and the old table kept: a truncated response that replaced a full table with a +handful of rows would turn most of your spending into "unpriced" and shrink your +reported bill with nothing failing anywhere. A first fill has nothing to compare +against and is never refused. Rows for models the catalogue no longer serves are +left alone, because io-harness prices a call by the model name on it and an old +row is what prices an old run correctly. + +**Whose price it is gets said, on every surface that draws money.** OpenAI, +Anthropic and Google publish no prices on any endpoint — their model endpoints +carry capabilities and limits and no cost field — so for those three the rates +necessarily come from the reference catalogue rather than from the vendor. The +page names which and on what date. On OpenRouter the two coincide, because the +reference catalogue is OpenRouter's own. `[app.io-cli.prices] source_url` points +at a different catalogue, which is how an operator on a self-hosted or +`compatible` endpoint gets prices at all. + +**The unit is `$`.** io-harness prices in micro-units and names no currency, +because it is not the layer that knows one. io-cli is: every catalogue it can read +quotes USD and every provider it can connect to bills in it. Four decimals below a +unit and two above, in integer arithmetic — a turn costs a fraction of a cent and +a month costs dollars, and one precision cannot show both. + ## Skills **Five of the things io can do have a plain-language door.** Say what you want @@ -1137,7 +1263,13 @@ from the next turn. every section io-cli has no type for come back byte for byte — one value's bytes are replaced and the rest is copied through. The write is staged in a temporary file and renamed over the original, so a failure cannot truncate a configuration, -and the mode is preserved. +and the mode is preserved. **That now holds for a removal as well**: through +0.21.0, removing or moving an entry took the *next* section's comment block away +with it, and moving one into the last position of a file with no trailing newline +concatenated it onto the final value. A key whose name carries a dot — a model id, +an MCP server id — is addressed correctly too; it could only be written quoted, +and the path splitter cut it in half, which surfaced as an unexplainable "the edit +would have produced a file that does not parse". **A project-scoped change that would widen the boundary is refused in io-harness's own words**, and the same value is accepted in `io.local.toml` — @@ -1185,7 +1317,7 @@ bundles](#capability-bundles). **`/profile`** switches to a named `[profile.]` for the session, and `--profile ` picks one for a single run without writing anything. -Eight keys live there, and five tables: +Eight keys live there, and six tables: | Key | Is | | --- | --- | @@ -1202,6 +1334,7 @@ Eight keys live there, and five tables: | `[[app.io-cli.mcp]]` | MCP servers for the turn, in io-harness's own shape. Merged with the top-level `[[mcp]]`, and wins a collision of ids. | | `[[app.io-cli.lsp]]` | language servers for this workspace. Merged with the top-level `[[lsp]]`, and wins a collision of ids. | | `[app.io-cli.browser]` | a browser the agent may drive. Never downloaded — it is one you already have. | +| `[app.io-cli.prices]` | where the rates in `[prices]` came from: `source_url` names a catalogue to read instead of io-harness's default, and `source` and `models` record what the last read was and how many models it priced. The last two are written by a fetch rather than by hand. See [Where a price comes from](#where-a-price-comes-from). | Because the section is unvalidated by design, an unrecognised *value* reads as the default rather than stopping a session from starting. A section io-harness cannot @@ -1332,12 +1465,17 @@ line beside what has been drawn against them. **`[[hook]]` and capability bundles are applied from 0.20.0**, each with the surface the omission was waiting on: `/plugin` for what a bundle brought and what was dropped, and io-harness's own refusal sentence for a hook a project file may -not declare. That leaves two sections and one key still unapplied, and each has a -reason. `[prices]` is not part of a contract at all and belongs with the release -that reads the provider-call rows; there is no `[verify]` section to apply, and +not declare. **`[prices]` is read from 0.22.0**, which is the release that reads +the provider-call rows it prices. That leaves one section and one key still +unapplied, and each has a reason: there is no `[verify]` section to apply, and giving a session verification gates needs its own surface; and `run.templates` is -the thirteenth `[run]` key, reachable only through its own accessor. None of them -is a silent omission — this is where they are named. +the thirteenth `[run]` key, reachable only through its own accessor. Neither is a +silent omission — this is where they are named. + +**A price is never invented, and a missing one is never a zero.** io-cli compiles +no rates in and estimates nothing, so an install that has connected no provider +reports tokens and no currency, and a total containing a model the table does not +price is reported as a floor. See [What it costs](#what-it-costs). **`/import` does not bring a permission boundary across, and never will.** Another tool spells a permission as a command line; io-harness matches a binary name. The diff --git a/docs/config.example.toml b/docs/config.example.toml index 9dfe5ce..6fff068 100644 --- a/docs/config.example.toml +++ b/docs/config.example.toml @@ -291,9 +291,61 @@ net = "deny" # on_failure = "refuse" # timeout_ms = 5000 +# --- What a token costs (0.22.0) -------------------------------------------- +# +# io-harness's own section, and the only place a price comes from. NOTHING IS +# COMPILED IN: a rate baked into a binary is a promise the binary cannot keep, +# because providers move prices without announcing it and a release cadence is +# not a pricing cadence. So this table ships EMPTY, and `/cost` reports tokens +# and no currency until it is filled. +# +# Filling it is a row on `/config`: io-cli reads the model catalogue your +# connected provider already serves — the same fetch that offers the wizard its +# model list, whose prices it used to throw away — and writes what it found here. +# EVERY RATE THAT WOULD MOVE IS SHOWN BEFORE A BYTE IS WRITTEN, with what it was +# and what it would become, and declining writes nothing. That is not courtesy: +# this file records what a rate IS and never where it came from, so io-cli cannot +# tell a correction you made by hand from a value an older catalogue served. It +# does not guess; it shows you and lets you refuse the lot. +# +# A refetch that comes back EMPTY, or far shorter than the table it would +# replace, is refused and the old table kept — a truncated response that replaced +# a full table with a handful of rows would turn most of your spending into +# "unpriced" and shrink your reported bill with nothing failing anywhere. A first +# fill has nothing to compare against and is never refused. +# +# WHOSE PRICE IT IS. OpenAI, Anthropic and Google publish no prices on any +# endpoint — their model endpoints carry capabilities and limits and no cost +# field. So for those three the rates necessarily come from the reference +# catalogue rather than from the vendor, and every surface that draws money says +# which and on what date. On OpenRouter the two coincide, because the reference +# catalogue is OpenRouter's own. +# +# THIS SECTION IS deny_unknown_fields AND CARRIES EXACTLY `as_of` AND `models`. A +# key of io-cli's own put beside them would not be ignored — it would make this +# whole file unreadable. io-cli's bookkeeping lives in [app.io-cli.prices] below. +# +# as_of the date the rates were read. REQUIRED: a price list with no date is +# a claim with no expiry. Written by a fetch rather than typed. +# models one entry per model, in micro-units PER MILLION TOKENS +# +# Rows for models the catalogue no longer serves are left alone rather than +# pruned. They are not stale: io-harness prices a call by the model name on it, so +# an old row is what prices an old run correctly, and `/cost` reports history as +# well as today. +# +# [prices] +# as_of = "2026-08-27" +# +# [prices.models."anthropic/claude-sonnet-4.5"] +# input = 3000000 +# output = 15000000 +# cache_read = 300000 +# cache_write = 3750000 + # --- io-cli's own section --------------------------------------------------- # -# Eight keys and five tables. io-harness does not validate any of it, so an +# Eight keys and six tables. io-harness does not validate any of it, so an # unrecognised VALUE reads as the default rather than stopping a session. A # table io-harness cannot parse at all is different: the session starts on the # defaults and says so, carrying the harness's own message. @@ -374,10 +426,10 @@ fleet = "ctrl+f" # task at all**, and it is absent by default: io-harness has one session entry # point that reaches its spawn loop, and these caps are what selects it. # -# What it costs is smaller than it was, and it is not a mode. No session turn -# takes a steer inbox — none has since 0.11.0 — so a contained turn gives up no -# steering an ordinary one has. `Ctrl+C` still ends it, at the next point where no -# child is in flight, and `/contain off` gives the next turn back. +# What it costs is smaller than it was, and it is not a mode. Every session turn +# carries a steer inbox — every one has since 0.17.0 — so a contained turn gives +# up no steering an ordinary one has. `Ctrl+C` still ends it, at the next point +# where no child is in flight, and `/contain off` gives the next turn back. # # THROUGH 0.11.0 THIS TABLE DECIDED EVERY KEY BELOW IT, AND IT NO LONGER DOES. # io-harness's contained turn was then the only session entry point that took a @@ -475,3 +527,25 @@ fleet = "ctrl+f" # [app.io-cli.browser] # binary = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" # headless = true + +# Where the prices above came from (0.22.0). The rates themselves are io-harness's +# and live in [prices]; this is the bookkeeping io-harness does not model, in the +# one section that will not refuse a key it has never heard of. +# +# source_url the catalogue to read. ABSENT MEANS io-harness's own default, +# which is the reference catalogue. Write this when you are on a +# self-hosted or `compatible` endpoint: the reference catalogue +# cannot speak for a server it has never heard of, and this key is +# the only way such an operator gets prices at all. +# source what the last read was, IN WORDS — the vendor speaking for +# itself, or a named third-party catalogue. WRITTEN BY A FETCH +# RATHER THAN BY HAND: it is what the surfaces that draw money +# cite, and a number attributed to a vendor that never published +# one is the thing it exists to prevent. +# models how many models the last read priced. `PriceTable` has no length +# and no iterator — it answers `price(model)` and nothing else — so +# this is the only count there is, and it is what refuses a refetch +# that comes back far shorter than the table it would replace. +# +# [app.io-cli.prices] +# source_url = "https://openrouter.ai/api/v1/models" diff --git a/src/configure.rs b/src/configure.rs index 4ebf715..4762f83 100644 --- a/src/configure.rs +++ b/src/configure.rs @@ -78,6 +78,20 @@ impl Decided { Decided::Default => None, } } + + /// The scope that decided it, where a file did. + /// + /// **What a rewrite of an existing value has to be aimed at.** Writing into a + /// higher-precedence scope than the one that holds a key shadows it rather + /// than updating it: the value in force changes, and the file the operator + /// opens still shows the old one. `None` is io-harness's own default, which no + /// file holds and which a caller therefore has to choose a scope for. + pub fn scope(&self) -> Option { + match self { + Decided::File { scope, .. } => Some(*scope), + Decided::Default => None, + } + } } /// One row of the surface. @@ -147,6 +161,29 @@ pub const CATALOGUE: &[&str] = &[ "app.io-cli.prices.source_url", ]; +/// The `/config` row that re-reads the price catalogue. +/// +/// **A row rather than a key, because it is an act and not a setting.** Every +/// other row on that surface names something in a file and puts it in the +/// composer for the operator to type a value after; this one does something. It +/// is a sentinel and not a path so that `settings` and `setting` never see it — +/// a key that is not in any file and never will be would show as "not set" +/// forever on a surface whose whole job is saying what is in force. +/// +/// It lives here rather than in the driver so a test can reach it: nothing under +/// `tests/` can link `src/main.rs`, and a row spelled in the driver is a row no +/// test can assert on. +pub const REFRESH_PRICES: &str = "!refresh-prices"; + +/// The label that sentinel wears on the picker. +pub fn refresh_row(setting: &Setting) -> crate::picker::Row { + let detail = match &setting.value { + Some(as_of) => format!("last read {as_of}"), + None => "no prices are configured".to_string(), + }; + crate::picker::Row::with_detail("prices: re-read the catalogue", detail) +} + /// Whether a key's value is a credential and must never be shown in full. /// /// N2. The test is the key rather than the value, because a credential that diff --git a/src/main.rs b/src/main.rs index b4103d1..a0d532f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1825,11 +1825,27 @@ async fn loop_over Result>( } } } - Pick::Config(paths) => { - if let Some(key) = paths.get(index) { - app.composer.set(&format!("/config {key} ")); + Pick::Config(paths) => match paths.get(index).map(String::as_str) { + // The one row on this surface that acts rather than + // naming something. It re-reads the catalogue the + // operator's provider serves and writes what moved + // into the scope that already declares the prices — + // or the user scope, for a first fill. + Some(io_cli::configure::REFRESH_PRICES) => { + refresh_prices(screen, &mut app, &config, &spec, session.root()) + .await?; + match io_cli::configure::reload(session.root()) { + Ok((fresh, stored)) => { + capabilities = + io_cli::contract::Capabilities::stored(stored.as_ref()); + config = fresh; + } + Err(error) => app.record(Tone::Error, error), + } } - } + Some(key) => app.composer.set(&format!("/config {key} ")), + None => {} + }, Pick::ConfigScope { key, value, paths } => { // An index past the end puts nothing in the file, // which is the same answer every other picker arm @@ -2428,9 +2444,18 @@ async fn loop_over Result>( } Action::Config(None) => { let settings = io_cli::configure::settings(&config); - let paths: Vec = settings.iter().map(|s| s.path.clone()).collect(); + let mut paths: Vec = settings.iter().map(|s| s.path.clone()).collect(); + let mut rows = io_cli::configure::rows(&settings); + // **Last, because it is the one row that acts.** Every other + // row names a setting and puts it in the composer; a reader + // scanning for `policy.defaults.write` should not have to step + // over something that does work on the way there. + rows.push(io_cli::configure::refresh_row( + &io_cli::configure::setting(&config, "prices.as_of"), + )); + paths.push(io_cli::configure::REFRESH_PRICES.to_string()); picker = Some(( - Picker::new("Which setting?", io_cli::configure::rows(&settings)), + Picker::new("Which setting?", rows), Pick::Config(paths), )); } @@ -4287,6 +4312,179 @@ fn to_copy(session: &Session, store: &Store, what: Copied) -> (Option, S } } +/// The ids of a catalogue read, sorted, for a picker. +/// +/// The wizard wants names and [`io_cli::prices`] wants rates, and both come off +/// the same rows. Splitting them here is what stops the catalogue being fetched +/// twice for one screen. +fn ids(served: &[io_harness::ModelInfo]) -> Vec { + let mut ids: Vec = served.iter().map(|model| model.id.clone()).collect(); + ids.sort(); + ids.dedup(); + ids +} + +/// Write a catalogue read into `path` as a `[prices]` section, and say what +/// happened. +/// +/// **The clock is read here because here is the only place it may be read.** +/// `tests/timing.rs` permits `SystemTime::now` in this file and refuses it in +/// every other file under `src/`, so the driver takes the number and +/// [`io_cli::prices::date`] converts it. That is a gate rather than a preference, +/// and it is also what makes the conversion testable. +/// +/// `existing` is how many models the table being replaced priced, which is what +/// [`io_cli::prices::Catalogue::too_short`] refuses a short answer against. Zero +/// for a first fill, which is never refused. +/// +/// Never an error. A catalogue that could not be read is an operator with no +/// prices, which is a supported state — `/cost` draws tokens and no currency and +/// says so. A first run that failed because a price list did not arrive would be a +/// first run failed over a decoration. +fn fill_prices( + path: &std::path::Path, + spec: &io_harness::ProviderSpec, + served: Vec, + existing: usize, + theme: &Theme, +) -> Vec> { + let secs = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|since| since.as_secs()) + .unwrap_or(0); + let catalogue = io_cli::prices::Catalogue::of(spec, served, io_cli::prices::date(secs)); + if catalogue.rows.is_empty() { + return vec![theme.notice( + Tone::Muted, + format!( + "no prices were written: the catalogue served {} model{} and priced none of them, \ + so /cost will report tokens and no money", + catalogue.served, + if catalogue.served == 1 { "" } else { "s" } + ), + )]; + } + if catalogue.too_short(existing) { + return vec![theme.notice( + Tone::Warning, + format!( + "the catalogue answered with {} priced model{} where {existing} were expected, \ + which is short enough to be a truncated read — the prices you have were kept", + catalogue.rows.len(), + if catalogue.rows.len() == 1 { "" } else { "s" } + ), + )]; + } + let source = io_cli::prices::source_word(&catalogue.source); + let mut edits = catalogue.edits(); + edits.push(io_cli::edit::Edit::set( + "app.io-cli.prices.source", + format!("\"{source}\""), + )); + edits.push(io_cli::edit::Edit::set( + "app.io-cli.prices.models", + catalogue.rows.len().to_string(), + )); + match io_cli::edit::write(path, &edits) { + Ok(()) => vec![theme.notice( + Tone::Success, + format!( + "priced {} of the {} models {source} serves, as of {}", + catalogue.rows.len(), + catalogue.served, + catalogue.as_of + ), + )], + // Reported and not fatal, and the sentence says what was lost rather than + // that something went wrong: an operator who knows they have no prices can + // go and get them, and one told "an error occurred" cannot. + Err(error) => vec![theme.notice( + Tone::Warning, + format!("no prices were written ({error}); /cost will report tokens and no money"), + )], + } +} + +/// Re-read the price catalogue and write what moved. +/// +/// **Everything that would change is committed before anything is written**, which +/// is the shape `/import` established in 0.21.0 and it is here for a sharper +/// reason. io-cli cannot tell a rate the operator corrected by hand from one an +/// older catalogue served — the file records a number, not where it came from — so +/// it does not guess which is which. It shows every rate that would move, with +/// what it was and what it would become, and the operator who sees their own +/// correction in that list can decline the lot. +/// +/// The scope written is the one that already declares `prices.as_of`, so a refresh +/// lands where the last one did rather than shadowing it from a higher layer. A +/// first fill goes to the user scope, which is where the wizard writes. +async fn refresh_prices( + screen: &mut Screen>, + app: &mut App, + config: &Config, + spec: &io_harness::ProviderSpec, + root: &std::path::Path, +) -> Result<(), String> { + let (stored, _) = settings::stored(config); + let source = stored + .as_ref() + .and_then(|s| s.prices.as_ref()) + .and_then(|p| p.source_url.clone()); + let existing = stored + .as_ref() + .and_then(|s| s.prices.as_ref()) + .and_then(|p| p.models) + .unwrap_or(0); + + app.say(Tone::Muted, "re-reading the catalogue…"); + let served = verify::served(spec, source.as_deref()).await; + + let secs = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|since| since.as_secs()) + .unwrap_or(0); + let catalogue = + io_cli::prices::Catalogue::of(spec, served, io_cli::prices::date(secs)); + + let moved = io_cli::prices::changes(config.prices().as_ref(), &catalogue); + let lines = io_cli::prices::report(&catalogue, &moved, existing, &app.theme, screen.width()); + screen.commit(&lines).map_err(|error| error.to_string())?; + if moved.is_empty() || catalogue.too_short(existing) { + return Ok(()); + } + + // The scope that already holds the prices, so a refresh lands where the last + // one did. A higher-precedence scope would shadow rather than update, and the + // operator would be looking at the old numbers in the file they edit. + let scope = io_cli::configure::setting(config, "prices.as_of") + .decided + .scope() + .unwrap_or(io_harness::config::Scope::User); + let source_word = io_cli::prices::source_word(&catalogue.source); + let mut edits = catalogue.edits(); + edits.push(io_cli::edit::Edit::set( + "app.io-cli.prices.source", + format!("\"{source_word}\""), + )); + edits.push(io_cli::edit::Edit::set( + "app.io-cli.prices.models", + catalogue.rows.len().to_string(), + )); + match io_cli::configure::write(root, scope, &edits) { + Ok(()) => app.record( + Tone::Success, + format!( + "{} rate{} written, dated {}", + moved.len(), + if moved.len() == 1 { "" } else { "s" }, + catalogue.as_of + ), + ), + Err(error) => app.record(Tone::Error, error), + } + Ok(()) +} + /// The first-run wizard. Returns the theme chosen, or `None` if it was abandoned. async fn wizard( screen: &mut Screen>, @@ -4294,6 +4492,14 @@ async fn wizard( theme: Theme, ) -> Result, String> { let mut wizard = Wizard::new(theme); + // **What the catalogue read already returned, kept instead of thrown away.** + // The wizard reads the provider's catalogue to offer a model list, and until + // 0.22.0 mapped every row down to its id and dropped the price on it. Holding + // the rows here is what lets the file this wizard writes arrive with prices in + // it, at the cost of no second call: the fetch is the one that was already + // being made, one step earlier in this same loop. + let mut served: Vec = Vec::new(); + let mut priced_for: Option = None; loop { screen .draw(|frame| wizard.render(frame, frame.area())) @@ -4333,8 +4539,9 @@ async fn wizard( match verify::credential(&spec).await { Ok(()) => { if let Progress::Catalogue(spec) = wizard.verified() { - let models = verify::catalogue(&spec).await; - wizard.catalogue(models); + served = verify::served(&spec, None).await; + wizard.catalogue(ids(&served)); + priced_for = Some(spec); } } Err(message) => { @@ -4343,19 +4550,28 @@ async fn wizard( } } Progress::Catalogue(spec) => { - let models = verify::catalogue(&spec).await; - wizard.catalogue(models); + served = verify::served(&spec, None).await; + wizard.catalogue(ids(&served)); + priced_for = Some(spec); } Progress::Write(path, contents) => { settings::write(&path, &contents) .map_err(|error| format!("could not write {}: {error}", path.display()))?; let theme = wizard.theme(); - screen - .commit(&[ - theme.notice(Tone::Success, format!("wrote {}", path.display())), - ratatui::text::Line::from(""), - ]) - .map_err(|error| error.to_string())?; + let mut lines = vec![theme.notice( + Tone::Success, + format!("wrote {}", path.display()), + )]; + // **The prices go into the file the wizard just wrote, not into a + // second one.** `settings::render` cannot carry them: it runs + // before the credential is checked and therefore before any + // catalogue has been read, and a section written from a fetch that + // has not happened would be a date with nothing behind it. + if let Some(spec) = &priced_for { + lines.extend(fill_prices(&path, spec, std::mem::take(&mut served), 0, &theme)); + } + lines.push(ratatui::text::Line::from("")); + screen.commit(&lines).map_err(|error| error.to_string())?; return Ok(Some(theme)); } } diff --git a/src/prices.rs b/src/prices.rs index 7f30652..89001ee 100644 --- a/src/prices.rs +++ b/src/prices.rs @@ -199,6 +199,90 @@ pub fn changes(existing: Option<&io_harness::pricing::PriceTable>, fresh: &Catal .collect() } +/// What a refresh found, committed before anything is written. +/// +/// **Everything shown before a byte moves, and the reason is sharper than +/// courtesy.** io-cli cannot tell a rate the operator corrected by hand from one +/// an older catalogue served: the file records a number, not where it came from. +/// So it does not guess. It lists every rate that would move, with what it was +/// and what it would become, and an operator who finds their own correction in +/// that list can decline the whole refresh — which is the same answer `/import` +/// gives to the same problem, and the only honest one available. +/// +/// A refusal is reported here too rather than by the caller, so the sentence that +/// explains a short catalogue lives beside the rule that refuses it. +pub fn report( + catalogue: &Catalogue, + moved: &[Change], + existing: usize, + theme: &crate::theme::Theme, + width: u16, +) -> Vec> { + let mut rows: Vec = Vec::new(); + let source = source_word(&catalogue.source); + rows.push(crate::page::Row::note(format!( + "{source}, read {}: {} model{} served, {} priced", + catalogue.as_of, + catalogue.served, + if catalogue.served == 1 { "" } else { "s" }, + catalogue.rows.len(), + ))); + + if catalogue.too_short(existing) { + rows.push(crate::page::Row::caveat(format!( + "that is short enough against the {existing} model{} you already have to be a \ + truncated read rather than a price change, so nothing was written and the prices \ + you have were kept", + if existing == 1 { "" } else { "s" } + ))); + return crate::page::commit("prices", &rows, theme, width); + } + + if moved.is_empty() { + rows.push(crate::page::Row::note( + "no rate has moved since the last read, so nothing was written", + )); + return crate::page::commit("prices", &rows, theme, width); + } + + rows.push(crate::page::Row::Blank); + rows.push(crate::page::Row::heading(format!( + "{} rate{} would change", + moved.len(), + if moved.len() == 1 { "" } else { "s" } + ))); + for change in moved { + let value = match change.was { + Some(was) => format!("{} -> {}", rate(&was), rate(&change.now)), + // A model the table has never priced. Said as "new" rather than shown + // as a change from nothing, because an operator scanning for what + // moved is asking a different question from one asking what arrived. + None => format!("new, {}", rate(&change.now)), + }; + rows.push(crate::page::Row::fact(change.model.clone(), value)); + } + rows.push(crate::page::Row::Blank); + rows.push(crate::page::Row::caveat( + "if one of these is a rate you corrected by hand, it is about to be replaced: \ + io-cli records what a rate is and not where it came from, so it cannot tell yours \ + from an older catalogue's. Decline and edit the file if so.", + )); + crate::page::commit("prices", &rows, theme, width) +} + +/// One rate, as input and output per million tokens. +/// +/// The two dimensions that move a bill, and not the other three: a refresh report +/// is read to answer "did this get more expensive", and five numbers per row is a +/// table nobody scans. The whole rate is in the file for anyone who wants it. +fn rate(price: &Price) -> String { + format!( + "{} in / {} out", + crate::cost::money(price.input), + crate::cost::money(price.output) + ) +} + /// A `Price` as a one-line TOML inline table, naming only what is charged for. /// /// A dimension the vendor does not charge for is left out rather than written as diff --git a/tests/commands.rs b/tests/commands.rs index cf693d4..1682024 100644 --- a/tests/commands.rs +++ b/tests/commands.rs @@ -82,6 +82,13 @@ fn the_commands_are_the_commands() { "/fleet", "/image", "/clear", + // 0.22.0 — the two halves of one question, and two commands because + // they are two questions. `/cost` says what the work cost and + // `/stats` says whether it worked; every agent that has both keeps + // them apart. `/usage` is an alias for the first and is absent from + // this list, which is the rule every alias here follows. + "/cost", + "/stats", ], "the fuzzy palette is still 0.7.0; this list is written out so that adding \ a command is a decision somebody makes rather than a line somebody adds", @@ -1073,8 +1080,17 @@ fn the_memory_page_separates_the_two_memories_and_qualifies_a_cut_draw_count() { fn f17_usage_answers_and_is_never_listed() { use io_cli::commands::{group_of, palette, GROUPS}; - // It answers, with what `/status` answers. - assert_eq!(commands::parse("usage", &defaults(), &DARK), Action::Status); + // **It answers `/cost`, and until 0.22.0 it answered `/status`.** The + // argument for that was that `/status` was the closest thing this program had + // to a spending surface — it commits the token draw, the budgets and what is + // left of them. It stopped holding the moment a page existed whose whole + // subject is what has been spent: an operator typing the field's own word for + // that page and landing on the session's configuration is being answered a + // question they did not ask. + assert_eq!(commands::parse("usage", &defaults(), &DARK), Action::Cost); + assert_eq!(commands::parse("cost", &defaults(), &DARK), Action::Cost); + // And `/status` still answers itself, which is the half of this that did not + // change and the half a careless repoint would have taken with it. assert_eq!( commands::parse("status", &defaults(), &DARK), Action::Status diff --git a/tests/edit.rs b/tests/edit.rs index 68924e7..411f197 100644 --- a/tests/edit.rs +++ b/tests/edit.rs @@ -459,14 +459,20 @@ input = 1.0 ); // 3. The write half reaches the same section, and reaches it exactly once. + // `4.5` rather than `4.0`, and the digit after the point is the point. + // `assert_only_span_changed` reports the span between the longest common + // prefix and the longest common suffix, so replacing `3.0` with `4.0` reports + // `3` -> `4`: the shared `.0` is swallowed by the suffix scan. That would be a + // test asserting against its own helper's arithmetic rather than against the + // writer, and it would go on passing if the writer started truncating floats. let after = edit::apply( QUOTED, - &[Edit::set("prices.models.\"gpt-4.1\".input", "4.0")], + &[Edit::set("prices.models.\"gpt-4.1\".input", "4.5")], ) .unwrap(); assert_eq!( assert_only_span_changed(QUOTED, &after, "prices.models.\"gpt-4.1\".input"), - "4.0" + "4.5" ); assert_eq!( after.matches("[prices.models.\"gpt-4.1\"]").count(), diff --git a/tests/import.rs b/tests/import.rs index f1f273b..6e9d21a 100644 --- a/tests/import.rs +++ b/tests/import.rs @@ -21,6 +21,17 @@ const SECRET: &str = "sk-live-this-must-never-be-copied"; /// A variable no machine sets, so `${env:…}` for it is unresolvable. const UNSET: &str = "IO_CLI_IMPORT_TEST_TOKEN"; +/// A variable that is always exported, for the tests whose subject is something +/// other than an unresolvable reference. +/// +/// **[`UNSET`] is a premise, not a default.** io-harness treats an unset +/// `${env:…}` as a hard parse error and `configure::write` rolls the whole file +/// back when it hits one, so a test that uses `UNSET` while asserting a server +/// *arrives* is asserting two incompatible things and fails on the rollback — +/// which is correct behaviour reported as a broken test. A test about a quoted +/// server header wants an env reference that resolves. +const ALWAYS_SET: &str = "PATH"; + fn put(path: &Path, text: &str) { if let Some(parent) = path.parent() { std::fs::create_dir_all(parent).expect("a fixture directory"); @@ -375,7 +386,7 @@ fn a_codex_server_whose_name_must_be_quoted_still_arrives() { args = [\"--port\", \"8080\"]\n\ \n\ [mcp_servers.\"docs.v2\".env]\n\ - {UNSET} = \"{SECRET}\"\n" + {ALWAYS_SET} = \"{SECRET}\"\n" ), ); @@ -404,7 +415,7 @@ fn a_codex_server_whose_name_must_be_quoted_still_arrives() { assert_eq!(args, vec!["--port", "8080"]); assert_eq!( env, - vec![UNSET], + vec![ALWAYS_SET], "the env table of a quoted server was looked for under an unquoted header", ); assert!( From 4a2619395a316c7155fc200b3083e6b60b915b28 Mon Sep 17 00:00:00 2001 From: aakashpawar1999 Date: Thu, 27 Aug 2026 21:58:31 +0530 Subject: [PATCH 5/7] fix: what the adversarial review found behind a green suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Twenty-three findings, and the first one meant no operator would ever have got prices at all. **The first fill could not be written.** Both write paths pushed two bare `Edit::set`s into `[app.io-cli.prices]`, a section no file io-cli has ever written carries. Every edit in a batch resolves against the document as it was before the batch, so both fell to the append arm, each emitted its own header, and the read-back refused the lot — the exact defect `Edit::section` was added to fix, one section over, on the two edits the driver adds itself. It survived that fix because nothing under `tests/` can link the driver, so the bookkeeping edits now live in `prices` where a test can reach them. **`/clear` kept the previous conversation's money on the footer.** `forget_run` cleared every other run-scoped field and not `cost`, so a cleared session drew a currency figure beside a blank token count. **`source_url` could not work.** `named`'s catch-all discards every row for a `compatible` endpoint, so io-cli fetched the operator's own catalogue and threw it away — while `settings.rs` and the README both called that key the only way a self-hosted operator gets prices. A named catalogue is not narrowed: the operator has already answered the question the filter exists to ask. **A filled table would have turned `/config` into a thousand-row picker.** io-harness records leaf keys, so every model contributed two to five origins, and the `CATALOGUE` comment saying `[prices.models]` is deliberately not a row was being undone by the sweep below it. The rest: `note_cost` ran on every event under a comment saying it ran on steps; the footer's new trim popped the interrupt hint before any counter, so a full row lost `esc stops` exactly when a turn was running; an unrecorded model count rendered as a measured `0`; a failed catalogue read reported "no rate has moved", a positive claim about a read that did not happen; `too_short` compared against io-cli's own record rather than the file, so it was inert in every case it exists for; a real cost under a hundredth of a cent rendered `$0.0000`; the floor caveat pointed at a cost row that is suppressed when nothing is priced; "unpriced" meant two different things on one page; and a `[prices.models.""]` sub-table — legal TOML io-harness reads — is now refused by name instead of producing a duplicate-key error. Three test defects went with them: an assertion pinning the `models: 0` bug, a `contains("0 ms")` that is satisfied by `300 ms`, and a width sweep whose floor assumed the key hint was still droppable. 83 binaries, 1,164 passed, 0 failed, 24 ignored. --- CHANGELOG.md | 141 +++++ README.md | 13 +- docs/config.example.toml | 4 +- src/commands.rs | 9 +- src/configure.rs | 17 +- src/cost.rs | 153 ++++-- src/edit.rs | 69 ++- src/main.rs | 156 ++++-- src/prices.rs | 282 +++++++++- src/settings.rs | 4 +- src/stats.rs | 29 +- src/status.rs | 39 +- src/verify.rs | 60 ++- tests/cost.rs | 855 ++++++++++++++++++++++++++++++ tests/edit.rs | 83 +++ tests/page.rs | 519 +++++++++++++++++++ tests/palette.rs | 17 +- tests/prices.rs | 1066 ++++++++++++++++++++++++++++++++++++++ tests/stats.rs | 638 +++++++++++++++++++++++ tests/status.rs | 214 +++++++- 20 files changed, 4168 insertions(+), 200 deletions(-) create mode 100644 tests/cost.rs create mode 100644 tests/page.rs create mode 100644 tests/prices.rs create mode 100644 tests/stats.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index f4d2686..a5037f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,147 @@ All notable changes to this project are documented here. The format follows ## [Unreleased] +## [0.22.0] - 2026-08-27 + +What the work cost and whether it worked — two questions this interface has been +carrying the answers to since io-harness 0.18.0 and has never asked. + +**`/cost` commits what has been spent: this run, this session, by model, by day.** +Tokens and money, four sections, every figure a row already in `runs.db`. The +harness has recorded one row per provider call for four releases — the model, the +prompt, completion, cache-read, cache-write and reasoning split, the latency and +the time to first token — and io-cli reported token counts and treated the money +question as unanswerable. It was answerable then. + +**Nothing on that page is estimated, and three rules follow from it.** Cache reads +and cache writes are shown as the breakdown of the prompt they already are, never +added on top: they live inside `prompt_tokens`, and adding them would over-report +every cached turn, which is most of them. A call the provider reported no usage +for is **unknown**, never free — io-harness stores it as SQL `NULL` for exactly +that reason, and summing it as zero would report a turn that cost something as a +turn that cost nothing. And a model with no rate in the table makes the total a +**floor** rather than a total, which the page says in that word, with the number +of calls it applies to, and on the row itself in the grouped sections — a reader +scanning for the largest figure has to see which of them is incomplete without +reading past the list. io-harness's own pricing documentation calls a renderer +that hides that count "lying by omission", and it is right. + +**`/stats` commits how the runs have gone**: runs by outcome, runs by day, the +first-try counts, gate failures by phase, recovery, the slowest calls and the time +to first token over the last 200 runs, and what the store holds on disk. **First-try +is io-harness's own definition** — finished *and* successful *and* carrying no +gate failure — and it arrives as three counts rather than as a rate, because the +denominator is a choice: first-try over every run counts runs that are still +going. Where a share is drawn, the row names what it is a share *of*, instead of +printing a percentage whose meaning two readers would reconstruct differently. The +sandbox gate's phases and the review, command and contains gates are two +vocabularies that do not overlap, so they stay two lists; merging them would +produce a chart whose categories mean two things. Nothing here compacts anything — +`Store::compact` is a full `VACUUM` needing free disk roughly equal to the file, +and a page that reported free space and then reclaimed it would be a page that +surprised somebody mid-session. + +**They are two commands rather than one because they are two questions.** Every +agent that has both keeps them apart, and a single screen carrying thirteen +sections is one nobody reads to the end. `/usage` is an alias for `/cost` and +earns no row in any table — it resolved to `/status` until now, which was the +closest thing there was to an answer and was not one. + +**io-cli compiles no prices in, and this is the part worth reading twice.** A rate +baked into a binary is a promise the binary cannot keep: providers move prices +without announcing it, a release cadence is not a pricing cadence, and an operator +reading a confident wrong number is worse off than one reading no number at all. +So the table ships **empty**, and an install that has connected nothing sees +tokens and no currency. It is filled from the model catalogue the operator's own +provider serves — the same fetch `io setup` has made since 0.1.0 to offer a list +of models, whose `price`, `price_tiers` and `price_source` it read off the same +row and threw away. No JSON is parsed here and no dependency is added: io-harness +did the parsing and the unit conversion. The rates are written into `[prices]`, +which is io-harness's own section — `as_of` for the date, one line per model under +`[prices.models]`, a row per line so that correcting one by hand is a one-line +change you can find again. + +**Refreshing shows every rate that would move before it writes anything**, with +what each was and what it would become, and declining writes nothing — the shape +`/import` established in 0.21.0. That is not courtesy. io-cli records what a rate +*is* and never where it came from, so it cannot tell a correction you made by hand +from a value an older catalogue served; it does not guess, it shows you and lets +you refuse the lot. **A refetch that comes back empty, or far shorter than the +table it would replace, is refused and the old table kept** — the one failure in +this area that loses money quietly, because a truncated response replacing a full +table with a handful of rows would turn most of an operator's spending into +"unpriced" and shrink their reported bill with nothing failing anywhere. A first +fill has nothing to compare against and is never refused. Rows for models the +catalogue no longer serves are left alone rather than pruned: io-harness prices a +call by the model name on it, so an old row is what prices an old run correctly. + +**Whose price it is gets said, on every surface that draws money.** OpenAI and +Anthropic publish no prices on any endpoint — their model endpoints +carry capabilities and limits and no cost field, and their cost APIs report what +was *spent* rather than what a token *costs*. So for those two the rates +necessarily come from the reference catalogue rather than from the vendor, and the +page names which and on what date. io-harness models the distinction already, and +io-cli carries it through instead of flattening it into the connected provider's +name. On OpenRouter the two coincide, because the reference catalogue is +OpenRouter's own. + +**The status line carries a cost field**, right of the token count it is derived +from, so the two read in the order they are computed in. It is **absent — not +`$0`** — where there is no price table, where the table prices none of the models +this run called, or where nothing has run. Those are three different things and +none of them means free. + +**Three new keys and one new section for a file you already have.** +`[app.io-cli.prices]` carries `source_url`, which names a catalogue to read +instead of io-harness's default and is the only way an operator on a self-hosted +or `compatible` endpoint gets prices at all; and `source` and `models`, which +record what the last read was and how many models it priced and are written by a +fetch rather than by hand. They are in io-cli's own section rather than beside the +rates because `[prices]` is `deny_unknown_fields` and carries exactly `as_of` and +`models` — a key of io-cli's own put next to them would not be ignored, it would +make the operator's whole configuration file unreadable. `/config` gains +`prices.as_of` and `app.io-cli.prices.source_url` as rows; `[prices.models]` is +deliberately not one, because it is a list rather than a setting. + +**`/image`, `/copy` and `/copy diff` move from inspect to this turn**, and it is a +correction rather than a way of making room — the same sentence 0.19.0 wrote about +`/mcp` and `/provider`, and it is worth being able to say twice. All three act on +the turn that just finished; none of them asks the store a question, which is what +**inspect** means. They were filed there because they *show* something, and +showing is not the same as inspecting. `/cost` and `/stats` are what made it worth +correcting: no group may hold more than ten, **inspect** stood at nine, and the +choice was between re-filing three commands that were in the wrong group and +filing two more that would have been. Weakening the bound would have given up the +thing it protects — the grouped menu not turning back into the flat list of thirty +it replaced. + +**Two status-line priority inversions, both corrected here because this release +adds another counter to that row.** The footer dropped its whole right-hand group +— the policy layer, the containment mode, the planning phase — to keep every +counter, so a narrow terminal gave up the standing modes that explain why nothing +is happening in order to keep a number about what already happened; the counters +yield now. And `fields` pushed `planning` right of the counters, so the narrow +line dropped it before the token count — the exact inversion of the rule written +five lines above it in the same file, standing for four releases. + +**Four defects carried over from 0.21.0 are fixed.** `edit::split_path` is +quote-aware: a dot inside a quoted key is not a separator, TOML spells a bare key +out of `A-Za-z0-9_-` and nothing else, and a model id or MCP server name carrying +a dot was cut in half — the read half answered nothing and the write half appended +a second copy of a table that was already there, surfacing as an unexplainable +"the edit would have produced a file that does not parse". Removing or moving a +TOML entry no longer carries the *next* section's comment block away with it. +`move_entry` into the last position of a file with no trailing newline no longer +concatenates onto the final value. And a loose `CONVENTIONS.md` or `CLAUDE.md` +inside a `skills/` or `plugins/` directory is imported as a **skill**: it matched +on its basename before, and its whole body was appended into the instructions file +loaded on every turn, forever, instead of a named skill read on demand. Where a +file is decides what it is, ahead of what it is called. + +**crossterm moves to 0.29**, taken together with ratatui's `crossterm_0_29` +feature so the two agree on one version of the terminal backend. The dependabot +ignore that held it back is gone. + ## [0.21.0] - 2026-08-27 What you already told another agent tool, brought across once — and two surfaces diff --git a/README.md b/README.md index 35d8904..ff76570 100644 --- a/README.md +++ b/README.md @@ -154,6 +154,13 @@ name one — `gpt-5` could be OpenAI or any of the twenty-one presets pointed at compatible endpoint — so io hands you the id and the entry is built once you have chosen the vendor. +**Where a file *is* decides what it is, ahead of what it is called.** A loose +`CONVENTIONS.md` — or `CLAUDE.md`, or `MEMORY.md` — sitting inside a `skills/` or +`plugins/` directory is a skill, and from 0.22.0 it is imported as one. It used to +match on its basename and be appended whole into the instructions file that is +loaded on **every turn, forever**, instead of being a named skill the model reads +on demand. + **The whole plan is on screen before a single byte is written.** One row per thing found, saying where it came from and where it would go, and you accept them item by item. What you did not accept is not written. A cancelled import is not a @@ -652,9 +659,9 @@ against and is never refused. Rows for models the catalogue no longer serves are left alone, because io-harness prices a call by the model name on it and an old row is what prices an old run correctly. -**Whose price it is gets said, on every surface that draws money.** OpenAI, -Anthropic and Google publish no prices on any endpoint — their model endpoints -carry capabilities and limits and no cost field — so for those three the rates +**Whose price it is gets said, on every surface that draws money.** OpenAI and +Anthropic publish no prices on any endpoint — their model endpoints +carry capabilities and limits and no cost field — so for those two the rates necessarily come from the reference catalogue rather than from the vendor. The page names which and on what date. On OpenRouter the two coincide, because the reference catalogue is OpenRouter's own. `[app.io-cli.prices] source_url` points diff --git a/docs/config.example.toml b/docs/config.example.toml index 6fff068..77350e7 100644 --- a/docs/config.example.toml +++ b/docs/config.example.toml @@ -314,9 +314,9 @@ net = "deny" # "unpriced" and shrink your reported bill with nothing failing anywhere. A first # fill has nothing to compare against and is never refused. # -# WHOSE PRICE IT IS. OpenAI, Anthropic and Google publish no prices on any +# WHOSE PRICE IT IS. OpenAI and Anthropic publish no prices on any # endpoint — their model endpoints carry capabilities and limits and no cost -# field. So for those three the rates necessarily come from the reference +# field. So for those two the rates necessarily come from the reference # catalogue rather than from the vendor, and every surface that draws money says # which and on what date. On OpenRouter the two coincide, because the reference # catalogue is OpenRouter's own. diff --git a/src/commands.rs b/src/commands.rs index 958bd9b..e9e73f9 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -350,14 +350,7 @@ pub const GROUPS: &[(Group, &[&str])] = &[ ( Group::Inspect, &[ - "/help", - "/status", - "/context", - "/expand", - "/fleet", - "/skills", - "/cost", - "/stats", + "/help", "/status", "/context", "/expand", "/fleet", "/skills", "/cost", "/stats", ], ), // **`/mcp` and `/provider` moved here in 0.19.0, and it is a correction rather diff --git a/src/configure.rs b/src/configure.rs index 4762f83..dfddaf6 100644 --- a/src/configure.rs +++ b/src/configure.rs @@ -253,11 +253,26 @@ pub fn settings(config: &Config) -> Vec { rows.push(setting(config, key)); } - // Anything a file named that the catalogue does not carry. + // Anything a file named that the catalogue does not carry — **except the + // price table, which is a list and not a set of settings.** + // + // io-harness's origin index records *leaf* keys, so a filled `[prices.models]` + // contributes two to five of them per model: `prices.models..input`, + // `.output`, `.cache_read`, and so on. On an OpenRouter install that is well + // over a thousand rows, and `/config` — a picker an operator opens to change + // one setting — becomes a list of rates with every real setting buried above + // them. The `CATALOGUE` comment already says `[prices.models]` is deliberately + // not a row; this is what makes that true, because the sweep below was putting + // every one of them back. + // + // They would not even render: `record_origins` joins the path with `.` and + // does not quote, so a model id containing a dot arrives as a key that + // `edit::value_at` cannot resolve, and the row draws with no value at all. let mut extra: Vec = config .origins() .map(|(key, _)| key.to_string()) .filter(|key| !seen.contains(key)) + .filter(|key| !key.starts_with("prices.models.")) .collect(); extra.sort(); for key in extra { diff --git a/src/cost.rs b/src/cost.rs index 056a4f8..0440fdc 100644 --- a/src/cost.rs +++ b/src/cost.rs @@ -147,8 +147,15 @@ pub struct Provenance { pub as_of: Option, /// The catalogue, in words. `None` when nothing has been read yet. pub source: Option, - /// How many models the table prices, as the last read recorded it. - pub models: usize, + /// How many models the table prices, as the last read recorded it, or `None` + /// where nothing recorded it. + /// + /// **Absent is not zero, and this field had the bug the rest of the module is + /// built to prevent.** It was a `usize` defaulting to 0, so a `[prices]` an + /// operator wrote by hand — or one carried forward from an install older than + /// this release, neither of which has an `[app.io-cli.prices]` beside it — + /// made the footer read "0 models" over a table pricing four hundred. + pub models: Option, } impl Provenance { @@ -165,7 +172,7 @@ impl Provenance { Self { as_of: config.prices().map(|table| table.as_of().to_string()), source: prices.and_then(|p| p.source.clone()), - models: prices.and_then(|p| p.models).unwrap_or(0), + models: prices.and_then(|p| p.models), } } } @@ -223,7 +230,11 @@ pub fn committed( let turns = store.session_turns(id).map_err(|e| e.to_string())?; let mut calls: Vec = Vec::new(); for turn in &turns { - calls.extend(store.provider_calls(turn.run_id).map_err(|e| e.to_string())?); + calls.extend( + store + .provider_calls(turn.run_id) + .map_err(|e| e.to_string())?, + ); } // **Walked rather than grouped, because io-harness offers no session // grouping.** `spend_by_run` keys on a run and `spend_by_day` on a @@ -248,7 +259,9 @@ pub fn committed( rows.push(Row::Blank); rows.push(Row::heading("by day".to_string())); - rows.extend(grouped(store.spend_by_day(table).map_err(|e| e.to_string())?)); + rows.extend(grouped( + store.spend_by_day(table).map_err(|e| e.to_string())?, + )); rows.push(Row::Blank); rows.extend(footer(provenance)); @@ -262,26 +275,31 @@ fn section(total: &Total) -> Vec { return vec![Row::note("no provider calls")]; } let usage = &total.usage; - let mut rows = vec![ - Row::fact("calls", total.calls.to_string()), - Row::fact("cost", money(total.micros)), - Row::fact("prompt", tokens(usage.prompt_tokens)), - ]; + let mut rows = vec![Row::fact("calls", total.calls.to_string())]; + // **No currency at all when nothing was priced, rather than `$0`.** A cost row + // reading `$0` over a set of calls the table prices none of would be the + // invented number this whole module exists to avoid — and it would be the + // worst kind, because zero is a number an operator can believe. The token + // figures below are exact either way; it is only the money that is unknown. + if total.micros > 0 { + rows.push(Row::fact("cost", money(total.micros))); + } + rows.push(Row::fact("prompt", tokens(usage.prompt_tokens))); // The two cache figures are indented under the prompt they are part of, by // being named as parts of it rather than by being drawn further right — this // page has no columns to indent into, and a reader in `--plain` has only the // words. if usage.cache_read_tokens > 0 || usage.cache_write_tokens > 0 { rows.push(Row::fact( - " of which cache read", + "of which cache read", tokens(usage.cache_read_tokens), )); rows.push(Row::fact( - " of which cache written", + "of which cache written", tokens(usage.cache_write_tokens), )); rows.push(Row::fact( - " of which fresh", + "of which fresh", tokens( usage .prompt_tokens @@ -293,7 +311,7 @@ fn section(total: &Total) -> Vec { rows.push(Row::fact("completion", tokens(usage.completion_tokens))); if usage.reasoning_tokens > 0 { rows.push(Row::fact( - " of which reasoning", + "of which reasoning", tokens(usage.reasoning_tokens), )); } @@ -303,7 +321,7 @@ fn section(total: &Total) -> Vec { usage.server_tool_requests.to_string(), )); } - rows.extend(caveats(total.unknown, total.unpriced)); + rows.extend(caveats(total.unknown, total.unpriced, total.micros > 0)); rows } @@ -314,12 +332,24 @@ fn grouped(spend: Vec) -> Vec { } let mut rows: Vec = Vec::new(); for group in spend { - let mut value = format!( - "{} · {} call{}", - money(group.cost_micros), - group.calls, - if group.calls == 1 { "" } else { "s" } - ); + // Same rule as `section` above: no currency at all where nothing was + // priced. `$0` beside a group whose every call used a model the table does + // not know reads as a group that was free, and it is the one wrong number + // an operator has no reason to doubt. + let mut value = if group.cost_micros > 0 { + format!( + "{} · {} call{}", + money(group.cost_micros), + group.calls, + if group.calls == 1 { "" } else { "s" } + ) + } else { + format!( + "{} call{}", + group.calls, + if group.calls == 1 { "" } else { "s" } + ) + }; if group.usage.total_tokens > 0 { value.push_str(&format!(" · {} tok", tokens(group.usage.total_tokens))); } @@ -327,7 +357,15 @@ fn grouped(spend: Vec) -> Vec { // **Never hidden, and on the row rather than in a note below it.** A // reader scanning for the largest figure has to be able to see which // of them is incomplete without reading past the list. - value.push_str(&format!(" · {} unpriced", group.unpriced_calls)); + // + // **"not priced" rather than "unpriced", because it is a wider set + // than the word means two sections up.** io-harness's + // `Spend::unpriced_calls` counts a call with no usage as unpriced; + // this module's `Total` splits those out as *unknown* and counts only + // a missing rate as unpriced. The same call is therefore in different + // buckets on the two halves of this page, and one word for both would + // have made the numbers look like they disagreed. + value.push_str(&format!(" · {} not priced", group.unpriced_calls)); } rows.push(Row::fact(group.key, value)); } @@ -335,20 +373,33 @@ fn grouped(spend: Vec) -> Vec { } /// The sentences that qualify a figure, if any qualify it. -fn caveats(unknown: u64, unpriced: u64) -> Vec { +fn caveats(unknown: u64, unpriced: u64, drew_cost: bool) -> Vec { let mut rows: Vec = Vec::new(); if unpriced > 0 { - rows.push(Row::caveat(format!( - "{unpriced} call{} used a model with no rate in the price table, so \ - the cost above is a floor and not a total", - if unpriced == 1 { "" } else { "s" } - ))); + // **Two sentences, because the row this used to point at is not always + // there.** `section` suppresses the cost row entirely when nothing was + // priced, so a run every call of which used an unknown model showed no + // cost and a caveat reading "the cost above is a floor" — pointing at + // nothing. + rows.push(Row::caveat(if drew_cost { + format!( + "{unpriced} call{} used a model with no rate in the price table, so the cost \ + above is a floor and not a total", + if unpriced == 1 { "" } else { "s" } + ) + } else { + format!( + "no cost is shown because the price table has a rate for none of the {unpriced} \ + call{} here — the tokens above are exact, the money is simply unknown", + if unpriced == 1 { "" } else { "s" } + ) + })); } if unknown > 0 { rows.push(Row::caveat(format!( "{unknown} call{} reported no usage at all, which is unknown rather \ than free — neither its tokens nor its cost is in the figures above", - if unknown == 1 { " " } else { "s " } + if unknown == 1 { "" } else { "s" } ))); } rows @@ -356,21 +407,26 @@ fn caveats(unknown: u64, unpriced: u64) -> Vec { /// Where the prices came from and when, at the foot of the page. fn footer(provenance: &Provenance) -> Vec { - match (&provenance.as_of, &provenance.source) { - (Some(as_of), Some(source)) => vec![Row::note(format!( - "prices: {} model{} read from {source} on {as_of}", - provenance.models, - if provenance.models == 1 { "" } else { "s" } - ))], - (Some(as_of), None) => vec![Row::note(format!( - "prices: {} model{}, dated {as_of}, from a source this install did not record", - provenance.models, - if provenance.models == 1 { "" } else { "s" } - ))], - _ => vec![Row::note( - "prices: none configured".to_string(), - )], - } + let Some(as_of) = &provenance.as_of else { + return vec![Row::note("prices: none configured".to_string())]; + }; + // **The count is left out when nothing recorded one, rather than shown as + // zero.** A hand-written `[prices]`, or one from an install older than this + // release, has no `[app.io-cli.prices]` beside it — and "0 models" over a + // table pricing four hundred is the same class of invented figure as a `$0` + // over an unpriced call. + let how_many = match provenance.models { + Some(1) => "1 model".to_string(), + Some(n) => format!("{n} models"), + None => "prices".to_string(), + }; + let sentence = match &provenance.source { + Some(source) => format!("prices: {how_many} read from {source} on {as_of}"), + None => { + format!("prices: {how_many}, dated {as_of}, from a source this install did not record") + } + }; + vec![Row::note(sentence)] } /// Micro-units as money. @@ -381,6 +437,15 @@ fn footer(provenance: &Provenance) -> Vec { /// arithmetic throughout — a bill is not a floating-point quantity, and /// `cost_micros` is exact. pub fn money(micros: u64) -> String { + // **A real cost smaller than the smallest figure this renders is said to be + // smaller, not rounded to nothing.** Four decimals stop at a hundredth of a + // cent, and a cheap model answering a short prompt genuinely lands under it; + // truncating there would print `$0.0000`, which reads as free for a call that + // was not. The `$0` below is reserved for a cost that really is zero, which a + // free model has. + if micros > 0 && micros < 100 { + return "<$0.0001".to_string(); + } if micros == 0 { return "$0".to_string(); } diff --git a/src/edit.rs b/src/edit.rs index 1f74337..4539531 100644 --- a/src/edit.rs +++ b/src/edit.rs @@ -70,6 +70,8 @@ enum Kind { Set, /// Append a whole `[[path]]` entry to the end of the document. Append, + /// Create the `[path]` section with a whole body, refusing if it exists. + Section, /// Remove a whole `[[path]]` entry, or a `[path]` section, bytes and all. Remove, /// Move one `[[path]]` entry to another position in its array. @@ -108,6 +110,32 @@ impl Edit { } } + /// Create the `[path]` section with `body` as its whole contents. + /// + /// **The shape neither [`Edit::set`] nor [`Edit::append`] can express, and the + /// gap is not academic.** `append` writes `[[path]]`, an array-of-tables + /// entry. `set` writes one key, and when the section it names does not exist + /// it appends a whole new header to carry that one key — which is correct for + /// one key and catastrophic for many: sixty `set`s into a file with no + /// `[prices.models]` each append their own `[prices.models]` header, because + /// every edit in a batch is resolved against the document as it was *before* + /// the batch. The result is sixty duplicate table definitions, and the whole + /// write is refused by the read-back — so the very first fill of a price table + /// could never succeed. + /// + /// `body` is the section's own `key = value` lines without the header; this + /// writes the header. **It refuses when the section already exists**, rather + /// than replacing it: a caller with an existing section wants `set` per key, + /// which preserves every row this one does not name, and silently discarding + /// an operator's hand-added rows is not a thing to do by accident. + pub fn section(path: impl Into, body: impl Into) -> Self { + Self { + path: path.into(), + value: body.into(), + kind: Kind::Section, + } + } + /// Remove the whole `[path]` section or `[[path]][index]` entry. pub fn remove(path: impl Into) -> Self { Self { @@ -203,6 +231,36 @@ pub fn apply(text: &str, edits: &[Edit]) -> Result { splices.push((text.len()..text.len(), block)); continue; } + Kind::Section => { + // **Refused rather than replaced when it already exists**, and the + // refusal is the point: a caller holding an existing section wants + // `set` per key, which leaves every row it does not name alone. + // Replacing here would silently discard whatever the operator had + // added by hand, from a call site whose author believed they were + // adding rows. + let names = segments(&edit.path)?; + if regions.iter().any(|r| r.path == names) { + return Err(format!( + "`[{}]` is already in this file, so it was not written whole — \ + a section that exists is edited key by key, which is what keeps \ + the rows nobody named", + edit.path + )); + } + let header = names + .iter() + .map(|name| spell(name)) + .collect::>() + .join("."); + let mut block = String::new(); + if !text.is_empty() && !text.ends_with('\n') { + block.push('\n'); + } + let body = edit.value.trim_end(); + block.push_str(&format!("\n[{header}]\n{body}\n")); + splices.push((text.len()..text.len(), block)); + continue; + } Kind::Move => { let (from, to) = edit .value @@ -286,10 +344,7 @@ pub fn apply(text: &str, edits: &[Edit]) -> Result { // Not `region.body.end`: that is the first byte of the next // header, and the lines just above it document the next section // rather than this one. See [`removal_end`]. - splices.push(( - region.start..removal_end(text, ®ion.body), - String::new(), - )); + splices.push((region.start..removal_end(text, ®ion.body), String::new())); continue; } Kind::Set => {} @@ -353,11 +408,7 @@ pub fn apply(text: &str, edits: &[Edit]) -> Result { if !text.is_empty() && !text.ends_with('\n') { block.push('\n'); } - block.push_str(&format!( - "\n[{header}]\n{} = {}\n", - spell(&key), - edit.value - )); + block.push_str(&format!("\n[{header}]\n{} = {}\n", spell(&key), edit.value)); splices.push((text.len()..text.len(), block)); } } diff --git a/src/main.rs b/src/main.rs index a0d532f..16c8ad2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2450,14 +2450,12 @@ async fn loop_over Result>( // row names a setting and puts it in the composer; a reader // scanning for `policy.defaults.write` should not have to step // over something that does work on the way there. - rows.push(io_cli::configure::refresh_row( - &io_cli::configure::setting(&config, "prices.as_of"), - )); + rows.push(io_cli::configure::refresh_row(&io_cli::configure::setting( + &config, + "prices.as_of", + ))); paths.push(io_cli::configure::REFRESH_PRICES.to_string()); - picker = Some(( - Picker::new("Which setting?", rows), - Pick::Config(paths), - )); + picker = Some((Picker::new("Which setting?", rows), Pick::Config(paths))); } // A key with no value is a question. Naming what is in force and // which file decided it is the answer, and nothing is written. @@ -2875,8 +2873,7 @@ async fn loop_over Result>( screen.commit(&lines).map_err(|error| error.to_string())?; } Action::Stats => { - let lines = - io_cli::stats::committed(&store, &app.theme, screen.width())?; + let lines = io_cli::stats::committed(&store, &app.theme, screen.width())?; screen.commit(&lines).map_err(|error| error.to_string())?; } // Reached only at an idle prompt: while a turn runs the driver's @@ -3576,7 +3573,7 @@ async fn turn( // same turn. The trace is the fallback for the window between a // step landing and the first completion call being seen. note_context(app, store, &event, seen, &contract); - note_cost(app, store, config, event.run_id); + note_cost(app, store, config, &event); note_fleet(app, store, &event, &contract); commit_viewed(screen, app, &root, policy, &event)?; commit_fold(app, store, &event, &mut folding); @@ -3915,7 +3912,7 @@ async fn turn( // And on the drain too, for the same race: the last step of a turn is the // one whose event the select loop loses, and it is also the step that // makes the largest single difference to what the turn cost. - note_cost(app, store, config, event.run_id); + note_cost(app, store, config, &event); note_fleet(app, store, &event, &contract); // And the picture, for the same reason and the same race: a `view_image` // on the turn's last step is exactly the one the drain would otherwise @@ -4156,9 +4153,27 @@ fn note_context( /// table captured when the session opened would go on reporting the old rate for /// the rest of it — which is the one thing a figure with a currency in front of it /// must not do. -fn note_cost(app: &mut App, store: &Store, config: &io_harness::Config, run_id: i64) { +fn note_cost( + app: &mut App, + store: &Store, + config: &io_harness::Config, + event: &io_harness::RunEvent, +) { + // **Anchored on a step, for the reason `note_context` three functions up is** + // — and the doc on both this and `Status::note_cost_from` claimed it before + // the code did. Without the gate every event in the stream, token deltas + // included, cost one `provider_calls` read and one full `Total::of`; with it, + // the read happens when the answer can actually have changed. `Finished` + // carries the run's own totals and is the one that settles the last step, + // which the drain would otherwise lose. + if !matches!( + event.kind, + io_harness::EventKind::Step { .. } | io_harness::EventKind::Finished { .. } + ) { + return; + } let table = io_cli::cost::table(config); - app.status.note_cost_from(store, run_id, &table); + app.status.note_cost_from(store, event.run_id, &table); } /// Report a fold that was asked for — once, and only from the event that @@ -4364,6 +4379,20 @@ fn fill_prices( ), )]; } + // Read rather than assumed, even though the wizard has just written this file + // from `settings::render` and `settings::render` carries no `[prices]` at + // all. The assumption is true today and is exactly the kind that stops being + // true when somebody adds a section to the rendered file, and the failure it + // would cause — a refused write on a first run — is one nobody would connect + // back to here. + let text = std::fs::read_to_string(path).unwrap_or_default(); + if let Some(refusal) = io_cli::prices::refusal(&text) { + return vec![theme.notice(Tone::Warning, refusal)]; + } + // Counted off the file rather than off io-cli's own record of a previous + // write, which is absent on a first fill and on any hand-written table — see + // `prices::priced_in`. + let existing = io_cli::prices::priced_in(&text).max(existing); if catalogue.too_short(existing) { return vec![theme.notice( Tone::Warning, @@ -4376,14 +4405,13 @@ fn fill_prices( )]; } let source = io_cli::prices::source_word(&catalogue.source); - let mut edits = catalogue.edits(); - edits.push(io_cli::edit::Edit::set( - "app.io-cli.prices.source", - format!("\"{source}\""), - )); - edits.push(io_cli::edit::Edit::set( - "app.io-cli.prices.models", - catalogue.rows.len().to_string(), + let mut edits = catalogue.edits(io_cli::prices::has_models_section(&text)); + edits.extend(io_cli::prices::bookkeeping( + &source, + catalogue.rows.len(), + io_cli::edit::sections(&text) + .iter() + .any(|p| p == &["app", "io-cli", "prices"]), )); match io_cli::edit::write(path, &edits) { Ok(()) => vec![theme.notice( @@ -4430,45 +4458,65 @@ async fn refresh_prices( .as_ref() .and_then(|s| s.prices.as_ref()) .and_then(|p| p.source_url.clone()); - let existing = stored - .as_ref() - .and_then(|s| s.prices.as_ref()) - .and_then(|p| p.models) - .unwrap_or(0); + // The scope that already holds the prices, so a refresh lands where the last + // one did. A higher-precedence scope would shadow rather than update, and the + // operator would be looking at the old numbers in the file they edit. + let scope = io_cli::configure::setting(config, "prices.as_of") + .decided + .scope() + .unwrap_or(io_harness::config::Scope::User); + // Read of the file this write is aimed at, not of the merged configuration: a + // `[prices.models]` in a lower-precedence scope is not one this scope's file + // can have keys set into, and treating it as though it were would try to edit + // a section that is not in this document. + let text = io_cli::configure::scope_path(root, scope) + .and_then(|path| std::fs::read_to_string(path).ok()) + .unwrap_or_default(); + if let Some(refusal) = io_cli::prices::refusal(&text) { + app.record(Tone::Warning, refusal); + return Ok(()); + } + // Counted off the file, and only then falling back to io-cli's own record. + // The record is absent for a hand-written table and for every install from + // before this release, which is precisely when a truncated read would do the + // most damage. + let existing = io_cli::prices::priced_in(&text).max( + stored + .as_ref() + .and_then(|s| s.prices.as_ref()) + .and_then(|p| p.models) + .unwrap_or(0), + ); app.say(Tone::Muted, "re-reading the catalogue…"); - let served = verify::served(spec, source.as_deref()).await; + let served = verify::served(source.as_deref()).await; let secs = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|since| since.as_secs()) .unwrap_or(0); - let catalogue = - io_cli::prices::Catalogue::of(spec, served, io_cli::prices::date(secs)); + let catalogue = match source.as_deref().filter(|url| !url.is_empty()) { + Some(url) => { + io_cli::prices::Catalogue::named(spec, served, io_cli::prices::date(secs), url) + } + None => io_cli::prices::Catalogue::of(spec, served, io_cli::prices::date(secs)), + }; let moved = io_cli::prices::changes(config.prices().as_ref(), &catalogue); let lines = io_cli::prices::report(&catalogue, &moved, existing, &app.theme, screen.width()); screen.commit(&lines).map_err(|error| error.to_string())?; - if moved.is_empty() || catalogue.too_short(existing) { + if catalogue.rows.is_empty() || moved.is_empty() || catalogue.too_short(existing) { return Ok(()); } - // The scope that already holds the prices, so a refresh lands where the last - // one did. A higher-precedence scope would shadow rather than update, and the - // operator would be looking at the old numbers in the file they edit. - let scope = io_cli::configure::setting(config, "prices.as_of") - .decided - .scope() - .unwrap_or(io_harness::config::Scope::User); let source_word = io_cli::prices::source_word(&catalogue.source); - let mut edits = catalogue.edits(); - edits.push(io_cli::edit::Edit::set( - "app.io-cli.prices.source", - format!("\"{source_word}\""), - )); - edits.push(io_cli::edit::Edit::set( - "app.io-cli.prices.models", - catalogue.rows.len().to_string(), + let mut edits = catalogue.edits(io_cli::prices::has_models_section(&text)); + edits.extend(io_cli::prices::bookkeeping( + &source_word, + catalogue.rows.len(), + io_cli::edit::sections(&text) + .iter() + .any(|p| p == &["app", "io-cli", "prices"]), )); match io_cli::configure::write(root, scope, &edits) { Ok(()) => app.record( @@ -4539,7 +4587,7 @@ async fn wizard( match verify::credential(&spec).await { Ok(()) => { if let Progress::Catalogue(spec) = wizard.verified() { - served = verify::served(&spec, None).await; + served = verify::named(&spec, verify::served(None).await); wizard.catalogue(ids(&served)); priced_for = Some(spec); } @@ -4550,7 +4598,7 @@ async fn wizard( } } Progress::Catalogue(spec) => { - served = verify::served(&spec, None).await; + served = verify::named(&spec, verify::served(None).await); wizard.catalogue(ids(&served)); priced_for = Some(spec); } @@ -4558,17 +4606,21 @@ async fn wizard( settings::write(&path, &contents) .map_err(|error| format!("could not write {}: {error}", path.display()))?; let theme = wizard.theme(); - let mut lines = vec![theme.notice( - Tone::Success, - format!("wrote {}", path.display()), - )]; + let mut lines = + vec![theme.notice(Tone::Success, format!("wrote {}", path.display()))]; // **The prices go into the file the wizard just wrote, not into a // second one.** `settings::render` cannot carry them: it runs // before the credential is checked and therefore before any // catalogue has been read, and a section written from a fetch that // has not happened would be a date with nothing behind it. if let Some(spec) = &priced_for { - lines.extend(fill_prices(&path, spec, std::mem::take(&mut served), 0, &theme)); + lines.extend(fill_prices( + &path, + spec, + std::mem::take(&mut served), + 0, + &theme, + )); } lines.push(ratatui::text::Line::from("")); screen.commit(&lines).map_err(|error| error.to_string())?; diff --git a/src/prices.rs b/src/prices.rs index 89001ee..66bae22 100644 --- a/src/prices.rs +++ b/src/prices.rs @@ -15,20 +15,36 @@ //! [`crate::verify::catalogue`] has read the model catalogue since 0.1.0, to offer //! the wizard a list of models. It mapped every [`ModelInfo`] down to its `id` and //! discarded the `price`, `price_tiers` and `price_source` on the same row. This -//! module keeps them. No JSON is parsed here and no dependency is added: io-harness -//! did the parsing and the unit conversion, and `ModelInfo::price` arrives already -//! normalised into [`Price`]'s micro-units per million tokens. +//! module keeps the price and the source. No JSON is parsed here and no dependency +//! is added: io-harness did the parsing and the unit conversion, and +//! `ModelInfo::price` arrives already normalised into [`Price`]'s micro-units per +//! million tokens. +//! +//! **`price_tiers` is read and deliberately dropped, and the consequence is +//! stated rather than hidden.** io-harness's `[prices]` section carries `as_of` +//! and a map of `Price`, and is `deny_unknown_fields` — there is no TOML surface +//! for a tier, and `PriceTier` is constructible only in Rust. So a model whose +//! vendor charges more above a long-prompt threshold is priced here at its base +//! rate, and `/cost` cannot mark that as a floor because nothing in the store says +//! which calls crossed the line. It is an **understated** figure rather than an +//! invented one, and it is in the release record as a known limitation. //! //! # Whose price it is //! -//! For three of the four providers io-cli knows, the answer is not "the vendor's". -//! OpenAI, Anthropic and Google publish no prices on any endpoint — their model +//! For two of the three vendors io-cli can connect to, the answer is not "the +//! vendor's". OpenAI and Anthropic publish no prices on any endpoint — their model //! endpoints carry capabilities and limits and no cost field, and their cost APIs //! report what was *spent* rather than what a token *costs*. The reference //! catalogue does carry rows for their models, which is why //! [`crate::verify::catalogue`] strips the `anthropic/` and `openai/` prefixes off //! them to build its list. //! +//! The fourth `ProviderSpec` is `Compatible`, and the reference catalogue cannot +//! speak for it at all: a list of one vendor's models says nothing about what a +//! server it has never heard of serves. That operator names their own catalogue +//! with `app.io-cli.prices.source_url`, which [`Catalogue::named`] reads without +//! narrowing. +//! //! So an operator on Anthropic gets prices *about* Anthropic *from* the reference //! catalogue, and every surface that draws money says which. io-harness models the //! distinction already — [`PriceSource::Vendor`] against @@ -100,9 +116,60 @@ impl Catalogue { /// testable without a socket — the fetch itself is one io-harness call and /// has nothing in it worth a test double. pub fn of(spec: &ProviderSpec, models: Vec, as_of: impl Into) -> Self { - let served = models.len(); + Self::read(spec, models, as_of, None) + } + + /// The same, from a catalogue the operator named rather than the default. + /// + /// **A named catalogue is not narrowed, and that is the whole of what + /// `app.io-cli.prices.source_url` means.** The default catalogue is one + /// vendor's view of the entire field, so its rows have to be cut down to the + /// provider in force — and for a `compatible` endpoint that cut can only + /// remove everything, because a reference list cannot say what a server it has + /// never heard of serves. An operator who set `source_url` has answered that: + /// they pointed io-cli at the catalogue their own endpoint publishes, so every + /// row of it is theirs and the ids are already spelled the way their provider + /// names them. + /// + /// The source is recorded as that URL rather than as the vendor, because it is + /// still not the vendor speaking — it is a list the operator chose, and a page + /// that drew money under the provider's name would be attributing it wrongly. + pub fn named( + spec: &ProviderSpec, + models: Vec, + as_of: impl Into, + url: &str, + ) -> Self { + Self::read(spec, models, as_of, Some(url)) + } + + fn read( + spec: &ProviderSpec, + models: Vec, + as_of: impl Into, + url: Option<&str>, + ) -> Self { + if let Some(url) = url.filter(|url| !url.is_empty()) { + let served = models.len(); + let rows = crate::verify::priced(models); + return Self { + as_of: as_of.into(), + source: PriceSource::Reference(url.to_string()), + rows, + served, + }; + } let vendor = matches!(spec, ProviderSpec::OpenRouter { .. }); - let rows = crate::verify::priced(spec, models); + // **Counted after the filter, not before it.** The reference catalogue + // serves every model in the field; the number worth reporting is how many + // of them *this* provider serves, because that is the denominator the + // operator is being told their prices cover. Counting before the filter + // told an Anthropic operator their fifteen rates covered fifteen of four + // hundred and seventeen models, which is true of the catalogue and + // meaningless about them. + let mine = crate::verify::named(spec, models); + let served = mine.len(); + let rows = crate::verify::priced(mine); Self { as_of: as_of.into(), // **The one place the two coincide.** The reference catalogue is @@ -140,17 +207,45 @@ impl Catalogue { /// The edits that write this catalogue into a configuration file. /// - /// One `set` for the date and one per model, rather than one edit rewriting - /// the section whole. That is not a smaller diff for its own sake: a rate is - /// something the operator is invited to correct by hand, and a row per line is - /// what makes correcting one a one-line change they can find again. + /// A row per model rather than one enormous inline table, because a rate is + /// something the operator is invited to correct by hand and a row per line is + /// what makes correcting one a change they can find again. For OpenRouter that + /// is four hundred rows; as a single inline value it would be one + /// twenty-five-kilobyte line. + /// + /// **`has_section` decides the shape, and getting it wrong does not merely + /// produce a worse file — it produces no file.** Every edit in a batch is + /// resolved against the document as it was before the batch, so `set` on four + /// hundred keys of a `[prices.models]` that does not exist yet appends four + /// hundred `[prices.models]` headers and the read-back refuses the lot. A + /// first fill therefore writes the section whole, once; a refresh sets key by + /// key into the section that is already there. + /// + /// It is a parameter rather than something derived here because **it is a + /// question about the file, and this type has never seen the file.** The + /// obvious substitutes are both wrong: io-cli's own record of how many models + /// it last wrote is zero for a `[prices]` an operator wrote by hand, and + /// asking the `PriceTable` whether it prices any model this catalogue serves + /// answers `false` for a real section whose models the provider has since + /// replaced. [`has_models_section`] asks the file. /// - /// **Rows for models the catalogue no longer serves are left alone.** They are - /// not stale — io-harness prices a call by the model name on it, so an old row - /// is what prices an old run correctly, and `/cost` reports history as well as - /// today. - pub fn edits(&self) -> Vec { + /// That split has a second effect worth naming, because it is the behaviour + /// rather than an artefact: on a refresh, **rows for models the catalogue no + /// longer serves are left alone.** They are not stale. io-harness prices a + /// call by the model name recorded on it, so an old row is exactly what prices + /// an old run correctly, and `/cost` reports history as well as today. + pub fn edits(&self, has_section: bool) -> Vec { let mut edits = vec![Edit::set("prices.as_of", quoted(&self.as_of))]; + if !has_section { + let body = self + .rows + .iter() + .map(|(model, price)| format!("{} = {}", quoted(model), inline(price))) + .collect::>() + .join("\n"); + edits.push(Edit::section("prices.models", body)); + return edits; + } for (model, price) in &self.rows { edits.push(Edit::set( format!("prices.models.{}", quoted(model)), @@ -184,7 +279,10 @@ pub struct Change { /// A model the table does not price yet is a change with `was: None`. A rate that /// has not moved is not a change and is not listed, so a refresh that found /// nothing new says so in one line rather than in four hundred. -pub fn changes(existing: Option<&io_harness::pricing::PriceTable>, fresh: &Catalogue) -> Vec { +pub fn changes( + existing: Option<&io_harness::pricing::PriceTable>, + fresh: &Catalogue, +) -> Vec { fresh .rows .iter() @@ -199,6 +297,132 @@ pub fn changes(existing: Option<&io_harness::pricing::PriceTable>, fresh: &Catal .collect() } +/// How a file spells its price table, which decides how it can be written to. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Shape { + /// No `[prices.models]` anywhere. The section is created whole. + Absent, + /// `[prices.models]` with a row per model, which is what io-cli writes. Rows + /// are set key by key, so models nobody named survive. + Table, + /// A `[prices.models.""]` sub-table per model — legal TOML that + /// io-harness reads perfectly well, and that io-cli cannot safely rewrite. + SubTables, +} + +/// How `text` spells its price table. +/// +/// **The question [`Catalogue::edits`] needs answered, asked of the only thing +/// that can answer it**, and it has three answers rather than two. +/// +/// A `[prices]` carrying an `as_of` and no models is a real state — a refresh +/// whose catalogue priced nothing leaves exactly that — so the test is for the +/// child table and not the parent. +/// +/// [`Shape::SubTables`] is the one worth spelling out. `Price` deserializes from a +/// sub-table as happily as from an inline one, so an operator writing this section +/// by hand may well write `[prices.models."gpt-4.1"]` with the rates under it, and +/// io-harness will read it. io-cli cannot update it: setting a key named +/// `"gpt-4.1"` beside a table of the same name is a duplicate-key error, and +/// creating `[prices.models]` above the sub-tables collides with every id in both. +/// **So it is refused rather than attempted** — the operator owns this section, +/// and a writer that cannot express their shape should say so and leave it alone, +/// not produce a refusal from the TOML parser that names neither. +/// +/// Reads through [`crate::edit::sections`], the same header walk `edit::apply` +/// itself does, so the two cannot disagree about what is in the file. +/// Whether `text` carries a `[prices.models]` table io-cli can set keys into. +/// +/// The two-answer form of [`shape`], for [`Catalogue::edits`], which only needs to +/// know whether to create the section or write into it. [`Shape::SubTables`] +/// answers `false` here and must be refused by the caller *before* it gets this +/// far — see [`shape`] for why io-cli cannot write that spelling at all. +pub fn has_models_section(text: &str) -> bool { + matches!(shape(text), Shape::Table) +} + +pub fn shape(text: &str) -> Shape { + let mut found = Shape::Absent; + for path in crate::edit::sections(text) { + if path.len() < 2 || path[0] != "prices" || path[1] != "models" { + continue; + } + if path.len() > 2 { + // Any sub-table settles it: this file is not one io-cli can edit, and + // no later `[prices.models]` header makes it one. + return Shape::SubTables; + } + found = Shape::Table; + } + found +} + +/// Why io-cli will not write prices into `text`, if it will not. +/// +/// One sentence for the operator, or `None` to go ahead. The only refusal today is +/// [`Shape::SubTables`], and it is a refusal rather than an attempt because the +/// operator owns this section: a writer that cannot express their spelling should +/// say so and leave their file alone. +pub fn refusal(text: &str) -> Option { + match shape(text) { + Shape::SubTables => Some( + "no prices were written: this file spells its models as \ + `[prices.models.\"\"]` sub-tables, which io-harness reads and io-cli cannot \ + safely rewrite — the rates already there go on pricing your calls, and a refresh \ + means editing them by hand" + .to_string(), + ), + Shape::Absent | Shape::Table => None, + } +} + +/// How many models `text`'s own `[prices.models]` table prices. +/// +/// **What [`Catalogue::too_short`] should have been comparing against all along.** +/// It was handed `app.io-cli.prices.models`, io-cli's record of its own last write +/// — which is absent for a hand-written `[prices]`, absent for every install that +/// predates this release, and absent on the first fill. So `existing` was zero in +/// exactly the cases the guard exists for, and the guard was off. `PriceTable` has +/// no length, but the file has a row per model and can simply be counted. +pub fn priced_in(text: &str) -> usize { + let Ok(document) = text.parse::() else { + return 0; + }; + document + .get("prices") + .and_then(|prices| prices.get("models")) + .and_then(|models| models.as_table()) + .map_or(0, |models| models.len()) +} + +/// The `[app.io-cli.prices]` edits that record where a table came from. +/// +/// **The same defect as [`Catalogue::edits`]'s, one section over, and it is here +/// rather than at the call site because that is where it was.** The driver used to +/// push two bare `Edit::set`s for `source` and `models`. Neither section exists in +/// any file io-cli has ever written — `settings::render` writes `prices: None` and +/// the field is `skip_serializing_if` — so both fell to the append arm, each +/// emitted its own `[app.io-cli.prices]` header, and the read-back refused the +/// whole write. Every first fill would have ended in "the edit would have produced +/// a file that does not parse", including the one the wizard makes, and `/cost` +/// would have reported tokens forever. +/// +/// It survived the fix that added [`crate::edit::Edit::section`] because the two +/// edits it applies to are added by the driver, and nothing under `tests/` can +/// link the driver. Putting them here is what makes them testable. +pub fn bookkeeping(source: &str, models: usize, has_section: bool) -> Vec { + if has_section { + return vec![ + Edit::set("app.io-cli.prices.source", quoted(source)), + Edit::set("app.io-cli.prices.models", models.to_string()), + ]; + } + vec![Edit::section( + "app.io-cli.prices", + format!("source = {}\nmodels = {models}", quoted(source)), + )] +} + /// What a refresh found, committed before anything is written. /// /// **Everything shown before a byte moves, and the reason is sharper than @@ -238,6 +462,24 @@ pub fn report( return crate::page::commit("prices", &rows, theme, width); } + // **A catalogue that answered with nothing is not a catalogue that agreed with + // you**, and this arm exists because the one below it used to swallow the + // case. `verify::served` returns an empty vector for a network failure, a + // refused key and an unparseable body alike, so with no rows there is nothing + // to compare and `moved` is empty — under which the next arm reported "no rate + // has moved since the last read", a positive claim about a read that did not + // happen. An operator on a flaky connection would have been told their prices + // were confirmed current. + if catalogue.rows.is_empty() { + rows.push(crate::page::Row::caveat(format!( + "the catalogue could not be read, or served no prices — {} model{} came back. \ + Nothing was written and the prices you have are unchanged, which means they are \ + as old as their date says and not as fresh as this attempt.", + catalogue.served, + if catalogue.served == 1 { "" } else { "s" } + ))); + return crate::page::commit("prices", &rows, theme, width); + } if moved.is_empty() { rows.push(crate::page::Row::note( "no rate has moved since the last read, so nothing was written", @@ -318,7 +560,11 @@ fn inline(price: &Price) -> String { /// So this is not decoration: an unquoted `gpt-4.1` is two path segments and /// reaches the wrong place, or no place. `edit::array` sets the precedent of /// spelling TOML through `toml` rather than with `format!`. -fn quoted(text: &str) -> String { +/// +/// Public because the driver writes two values of its own beside the table — the +/// source and the count — and `format!("\"{text}\"")` at a call site is a quoting +/// rule reimplemented, which is how the wrong one eventually gets written. +pub fn quoted(text: &str) -> String { toml::Value::String(text.to_string()).to_string() } diff --git a/src/settings.rs b/src/settings.rs index c1606d5..b1530b4 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -157,8 +157,8 @@ pub struct PriceSettings { /// /// Written by a fetch rather than by hand. It records whether the rates came /// from the provider speaking for itself or from a third party's catalogue, - /// which for three of the four providers io-cli knows is the second — OpenAI, - /// Anthropic and Google publish no prices on any endpoint. A page that drew a + /// which for two of the three vendors io-cli can connect to is the second — + /// OpenAI and Anthropic publish no prices on any endpoint. A page that drew a /// figure without saying which would be attributing a number to a vendor that /// never published one. #[serde(default, skip_serializing_if = "Option::is_none")] diff --git a/src/stats.rs b/src/stats.rs index ec5f0ae..9b8da7a 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -51,12 +51,12 @@ use crate::theme::Theme; /// How many of the most recent runs the latency sections read. /// /// **A bound this module can actually keep, unlike the one `/cost` could not.** -/// `Store::runs` returns run ids and io-cli takes the tail of that list itself, so -/// the sample is genuinely the last `N` runs and the page says so in the heading. -/// That is the difference between a bound and a truncation: a heading reading -/// "slowest calls of the last 200 runs" is a true statement about a subset, where -/// "by model" over a truncated read would have been a false statement about the -/// whole. +/// `Store::runs` returns run ids newest first, and io-cli takes the head of that +/// list itself, so the sample is genuinely the last `N` runs and the page says so +/// in the heading. That is the difference between a bound and a truncation: a +/// heading reading "slowest calls of the last 200 runs" is a true statement about +/// a subset, where "by model" over a truncated read would have been a false +/// statement about the whole. /// /// The same shape and the same reason as `crate::sessions::MAX_RUNS_SCANNED`. pub const RECENT_RUNS: usize = 200; @@ -89,7 +89,10 @@ pub fn committed( rows.push(Row::heading("finished on the first try".to_string())); let first = store.first_try().map_err(|e| e.to_string())?; rows.push(Row::fact("runs finished", first.runs.to_string())); - rows.push(Row::fact("of those, succeeded", first.succeeded.to_string())); + rows.push(Row::fact( + "of those, succeeded", + first.succeeded.to_string(), + )); rows.push(Row::fact( "of those, with no gate failure", first.first_try.to_string(), @@ -126,7 +129,12 @@ pub fn committed( let ids = store.runs().map_err(|e| e.to_string())?; let scanned = ids.len().min(RECENT_RUNS); - let recent = &ids[ids.len() - scanned..]; + // **The head, because `Store::runs` is newest first.** Its own documentation + // says so — `ORDER BY id DESC` — and taking the tail would have read the + // *oldest* two hundred runs under a heading promising the most recent, which + // is a false statement about a subset rather than a true one. That is the + // whole difference this bound rests on. + let recent = &ids[..scanned]; let mut calls = Vec::new(); for id in recent { calls.extend(store.provider_calls(*id).map_err(|e| e.to_string())?); @@ -172,7 +180,10 @@ pub fn committed( "median", format!("{} ms", sorted[sorted.len() / 2]), )); - rows.push(Row::fact("slowest", format!("{} ms", sorted[sorted.len() - 1]))); + rows.push(Row::fact( + "slowest", + format!("{} ms", sorted[sorted.len() - 1]), + )); rows.push(Row::fact( "measured on", format!("{} of {} calls", measured.len(), calls.len()), diff --git a/src/status.rs b/src/status.rs index 26f028b..82df576 100644 --- a/src/status.rs +++ b/src/status.rs @@ -549,6 +549,14 @@ impl Status { /// would leave the line contradicting prompts that are still going to run. pub fn forget_run(&mut self) { self.tokens = None; + // **The money goes with the tokens it was derived from.** `start_run` + // clears this too, and clearing it there alone is not enough: `/clear`, + // `/resume`, `/fork` and a rewind all reach here *without* starting a run, + // so a session cleared at an idle prompt would have gone on drawing the + // previous conversation's cost beside a blank token count — a figure with + // a currency in front of it attributed to a session that has spent + // nothing, which is the invented number this field exists not to print. + self.cost = None; // Both new in 0.11.0, and both run facts. The provider is the one that // reads as a session fact and is not: a resumed conversation may have // been served by another provider entirely, and `Started` sets it again @@ -1214,14 +1222,21 @@ impl Status { // The keys that mean something at this exact moment, and only those. A // footer that listed every binding would be a help screen; `/help` is // the help screen. - counts.push( - if self.working { - "esc stops" - } else { - "/ for commands" - } - .to_string(), - ); + // + // **Held out of `counts` until after the narrowing below, because it is + // not a counter.** It used to be pushed here, which put it last — and the + // narrowing pops from the end, so the first thing a crowded row gave up + // was `esc stops`: the only place the footer tells an operator how to + // interrupt a turn, dropped exactly when the row is full because a turn is + // running. It is appended after the loop instead, and its width is counted + // during the loop so the arithmetic still describes the row that gets + // drawn. + let hint = if self.working { + "esc stops" + } else { + "/ for commands" + } + .to_string(); // What the agent is allowed to do, on the right, because that is the // question a reader asks of a footer when they ask anything of it. @@ -1272,12 +1287,16 @@ impl Status { // leftmost counter is never dropped: a row that had given up every // number would be reporting the mode by erasing the session. let wanted: usize = allowed.iter().map(|s| s.content.chars().count()).sum(); + // The hint is not in `counts` and is not droppable, so its width is added + // here rather than being carried by the join. + let hinted = hint.chars().count() + separator.chars().count(); while !allowed.is_empty() - && counts.len() > 1 - && counts.join(separator).chars().count() + wanted >= room + && !counts.is_empty() + && counts.join(separator).chars().count() + hinted + wanted >= room { counts.pop(); } + counts.push(hint); let counted = row( vec![Span::styled(counts.join(separator), muted)], allowed, diff --git a/src/verify.rs b/src/verify.rs index 4ed36e9..02f4123 100644 --- a/src/verify.rs +++ b/src/verify.rs @@ -69,31 +69,40 @@ pub async fn credential(spec: &ProviderSpec) -> Result<(), String> { } } -/// The catalogue as this provider serves it, whole rows and not just ids. +/// A catalogue read, whole rows and not just ids, and **unfiltered**. /// -/// **The rows carry prices, and until 0.22.0 this function threw them away.** It -/// read the same catalogue, mapped every [`io_harness::ModelInfo`] down to its -/// `id`, and dropped the `price`, `price_tiers` and `price_source` on the same -/// row — while the interface over it reported token counts and called the money -/// question unanswerable. [`crate::prices`] is what keeps them; this is the read -/// they both come from, so there is one filter and one spelling rather than two -/// that can drift. +/// **The rows carry prices, and until 0.22.0 this read threw them away.** It read +/// the same catalogue, mapped every [`io_harness::ModelInfo`] down to its `id`, +/// and dropped the `price`, `price_tiers` and `price_source` on the same row — +/// while the interface over it reported token counts and called the money question +/// unanswerable. [`crate::prices`] is what keeps them. /// /// `source` names the catalogue to read, for the operator on a self-hosted or /// `compatible` endpoint the reference catalogue has never heard of. `None` is /// io-harness's own default. /// +/// **Narrowing happens at the caller and not here, and that is the whole of what +/// makes `source_url` mean anything.** [`named`] narrows the default catalogue — +/// one vendor's view of the entire field — down to the provider in force, and for +/// a `compatible` endpoint it can only answer "none of these", because a reference +/// list cannot say what a server it has never heard of serves. An operator who +/// sets `source_url` has already answered that question: they have pointed io-cli +/// at the catalogue their own endpoint publishes, and every row of it is theirs. +/// Narrowing here would have applied the wrong filter to the right catalogue, +/// which is what it did — the key was inert, over a test asserting the empty +/// result, while `src/settings.rs` and the README both called it the only way a +/// self-hosted operator gets prices at all. +/// /// An error is not fatal and comes back as an empty vector: a catalogue that /// cannot be read is a reason to make the user type a model, not a reason to stop. -pub async fn served(spec: &ProviderSpec, source: Option<&str>) -> Vec { +/// A caller that needs to tell "nothing was served" from "nothing was priced" has +/// [`crate::prices::Catalogue::served`] for it. +pub async fn served(source: Option<&str>) -> Vec { let reference = match source { Some(url) if !url.is_empty() => Reference::at(url), _ => Reference::new(), }; - let Ok(models) = reference.models().await else { - return Vec::new(); - }; - named(spec, models) + reference.models().await.unwrap_or_default() } /// The catalogue filtered to what `spec` serves, spelled the way `spec` names it. @@ -103,7 +112,10 @@ pub async fn served(spec: &ProviderSpec, source: Option<&str>) -> Vec) -> Vec { +pub fn named( + spec: &ProviderSpec, + models: Vec, +) -> Vec { match spec { // The reference catalogue is OpenRouter's own, so for OpenRouter it is not // a reference at all — it is the provider speaking for itself. @@ -120,8 +132,7 @@ pub fn named(spec: &ProviderSpec, models: Vec) -> Vec Vec { - let mut ids: Vec = served(spec, None) - .await + let mut ids: Vec = named(spec, served(None).await) .into_iter() .map(|model| model.id) .collect(); @@ -132,16 +143,19 @@ pub async fn catalogue(spec: &ProviderSpec) -> Vec { /// The priced rows of a catalogue, sorted by model. /// -/// **A model the catalogue served with no price is absent, never entered at -/// zero.** io-harness's `PriceTable::price` returning `None` is what makes +/// **Takes rows [`named`] has already spelled, and does not filter again.** +/// Filtering is not idempotent — `named` strips a provider's prefix, and running +/// it twice on an Anthropic catalogue finds no `anthropic/` left to strip and +/// discards every row. So the caller filters once and hands the result both here +/// and to whatever else needs a count of it. +/// +/// **A model served with no price is absent, never entered at zero.** +/// io-harness's `PriceTable::price` returning `None` is what makes /// `Spend::unpriced_calls` count that call, which is what lets `/cost` say its /// total is a floor rather than reporting a partial sum as a total. A zero here /// would silently claim the model is free. -pub fn priced( - spec: &ProviderSpec, - models: Vec, -) -> Vec<(String, io_harness::pricing::Price)> { - let mut rows: Vec<(String, io_harness::pricing::Price)> = named(spec, models) +pub fn priced(models: Vec) -> Vec<(String, io_harness::pricing::Price)> { + let mut rows: Vec<(String, io_harness::pricing::Price)> = models .into_iter() .filter_map(|model| model.price.map(|price| (model.id, price))) .collect(); diff --git a/tests/cost.rs b/tests/cost.rs new file mode 100644 index 0000000..0429302 --- /dev/null +++ b/tests/cost.rs @@ -0,0 +1,855 @@ +//! What has been spent, and the three ways of not knowing it. +//! +//! **Every number on `/cost` is a row already in `runs.db` or that row multiplied +//! by a rate the operator can point at.** There is no sampling, no extrapolation +//! and no model of what a token "usually" costs — which means the interesting +//! failures in this module are not arithmetic failures at all. They are the three +//! ways a page can report a number it does not have: a call the provider said +//! nothing about summed as zero, a model outside the price table priced at +//! nothing, and a total containing either of those drawn as though it were +//! complete. io-harness's own pricing documentation calls the last one "lying by +//! omission", and it is right. +//! +//! So the arithmetic is asserted once, against a figure this file computes from +//! the rates and the token split rather than against a literal — a wrong constant +//! baked into both sides of an `assert_eq!` proves nothing at all — and the rest +//! of the file is about what the page says when it cannot answer. +//! +//! The store is real throughout. `ProviderCall` rows go in through +//! `Store::record_provider_call` and come back out through the same reads the +//! page makes, because `usage: None` survives the round trip as SQL `NULL` and +//! that column is the whole of the unknown/free distinction. + +use io_harness::pricing::{Price, PriceTable}; +use io_harness::{ProviderCall, Store, Usage}; + +use io_cli::cost::{self, Provenance, Total}; +use io_cli::glyphs::ASCII; +use io_cli::theme::{Theme, DARK}; + +// --------------------------------------------------------------------------- +// Fixtures +// --------------------------------------------------------------------------- + +/// Wide enough that nothing folds, so a `contains` here fails for the reason it +/// names rather than because a row broke across two. +const ROOMY: u16 = 200; + +/// The rates. Five dimensions, all different, and none of them a round number of +/// dollars — a table where every rate was the same could not tell an +/// implementation that priced completion at the input rate from one that did not. +const INPUT: u64 = 3_000_000; +const OUTPUT: u64 = 15_000_000; +const CACHE_READ: u64 = 300_000; +const CACHE_WRITE: u64 = 3_750_000; +const PER_REQUEST: u64 = 10_000; + +/// The model the table prices, and the one it does not. +const PRICED: &str = "claude-sonnet-4.5"; +const UNPRICED: &str = "some-lab/experimental-preview"; + +fn table() -> PriceTable { + PriceTable::new("2026-08-27").with( + PRICED, + Price { + input: INPUT, + output: OUTPUT, + cache_read: CACHE_READ, + cache_write: CACHE_WRITE, + per_server_tool_request: PER_REQUEST, + }, + ) +} + +/// The token split every arithmetic assertion below runs over. +/// +/// **The cache figures are inside the prompt, not beside it.** Ten thousand +/// prompt tokens of which four thousand were served from cache and one thousand +/// written into it leaves five thousand read fresh — and a page that added the +/// three would report fifteen thousand prompt tokens for a turn that had ten. +fn split() -> Usage { + Usage { + prompt_tokens: 10_000, + completion_tokens: 2_000, + total_tokens: 12_000, + cache_read_tokens: 4_000, + cache_write_tokens: 1_000, + reasoning_tokens: 500, + server_tool_requests: 3, + } +} + +fn call(model: Option<&str>, usage: Option) -> ProviderCall { + ProviderCall { + step: 1, + attempt: 0, + provider: "anthropic".into(), + model: model.map(str::to_string), + usage, + latency_ms: 1_200, + ttft_ms: Some(300), + // `..Default::default()` rather than every field, because io-harness adds + // one to this struct when a vendor starts reporting something new and a + // test that spelled all nine would stop compiling on a harness bump for a + // reason that has nothing to do with what it asserts. + ..Default::default() + } +} + +/// A store with one run inside one session, and the calls recorded against it. +struct Seeded { + store: Store, + run: i64, + session: i64, +} + +fn seeded(calls: &[ProviderCall]) -> Seeded { + let store = Store::memory().expect("an in-memory store"); + let run = store.start_run("summarise the module", "/repo").expect("a run"); + let session = store.create_session("/repo").expect("a session"); + store + .record_turn(session, None, run, "summarise the module") + .expect("a turn"); + for call in calls { + store + .record_provider_call(run, call) + .expect("the call is recorded"); + } + Seeded { + store, + run, + session, + } +} + +fn ascii() -> Theme { + DARK.with_glyphs(ASCII) +} + +/// The `/cost` page as a reader sees it: every row, spans concatenated. +fn page(seeded: &Seeded, table: &PriceTable, provenance: &Provenance) -> Vec { + cost::committed( + &seeded.store, + table, + provenance, + Some(seeded.run), + Some(seeded.session), + &ascii(), + ROOMY, + ) + .expect("the page draws") + .iter() + .map(|line| { + line.spans + .iter() + .map(|span| span.content.as_ref()) + .collect::() + }) + .collect() +} + +/// The whole page as one string, for asking whether a word is anywhere on it. +fn text(rows: &[String]) -> String { + rows.join("\n") +} + +/// The value of the first row reading `label: …`, or a failure naming the page. +fn field<'a>(rows: &'a [String], label: &str) -> &'a str { + let prefix = format!("{label}: "); + rows.iter() + .find_map(|row| row.trim_start().strip_prefix(&prefix)) + .unwrap_or_else(|| panic!("no `{label}` row on the page:\n{}", text(rows))) +} + +/// The provenance a configured operator has. +fn provenance() -> Provenance { + Provenance { + as_of: Some("2026-08-27".into()), + source: Some("the reference catalogue at https://openrouter.ai/api/v1/models".into()), + models: Some(417), + } +} + +// --------------------------------------------------------------------------- +// The arithmetic +// --------------------------------------------------------------------------- + +/// **The total is the rates multiplied by the split, and the expected figure is +/// computed here rather than copied out of a run.** +/// +/// A test that asserted `total.micros == 79_950` would go green on the day it was +/// written and would keep going green if somebody changed a rate constant in the +/// fixture — the two sides would move together and the assertion would be that +/// the function is the function. So the expected value is built from the same +/// five rates and the same seven token counts an operator would multiply by hand, +/// and the fixture is chosen so the division is exact: io-harness's rounding rule +/// is io-harness's to test, and re-implementing it here would smuggle it into an +/// assertion about io-cli's summing. +/// +/// The cross-check against `PriceTable::cost_micros` is what stops this file +/// inventing its own pricing model. If the arithmetic here and the harness's +/// disagree, this test says so before anything else does. +/// +/// Sabotage: sum `usage.prompt_tokens` at the input rate rather than +/// `prompt_tokens - cache_read - cache_write`, which is the natural mistake and +/// over-reports every cached turn — which is most of them. Under it this fails by +/// exactly the cache tokens times the input rate and no rendered page changes +/// shape. +#[test] +fn the_total_is_the_rates_times_the_split_and_not_a_figure_typed_here() { + let usage = split(); + let fresh = usage.prompt_tokens - usage.cache_read_tokens - usage.cache_write_tokens; + assert_eq!(fresh, 5_000, "the fixture's fresh prompt tokens"); + + // Per-million everywhere except the server tool requests, which vendors bill + // per request and which a token-only sum silently loses. + let mtok = fresh as u128 * INPUT as u128 + + usage.completion_tokens as u128 * OUTPUT as u128 + + usage.cache_read_tokens as u128 * CACHE_READ as u128 + + usage.cache_write_tokens as u128 * CACHE_WRITE as u128; + assert_eq!( + mtok % 1_000_000, + 0, + "the fixture is chosen so that no rounding rule is under test here", + ); + let expected = + (mtok / 1_000_000 + usage.server_tool_requests as u128 * PER_REQUEST as u128) as u64; + assert!(expected > 0, "a fixture that costs nothing proves nothing"); + + // io-harness agrees about what one call costs, so what follows is a claim + // about io-cli's summing rather than about pricing. + assert_eq!( + table().cost_micros(PRICED, &usage), + Some(expected), + "this file and io-harness disagree about what one call costs", + ); + + let total = Total::of(&[call(Some(PRICED), Some(usage))], &table()); + assert_eq!(total.calls, 1); + assert_eq!(total.unknown, 0); + assert_eq!(total.unpriced, 0); + assert_eq!(total.micros, expected); + assert!( + !total.is_floor(), + "a total with nothing missing from it was reported as a floor", + ); + + // **Reasoning is inside completion, never beside it.** A vendor that reports + // reasoning separately still bills it as output, so it is carried for the + // page to break down and contributes nothing of its own to the money. + let mut without_reasoning = usage; + without_reasoning.reasoning_tokens = 0; + assert_eq!( + Total::of(&[call(Some(PRICED), Some(without_reasoning))], &table()).micros, + expected, + "reasoning tokens were billed a second time on top of completion", + ); + + // And the sum over calls is a sum. Three of the same call cost three times as + // much, and the token counters add rather than replace. + let three = Total::of( + &[ + call(Some(PRICED), Some(usage)), + call(Some(PRICED), Some(usage)), + call(Some(PRICED), Some(usage)), + ], + &table(), + ); + assert_eq!(three.micros, expected * 3); + assert_eq!(three.usage.prompt_tokens, usage.prompt_tokens * 3); + assert_eq!(three.usage.cache_read_tokens, usage.cache_read_tokens * 3); + assert_eq!(three.usage.server_tool_requests, usage.server_tool_requests * 3); +} + +/// **A call the provider said nothing about is unknown, and unknown is not +/// free.** +/// +/// io-harness stores `total_tokens` as SQL `NULL` for a call whose provider +/// reported no usage and reads it back as `None` for exactly that reason. Summing +/// it as a zeroed `Usage` would report a turn that cost something as a turn that +/// cost nothing — and it would do it silently, because a zero adds correctly to +/// every other figure on the page. +/// +/// Three things therefore have to hold at once: the call is counted (it happened, +/// and it was billed by somebody), it contributes no tokens (there are none to +/// contribute), and the total is marked a floor (there is a cost nobody can +/// state). A page that got the first two right and the last one wrong would draw +/// a complete-looking total that is missing a turn. +/// +/// Sabotage: `let usage = call.usage.unwrap_or_default();` — one method call, +/// reads as a tidy-up of an `Option`, and turns every unreported call into a free +/// one. Under it `unknown` stays zero, `is_floor` goes false, and the caveat +/// leaves the page. +#[test] +fn a_call_the_provider_said_nothing_about_is_unknown_and_never_free() { + let calls = [ + call(Some(PRICED), Some(split())), + call(Some(PRICED), None), + call(None, None), + ]; + let total = Total::of(&calls, &table()); + + assert_eq!(total.calls, 3, "a call that reported nothing still happened"); + assert_eq!(total.unknown, 2); + assert_eq!( + total.unpriced, 0, + "an unreported call is unknown, which is a different gap from unpriced — \ + counting it as both would report one missing call twice", + ); + assert_eq!( + total.usage, + split(), + "the unreported calls contributed tokens they never reported", + ); + assert!( + total.is_floor(), + "two calls are missing from this total and it does not say so", + ); + + // And the page says it in words, in the vocabulary that distinguishes the two + // gaps. `unknown rather than free` is the sentence; a page that said "0 + // tokens" would be stating the thing that is not true. + let fixture = seeded(&calls); + let rows = page(&fixture, &table(), &provenance()); + let drawn = text(&rows); + assert!( + drawn.contains("reported no usage at all"), + "the page hides the calls it could not read:\n{drawn}", + ); + assert!( + drawn.contains("unknown rather than free"), + "the page does not distinguish an unreported call from a free one:\n{drawn}", + ); +} + +/// **A model outside the table is unpriced, and a total containing one is a floor +/// rather than a total.** +/// +/// The gap the whole module is shaped around. io-cli compiles no prices in, so an +/// operator's table covers whatever their catalogue happened to price on the day +/// they read it — a model added upstream last week is a real cost with no rate, +/// and it is the ordinary case rather than the exotic one. +/// +/// The tokens are still exact. That is the point of counting the two gaps +/// separately: an unpriced call contributes every token it reported and no money, +/// so the token figures on the page are complete and the money figure is a floor, +/// and the page has to be able to say both at once. +/// +/// Sabotage: price a missing model at `Price::ZERO` rather than counting it — +/// under which `unpriced` stays zero, `is_floor` goes false, the caveat leaves the +/// page, and the operator reads a total that is confidently short. +#[test] +fn a_model_the_table_does_not_price_is_a_floor_and_says_so() { + let calls = [ + call(Some(PRICED), Some(split())), + call(Some(UNPRICED), Some(split())), + // No model at all, which the provider is allowed not to name. It cannot + // be priced either, and for the same reason it cannot be attributed. + call(None, Some(split())), + ]; + let total = Total::of(&calls, &table()); + + assert_eq!(total.calls, 3); + assert_eq!(total.unknown, 0, "all three reported their usage"); + assert_eq!(total.unpriced, 2, "two of the three cannot be priced"); + assert!(total.is_floor(), "a total missing two calls' money is a floor"); + assert_eq!( + total.usage.prompt_tokens, + split().prompt_tokens * 3, + "an unpriced call still reported its tokens, and they are still exact", + ); + assert_eq!( + total.micros, + table().cost_micros(PRICED, &split()).expect("the priced one"), + "the money is exactly the one call the table can price", + ); + + let fixture = seeded(&calls); + let rows = page(&fixture, &table(), &provenance()); + let drawn = text(&rows); + assert!( + drawn.contains("no rate in the price table"), + "the page does not say why the total is short:\n{drawn}", + ); + assert!( + drawn.contains("floor and not a total"), + "the page draws an incomplete total as though it were complete:\n{drawn}", + ); + // **On the grouped rows too, and on the row rather than under it.** A reader + // scanning `by model` for the largest figure has to be able to see which of + // them is incomplete without reading past the list. + // + // "not priced" and not "unpriced", deliberately: io-harness's + // `Spend::unpriced_calls` counts a call with no usage as unpriced, while this + // module's `Total` splits those out as *unknown*. The same call falls in + // different buckets on the two halves of this page, and one word for both + // would make the two sections look like they disagreed about a number. + assert!( + drawn.contains("not priced"), + "the `by model` rows hide the calls they could not price:\n{drawn}", + ); + assert!( + rows.iter() + .any(|row| row.contains(UNPRICED) && row.contains("not priced")), + "the unpriced model's own row does not say it could not be priced:\n{drawn}", + ); +} + +/// **The cache figures are a breakdown of the prompt, and the three parts sum to +/// it rather than past it.** +/// +/// `Usage::cache_read_tokens` and `cache_write_tokens` are already inside +/// `prompt_tokens`, so a page that listed them as separate figures would +/// over-report every cached turn — and on a long agentic run almost every turn is +/// cached. The page draws the prompt total and the three parts beneath it, named +/// as parts (`of which cache read`) rather than drawn further right, because this +/// page has no columns to indent into and a reader in `--plain` has only the +/// words. +/// +/// Asserted on the underlying counts as well as on the rendered strings: the +/// rendered figures are rounded to one decimal by `format_tokens`, so agreement +/// between them is necessary and not sufficient. +/// +/// Sabotage: add the two cache counters to the prompt figure — under which +/// `prompt` reads 15.0k for a turn that had 10.0k of prompt, the three parts sum +/// past it, and every other test in this file still passes. +#[test] +fn the_cache_figures_are_inside_the_prompt_and_never_added_to_it() { + let usage = split(); + let fresh = usage.prompt_tokens - usage.cache_read_tokens - usage.cache_write_tokens; + assert_eq!( + usage.cache_read_tokens + usage.cache_write_tokens + fresh, + usage.prompt_tokens, + "the fixture's own parts do not sum to its prompt", + ); + + let fixture = seeded(&[call(Some(PRICED), Some(usage))]); + let rows = page(&fixture, &table(), &provenance()); + + let spelled = io_cli::status::format_tokens; + assert_eq!( + field(&rows, "prompt"), + spelled(usage.prompt_tokens), + "the prompt figure is not the prompt the provider reported:\n{}", + text(&rows), + ); + assert_eq!( + field(&rows, "of which cache read"), + spelled(usage.cache_read_tokens), + ); + assert_eq!( + field(&rows, "of which cache written"), + spelled(usage.cache_write_tokens), + ); + assert_eq!( + field(&rows, "of which fresh"), + spelled(fresh), + "the fresh figure is not the prompt less what came from cache", + ); + assert_eq!(field(&rows, "completion"), spelled(usage.completion_tokens)); + assert_eq!( + field(&rows, "of which reasoning"), + spelled(usage.reasoning_tokens), + "reasoning is a breakdown of completion and is drawn as one", + ); + + // A turn with no cache at all draws no cache rows, rather than three zeroes + // an operator has to read to learn nothing. + let plain = Usage { + prompt_tokens: 900, + completion_tokens: 100, + total_tokens: 1_000, + ..Default::default() + }; + let fixture = seeded(&[call(Some(PRICED), Some(plain))]); + let drawn = text(&page(&fixture, &table(), &provenance())); + assert!( + !drawn.contains("of which cache"), + "a turn that cached nothing drew a cache breakdown of zeroes:\n{drawn}", + ); + assert!( + !drawn.contains("of which reasoning"), + "a turn that reasoned about nothing drew a reasoning row:\n{drawn}", + ); +} + +// --------------------------------------------------------------------------- +// `money` +// --------------------------------------------------------------------------- + +/// **Money is integer arithmetic at every magnitude, and the precision changes +/// with the magnitude because one precision cannot show both ends.** +/// +/// A turn costs a fraction of a cent and a month costs dollars. Two decimals +/// round the turn to `$0.00`, which reads as free; four decimals pad the month +/// with `$412.9100`, which reads as noise. So it is four below a unit and two +/// above it, and the switch is at the unit rather than at some threshold nobody +/// could name. +/// +/// Every figure here is a division of `u64` by `u64`. A bill is not a +/// floating-point quantity: `cost_micros` is exact, and rendering it through an +/// `f64` would reintroduce a drift the harness went to the trouble of summing in +/// `u128` to avoid. +/// +/// Sabotage: render through `micros as f64 / 1_000_000.0` — under which most of +/// these still pass and the multi-dollar cases drift in the last place. +#[test] +fn money_is_integer_arithmetic_at_every_magnitude() { + // Zero is bare. A total of nothing is not `$0.0000`, which reads as a small + // number somebody rounded. + assert_eq!(cost::money(0), "$0"); + + // **Below the smallest figure four decimals can show, the answer says so + // rather than rounding to a zero.** `$0.0000` reads as free, and a cheap model + // answering a short prompt lands under a hundredth of a cent every time — so + // that form would have printed "free" for most single turns on a small model. + // `$0` above is reserved for a cost that really is zero, which a free model + // has and this does not. + assert_eq!(cost::money(1), "<$0.0001"); + assert_eq!(cost::money(99), "<$0.0001"); + assert_eq!(cost::money(100), "$0.0001"); + assert_eq!(cost::money(12_345), "$0.0123"); + + // Sub-dollar, up to the last micro-unit before the switch. + assert_eq!(cost::money(999_999), "$0.9999"); + + // At and above a unit, two decimals. + assert_eq!(cost::money(1_000_000), "$1.00"); + assert_eq!(cost::money(1_234_567), "$1.23"); + assert_eq!(cost::money(12_345_678_901), "$12345.67"); + + // **Truncated and never rounded up**, which is the honest direction for a + // floor: a figure that rounded to `$2.00` from `$1.999999` would report more + // than the store can account for. + assert_eq!(cost::money(1_999_999), "$1.99"); + assert_eq!(cost::money(999_99), "$0.0999"); + + // The rendering never loses a digit or gains one: every sub-dollar figure it + // can express has four decimals, at every magnitude in the sweep. The + // below-a-hundred-micro-units form is deliberately a different width — it is a + // different *claim*, "smaller than this" rather than "this", and a bound that + // looked identical to a figure would be read as one. + for micros in [100u64, 500, 99_999, 999_999] { + let drawn = cost::money(micros); + assert_eq!( + drawn.len(), + "$0.0000".len(), + "a sub-dollar figure changed width: {drawn}", + ); + } + for micros in [1u64, 7, 99] { + assert_eq!(cost::money(micros), "<$0.0001"); + } + for micros in [1_000_000u64, 9_999_999, 123_456_789] { + let drawn = cost::money(micros); + assert_eq!( + drawn.split('.').next_back().map(str::len), + Some(2), + "a figure at or above a unit did not carry two decimals: {drawn}", + ); + } +} + +// --------------------------------------------------------------------------- +// The page as a whole +// --------------------------------------------------------------------------- + +/// **An empty price table draws every token figure and no money at all.** +/// +/// This is the state every operator is in before their first catalogue read, and +/// the state an operator on a self-hosted endpoint stays in permanently. The +/// module's own argument for having one code path rather than two is written on +/// `cost::table`: "the page is the same page with the money left off, rather than +/// a second page for the unconfigured case that would go stale unread." +/// +/// *Left off* is the claim. A page that draws `cost: $0` under a caveat saying the +/// figure is a floor has not left the money off — it has stated a number nobody +/// measured, in the one situation where no number is available, which is the +/// invented figure the whole module exists to refuse. `Status::note_cost_from` +/// already holds the status line to exactly this rule and says so in as many +/// words: "Nothing priced is not zero priced. A run whose models are all outside +/// the table has a real cost that this program cannot state, and stating it as +/// `$0` would be the invented number the whole of `/cost` is built to avoid." The +/// two surfaces have to agree, and the one-row form is the one that currently +/// does. +/// +/// Sabotage: draw the `cost` row unconditionally, and draw `money(0)` on every +/// grouped row whose calls could none of them be priced. +#[test] +fn an_empty_price_table_draws_tokens_and_no_money() { + let empty = PriceTable::new(""); + let fixture = seeded(&[ + call(Some(PRICED), Some(split())), + call(Some(UNPRICED), Some(split())), + ]); + // No `[prices]` section at all, which is what `Provenance::of` reports for a + // file that has never had one. + let rows = page(&fixture, &empty, &Provenance::default()); + let drawn = text(&rows); + + // The page says so, in the sentence that also says what to do about it. + assert!( + drawn.contains("no prices are configured"), + "the page does not say the table is empty:\n{drawn}", + ); + assert!( + drawn.contains("tokens and no money"), + "the page does not say what it can still answer:\n{drawn}", + ); + + // Every token figure is still exact, because nothing about counting tokens + // needs a rate. + let spelled = io_cli::status::format_tokens; + assert_eq!( + field(&rows, "prompt"), + spelled(split().prompt_tokens * 2), + "the token figures went missing with the prices:\n{drawn}", + ); + assert_eq!(field(&rows, "completion"), spelled(split().completion_tokens * 2)); + + // And no currency anywhere. Not a zero, not a floor of zero, not a dollar + // sign on a grouped row: there is no rate on this install and every figure + // that would need one is absent. + assert!( + !drawn.contains('$'), + "a page with no prices drew a money figure, which is the invented number \ + `Status::note_cost_from` refuses on the status line:\n{drawn}", + ); +} + +/// **Before anything has run, the page says so rather than drawing a row of +/// zeroes.** +/// +/// The same distinction the status line holds every one of its counters to: a +/// session that has run nothing has no numbers, and a row of zeroes is a claim +/// that it ran and cost nothing. `run` and `session` are both `None` before the +/// first turn, and they are different absences — a session can have turns behind +/// it with no run in flight. +/// +/// Sabotage: `unwrap_or_default()` either id and read run zero, which exists in +/// no store and reads back as an empty call list — under which the page draws +/// `no provider calls` for a session that has never had one, losing the +/// difference between "nothing yet" and "nothing this time". +#[test] +fn a_session_that_has_run_nothing_says_so_rather_than_drawing_zeroes() { + let store = Store::memory().expect("an in-memory store"); + let lines = cost::committed( + &store, + &table(), + &provenance(), + None, + None, + &ascii(), + ROOMY, + ) + .expect("the page draws with nothing in the store"); + let drawn: String = lines + .iter() + .map(|line| { + line.spans + .iter() + .map(|span| span.content.as_ref()) + .collect::() + }) + .collect::>() + .join("\n"); + + assert!( + drawn.contains("nothing has run in this session yet"), + "the run section drew something for a session with no run:\n{drawn}", + ); + assert!( + drawn.contains("this session has no turns yet"), + "the session section drew something for a session with no turns:\n{drawn}", + ); + assert!( + drawn.contains("nothing recorded"), + "the grouped sections drew something for an empty store:\n{drawn}", + ); + assert!( + !drawn.contains("calls: 0"), + "an empty store was reported as a run that made no calls:\n{drawn}", + ); +} + +/// **The page's sections are the questions an operator asks, and each figure is +/// under the heading that names its scope.** +/// +/// Seven lists on one page — this run, this session, by model, by day — and a +/// reader has to be able to tell which figure belongs to which question. A page +/// that reported one run's cost under the heading of a whole conversation would be +/// the wrong number under the right word, which is why the session total is walked +/// turn by turn rather than taken from `spend_by_run`. +/// +/// Sabotage: report `Total::of` over the run's calls under `this session` as well +/// — which on a one-turn fixture like this one produces the same number, so the +/// second turn below is what makes the test able to tell. +#[test] +fn the_session_total_is_every_turn_and_not_the_run_in_flight() { + let store = Store::memory().expect("an in-memory store"); + let session = store.create_session("/repo").expect("a session"); + let mut runs = Vec::new(); + for _ in 0..2 { + let run = store.start_run("summarise", "/repo").expect("a run"); + store + .record_turn(session, None, run, "summarise") + .expect("a turn"); + store + .record_provider_call(run, &call(Some(PRICED), Some(split()))) + .expect("recorded"); + runs.push(run); + } + + let table = table(); + let one = table.cost_micros(PRICED, &split()).expect("one call's cost"); + let lines = cost::committed( + &store, + &table, + &provenance(), + Some(runs[1]), + Some(session), + &ascii(), + ROOMY, + ) + .expect("the page draws"); + let rows: Vec = lines + .iter() + .map(|line| { + line.spans + .iter() + .map(|span| span.content.as_ref()) + .collect::() + }) + .collect(); + + // Both sections exist, and they carry different numbers: one run against two + // turns. The `cost` rows come in section order, so the first belongs to the + // run in flight and the second to the conversation. + let costs: Vec<&str> = rows + .iter() + .filter_map(|row| row.trim_start().strip_prefix("cost: ")) + .collect(); + assert_eq!( + costs.first().copied(), + Some(cost::money(one).as_str()), + "the run section is not this run:\n{}", + text(&rows), + ); + assert_eq!( + costs.get(1).copied(), + Some(cost::money(one * 2).as_str()), + "the session section is not every turn of the conversation:\n{}", + text(&rows), + ); + assert!( + text(&rows).contains("across 2 turns"), + "the session section does not say how many turns it walked:\n{}", + text(&rows), + ); +} + +/// **Every `$` on the page is backed by a sentence naming where the rate came +/// from and when it was read.** +/// +/// io-cli reads prices from a catalogue the operator's own provider serves, and +/// for three of the four providers it knows that catalogue is a third party's — +/// OpenAI, Anthropic and Google publish no prices on any endpoint. A figure drawn +/// without that sentence attributes a number to a vendor who never published one. +/// +/// The three cases are three different states of an install, and the middle one +/// is the one a hand-written file produces: prices with no record of where they +/// came from. It gets its date and an honest admission rather than an invented +/// source. +/// +/// Sabotage: fall through to "prices: none configured" whenever `source` is +/// `None`, and a hand-written table reads as no table at all. +#[test] +fn the_footer_names_the_catalogue_and_the_date_or_admits_it_cannot() { + let fixture = seeded(&[call(Some(PRICED), Some(split()))]); + + let full = text(&page(&fixture, &table(), &provenance())); + assert!( + full.contains("417 models") && full.contains("2026-08-27"), + "the footer does not say how many models the rates cover, or when:\n{full}", + ); + assert!( + full.contains("openrouter.ai"), + "the footer does not name the catalogue the rates came from:\n{full}", + ); + + let dated_only = Provenance { + as_of: Some("2026-08-27".into()), + source: None, + models: Some(3), + }; + let hand_written = text(&page(&fixture, &table(), &dated_only)); + assert!( + hand_written.contains("2026-08-27"), + "a hand-written table lost its date:\n{hand_written}", + ); + assert!( + hand_written.contains("did not record"), + "a table with no recorded source got one invented for it:\n{hand_written}", + ); + + let none = text(&page(&fixture, &PriceTable::new(""), &Provenance::default())); + assert!( + none.contains("prices: none configured"), + "an install with no prices does not say so at the foot of the page:\n{none}", + ); +} + +/// **`cost::table` gives one code path rather than two, and `Provenance::of` +/// reads the date from the section that is actually pricing the calls.** +/// +/// The date lives under `[prices]`, which io-harness owns and validates; the +/// source and the model count live under `[app.io-cli.prices]`, which io-harness +/// deliberately does not. That split is not filing: `[prices]` is +/// `deny_unknown_fields` and carries exactly `as_of` and `models`, so a key of +/// io-cli's own put beside them would not be ignored — it would make the +/// operator's whole configuration file fail to parse, taking the policy, the +/// providers and the run budgets down with it. +/// +/// Sabotage: read `as_of` off io-cli's own section, and a table an operator +/// edited by hand reports the date of the last catalogue read instead of the date +/// on the rates in force. +#[test] +fn the_table_in_force_and_its_date_come_from_the_section_that_prices_the_calls() { + let configured = io_harness::Config::from_toml( + "[prices]\nas_of = \"2026-08-27\"\n\n[prices.models]\n\ + \"claude-sonnet-4.5\" = { input = 3000000, output = 15000000 }\n", + ) + .expect("io-harness accepts the fixture"); + + let table = cost::table(&configured); + assert_eq!(table.as_of(), "2026-08-27"); + assert!(table.price(PRICED).is_some()); + + let provenance = Provenance::of(&configured, None); + assert_eq!(provenance.as_of.as_deref(), Some("2026-08-27")); + assert_eq!( + provenance.source, None, + "a file with no io-cli section got a source invented for it", + ); + // **`None`, and asserting `0` here was asserting the defect.** A `[prices]` + // an operator wrote by hand has no `[app.io-cli.prices]` beside it, so nothing + // recorded how many models it prices — which is not the same as its pricing + // none. The field was a `usize` defaulting to zero, and the footer read + // "0 models" over a table pricing four hundred. + assert_eq!( + provenance.models, None, + "an unrecorded count was reported as a measured zero", + ); + + // No `[prices]` at all: an empty table dated to nothing, under which every + // call is unpriced and every token figure is still exact. + let bare = io_harness::Config::from_toml("").expect("an empty configuration"); + let empty = cost::table(&bare); + assert_eq!(empty.as_of(), ""); + assert_eq!(empty.price(PRICED), None); + assert_eq!( + Provenance::of(&bare, None).as_of, + None, + "an install with no prices reported a date it does not have", + ); +} diff --git a/tests/edit.rs b/tests/edit.rs index 411f197..dd7fe26 100644 --- a/tests/edit.rs +++ b/tests/edit.rs @@ -623,3 +623,86 @@ model = \"b\""; "the move did not reorder the chain:\n{up}" ); } + +/// **A whole new section, once — the shape `set` cannot express and the one the +/// price fill could not do without.** +/// +/// Every edit in a batch is resolved against the document as it was *before* the +/// batch: [`edit::apply`] walks the headers once and answers every `set` from that +/// walk. So N `set`s addressing keys of a section that does not exist yet each +/// fall through to the append arm, and each emits its own copy of the header. The +/// file gains N definitions of one table and the read-back refuses the lot — which +/// is what happened to the first price fill, at four hundred models, before this +/// existed. Nothing about the failure named the cause: the operator was told "the +/// edit would have produced a file that does not parse". +/// +/// `Kind::Section` is the answer, and the constraint on it is as important as the +/// capability. It **refuses** a section that is already there rather than replacing +/// it, because a caller holding an existing section wants `set` per key — that +/// leaves every row it did not name alone, which for `[prices.models]` is every +/// model the catalogue stopped serving and every rate the operator corrected by +/// hand. A replace here would delete both from a call site whose author believed +/// they were adding rows. +/// +/// Sabotage: drop the existence check and splice unconditionally, under which the +/// second half of this test passes by writing a file with two `[app.io-cli]` +/// headers in it. Or emit the header per key rather than once, which is the shape +/// that made the failure this exists to remove. +#[test] +fn f1_a_whole_section_is_written_once_and_refused_when_it_is_already_there() { + let body = "\"gpt-4.1\" = { input = 2000000, output = 8000000 }\n\ + \"gpt-4o\" = { input = 2500000, output = 10000000 }\n\ + \"o3\" = { input = 2000000, output = 8000000 }"; + let after = edit::apply( + OPERATORS_FILE, + &[ + Edit::set("prices.as_of", "\"2026-08-27\""), + Edit::section("prices.models", body), + ], + ) + .expect("a section the file does not have is written whole"); + + // One header, whatever the row count. The count is asserted rather than left + // to the parse, because a duplicate is only usually a parse error. + assert_eq!( + after.matches("[prices.models]").count(), + 1, + "the section was written more than once:\n{after}" + ); + + // The operator's own file is still their file, byte for byte, with the new + // sections after it — the same preservation property every other edit here + // keeps. + assert!( + after.starts_with(OPERATORS_FILE), + "writing a new section rewrote the document above it:\n{after}" + ); + + // And it says what it was asked to say. Three dotted ids, each a key rather + // than a path, read back through a parser that has no opinion about io-cli. + let parsed: toml::Value = toml::from_str(&after).expect("the result parses"); + assert_eq!(parsed["prices"]["as_of"].as_str(), Some("2026-08-27")); + for model in ["gpt-4.1", "gpt-4o", "o3"] { + assert!( + parsed["prices"]["models"][model]["input"].as_integer().is_some(), + "`{model}` did not land as a key of the section:\n{after}" + ); + } + + // The refusal. `[app.io-cli]` is in the fixture, so writing it whole would + // discard the theme — and the message says which section and why rather than + // reporting a parse failure the caller cannot act on. + let error = edit::apply( + OPERATORS_FILE, + &[Edit::section("app.io-cli", "theme = \"light\"")], + ) + .expect_err("a section that already exists must not be written whole"); + assert!( + error.contains("app.io-cli") && error.contains("already in this file"), + "the refusal does not name the section it refused: {error}" + ); + assert!( + error.contains("key by key"), + "the refusal does not say what to do instead: {error}" + ); +} diff --git a/tests/page.rs b/tests/page.rs new file mode 100644 index 0000000..75d186c --- /dev/null +++ b/tests/page.rs @@ -0,0 +1,519 @@ +//! The shape of a committed page, and the folding four surfaces now share. +//! +//! **This module is a de-duplication, which is the most dangerous kind of change +//! to test.** `status::committed` and `context::committed` each carried their own +//! copy of the folding until 0.22.0 — twenty lines under two names, `folded` and +//! `wrapped`, differing only in that one took its indents as arguments and the +//! other hard-coded the same two numbers. Both had shipped and both were correct. +//! A shared replacement that is merely *plausible* passes every test those two +//! surfaces already have, because none of them asserts a fold at a width where +//! the two implementations could disagree. +//! +//! So the last test here is not a property test at all. It carries a verbatim +//! copy of each of the two functions as 0.21.0 shipped them and runs the new one +//! against both, over inputs chosen to reach the arm that splits a word — which +//! is the arm a rewrite gets wrong and the arm no existing test reaches. The +//! copies are dead code that exists to disagree. + +use io_cli::glyphs::ASCII; +use io_cli::page::{self, Row}; +use io_cli::theme::{Theme, Tone, DARK}; +use ratatui::text::Line; + +/// The theme the sweeps draw in: the ordinary palette, ASCII glyph set, so a +/// failure message is readable in a terminal that cannot draw the Unicode rule. +fn ascii() -> Theme { + DARK.with_glyphs(ASCII) +} + +/// One rendered line as a reader sees it. +fn row(line: &Line<'_>) -> String { + line.spans.iter().map(|s| s.content.as_ref()).collect() +} + +fn drawn(lines: &[Line<'_>]) -> Vec { + lines.iter().map(row).collect() +} + +/// The texts every sweep runs over. +/// +/// Chosen for the arms they reach rather than for realism, though every one of +/// them is a string this product actually draws: a workspace path with no spaces +/// in it, a policy layer's act list, a namespaced model id, a whole sentence, and +/// a single word longer than eighty columns — which is the one that separates +/// folding from fitting. +fn texts() -> Vec { + vec![ + String::new(), + "workspace".to_string(), + "policy: ops-baseline allow read src/* allow write out/* deny read .env".to_string(), + "model: anthropic/claude-sonnet-4.5".to_string(), + "/Users/somebody/Documents/work/monorepo/services/billing/src/domain/invoice.rs" + .to_string(), + "a".repeat(200), + format!("prefix {} suffix", "x".repeat(120)), + "the total above is a floor and not a total, because four calls used a model \ + with no rate in the price table" + .to_string(), + // Multi-byte, because `folded` slices a word by byte index after counting + // its characters, and the two are the same number only in ASCII. + "café ☕ ünïcödé ✱ ".repeat(12), + ] +} + +// --------------------------------------------------------------------------- +// `folded` +// --------------------------------------------------------------------------- + +/// **No row is ever wider than the width it was given, at any width.** +/// +/// The claim the module is named for. A committed page lands in the terminal's +/// own scrollback beside every earlier turn, so a row that runs past the edge is +/// not truncated — it is wrapped by the terminal at a column nothing here chose, +/// which puts the break in the middle of a path and the continuation hard against +/// the left margin where it reads as a new fact. +/// +/// The companion claim is that nothing is lost. Folding is not fitting: the whole +/// argument for it is written on the function, and it is that a committed surface +/// owns as many rows as it needs, so there is no reason left to lose a character — +/// and the characters most likely to be lost are the tail of a path, the tail of +/// an act list and the tail of a model id, which is to say the answer. Compared on +/// the non-whitespace characters, because folding is allowed to move whitespace +/// around and is not allowed to drop anything else. +/// +/// Sabotage: replace the split arm with a truncation, or use `picker::fit` here — +/// under which every width still holds and the character count fails on the two +/// long-word fixtures. +#[test] +fn folding_never_exceeds_the_width_and_never_loses_a_character() { + for text in texts() { + // From one column, which is narrower than either indent and is the case + // the `.max(1)` in the loop exists for. + for width in 1..=120usize { + for (first, rest) in [(0usize, 0usize), (2, 4), (0, 2), (4, 2), (8, 8)] { + let rows = page::folded(&text, width, first, rest); + assert!(!rows.is_empty(), "folding produced no rows at all"); + + // The general bound. A row is its indent plus its content, and + // content is at least one character however narrow the terminal + // is — the `.max(1)` that stops a width under the indent from + // looping forever. So `indent + 1` is the floor of what a row can + // be, and below it the terminal is narrower than a row can be + // drawn at all. + let bound = width.max(first.max(rest) + 1); + for line in &rows { + assert!( + line.chars().count() <= bound, + "a row of {} characters was folded to {width}: {line:?}", + line.chars().count(), + ); + } + // At every width this product supports, the indent fits and the + // bound is the width itself. + if width > first.max(rest) { + for line in &rows { + assert!( + line.chars().count() <= width, + "a row overflowed {width} columns: {line:?}", + ); + } + } + + let before: String = text.chars().filter(|c| !c.is_whitespace()).collect(); + let after: String = rows + .concat() + .chars() + .filter(|c| !c.is_whitespace()) + .collect(); + assert_eq!( + before, after, + "folding lost or invented a character at {width} columns, indents {first}/{rest}", + ); + } + } + } +} + +/// **A word longer than the room it has is split, and a word that merely does not +/// fit *this* row is not.** +/// +/// Two rules that look like one. A word is moved to a fresh row first and only +/// split when a row of its own cannot hold it — so an ordinary sentence never has +/// a word broken across rows, and a two-hundred-character path is broken rather +/// than allowed to run past the edge. A folder that split eagerly would hyphenate +/// the middle of every long word in every sentence; one that never split would put +/// the path past the margin. +/// +/// Sabotage: delete the `if used > 0 { … continue; }` arm, which is the retry on a +/// fresh row — under which the sentence fixture starts splitting words at the end +/// of every row and this test fails while the width property above still holds. +#[test] +fn a_word_longer_than_its_room_is_split_and_one_that_merely_does_not_fit_is_not() { + // Eighty columns, two-space indent: seventy-eight characters of room. A word + // of a hundred and twenty cannot be held by any row and has to be cut. + let long = "x".repeat(120); + let rows = page::folded(&format!("prefix {long} suffix"), 80, 2, 4); + assert!( + rows.len() >= 3, + "a hundred-and-twenty-character word fitted somewhere it could not: {rows:?}", + ); + // Cut at exactly the room a continuation row has — eighty columns less the + // four-space hanging indent — rather than at the room left on the row it + // could not fit, which is what "retried whole on the fresh row" means. + assert!( + rows.iter().any(|line| line.trim() == "x".repeat(76)), + "the word was cut somewhere other than the full width of a fresh row: {rows:?}", + ); + + // The same width, and a sentence of ordinary words. Every row's content is + // whole words, so splitting `text` on whitespace and re-splitting each row + // yields the same sequence of words. + let sentence = "the total above is a floor and not a total, because four calls used a \ + model with no rate in the price table"; + let rows = page::folded(sentence, 80, 2, 4); + assert!(rows.len() > 1, "the fixture has to fold to say anything"); + let words: Vec<&str> = sentence.split_whitespace().collect(); + let folded_words: Vec<&str> = rows.iter().flat_map(|line| line.split_whitespace()).collect(); + assert_eq!( + words, folded_words, + "a word was broken across two rows when a fresh row would have held it: {rows:?}", + ); +} + +/// **The hanging indent starts on the second row, which is what says a row is a +/// continuation without aligning anything into a column.** +/// +/// The page is deliberately not a table — the argument is written out at +/// `status::committed` and it is about eighty columns rather than about taste: a +/// column width is decided by the widest cell, and the widest cell on these pages +/// is a model id. So a continuation cannot be marked by aligning under a column +/// that does not exist, and it is marked by being indented further than the row +/// it continues. +/// +/// Sabotage: set `indent = rest` before the first row rather than after it, and +/// every fact on every page gains two spaces while this is the only test that +/// notices — the width property still holds and the characters are all still +/// there. +#[test] +fn the_hanging_indent_marks_a_continuation_from_the_second_row() { + let rows = page::folded( + "policy: ops-baseline allow read src/* allow write out/* deny read .env deny net \ + ads.example.com", + 40, + 2, + 4, + ); + assert!(rows.len() > 2, "the fixture has to fold more than once"); + + assert!( + rows[0].starts_with(" ") && !rows[0].starts_with(" "), + "the first row does not carry the first indent: {:?}", + rows[0], + ); + for line in &rows[1..] { + assert!( + line.starts_with(" ") && !line.starts_with(" "), + "a continuation row does not carry the hanging indent: {line:?}", + ); + } + + // A heading folds at zero and continues at two, which is what makes it + // distinguishable from a fact by POSITION as well as by colour — the whole of + // what a reader under `NO_COLOR` or `--plain` has to go on. + let heading = page::folded("slowest calls, of the last two hundred runs", 20, 0, 2); + assert!(heading.len() > 1); + assert!(!heading[0].starts_with(' '), "a heading is indented to zero"); + assert!(heading[1].starts_with(" ")); +} + +/// **An empty text is one indented row, not no rows at all.** +/// +/// A caller folding an empty value gets a row it can put on the page, which is +/// what keeps `Row::Fact("label", "")` from silently disappearing. The tail +/// condition is `used > 0 || rows.is_empty()`, and the second half of it is the +/// whole of this behaviour. +/// +/// Sabotage: drop `|| rows.is_empty()`, under which an empty string folds to +/// nothing and a fact with an empty value vanishes from the page rather than +/// reading as empty. +#[test] +fn an_empty_text_folds_to_one_row_rather_than_to_none() { + assert_eq!(page::folded("", 80, 2, 4), vec![" ".to_string()]); + assert_eq!(page::folded(" ", 80, 0, 0), vec![String::new()]); +} + +// --------------------------------------------------------------------------- +// `commit` +// --------------------------------------------------------------------------- + +/// **A committed page has edges, and both of them name it.** +/// +/// It lands in the terminal's own scrollback beside every earlier turn and every +/// line of transcript, so a passage with no edges is one a reader cannot tell the +/// extent of — where the page started, and whether what follows is still part of +/// it. The closing rule carries `ends` for the same reason `transcript` does: two +/// identical rules would be two openings. +/// +/// Sabotage: drop the closing line, under which a page runs into whatever the +/// next turn writes and nothing fails anywhere else. +#[test] +fn a_page_is_edged_at_both_ends_with_the_title_on_each() { + let theme = ascii(); + let rule = theme.glyphs.rule; + let lines = page::commit( + "cost", + &[Row::fact("calls", "12"), Row::note("nothing recorded")], + &theme, + 80, + ); + let rows = drawn(&lines); + + assert_eq!(rows[0], format!("{rule}{rule}{rule} cost")); + assert_eq!( + rows[rows.len() - 1], + format!("{rule}{rule}{rule} cost ends"), + "the closing rule does not say the page ended, so it reads as a second opening", + ); + assert_eq!( + rows.len(), + 4, + "two rows between two rules, and nothing else: {rows:?}", + ); +} + +/// **Every row kind reaches the page, and the two that qualify a figure do not +/// read the same as the two that state one.** +/// +/// `Row::Note` and `Row::caveat` are the same variant with different tones, and +/// the difference is not decoration: io-harness's own pricing documentation calls +/// a renderer that draws a floor and a total identically "lying by omission". A +/// heading is separated by position *and* tone for the same reason, since a reader +/// under `NO_COLOR` has only the position. +/// +/// Sabotage: make `Row::caveat` call `Row::note`, under which a total that is a +/// floor reads exactly like one that is not, and no test that reads text alone +/// can tell. +#[test] +fn every_row_kind_draws_and_a_caveat_does_not_read_like_a_note() { + let theme = ascii(); + let lines = page::commit( + "stats", + &[ + Row::heading("runs by outcome"), + Row::fact("success", "12"), + Row::Blank, + Row::note("nothing recorded"), + Row::caveat("the cost above is a floor and not a total"), + ], + &theme, + 80, + ); + let rows = drawn(&lines); + + assert_eq!(rows[1], "runs by outcome", "a heading is indented to zero"); + assert_eq!(rows[2], " success: 12"); + assert_eq!(rows[3], "", "a blank row is blank"); + assert_eq!(rows[4], " nothing recorded"); + assert_eq!(rows[5], " the cost above is a floor and not a total"); + + // The tones. Read off the theme rather than compared to a colour typed here, + // so a palette change moves both sides together and only a collapse of the + // two tones fails. + let style_of = |index: usize| lines[index].spans[0].style; + assert_eq!(style_of(1), theme.style(Tone::Accent), "the heading"); + assert_eq!(style_of(2), theme.style(Tone::Normal), "the fact"); + assert_eq!(style_of(4), theme.style(Tone::Muted), "the note"); + assert_eq!(style_of(5), theme.style(Tone::Warning), "the caveat"); + assert_ne!( + style_of(4), + style_of(5), + "a qualification and an ordinary sentence draw identically, so a floor \ + reads as a total", + ); +} + +/// **A page folds its rows rather than letting one run past the edge, at every +/// width the product supports.** +/// +/// The integration of the two halves: `commit` hands each row to `folded` with +/// the indents that row kind takes, and a page whose facts were merely `format!`ed +/// straight into `Line`s would pass every content assertion above and overflow the +/// terminal on the one row that matters — which on `/cost` and `/stats` is the row +/// carrying a namespaced model id. +/// +/// Sabotage: push `Line::from(format!(" {label}: {value}"))` in the `Fact` arm +/// instead of folding it, and this is the only test that fails. +#[test] +fn no_row_of_a_committed_page_runs_past_the_terminal() { + let theme = ascii(); + let rows = [ + Row::heading("slowest calls, of the last 200 runs"), + Row::fact( + "anthropic/claude-sonnet-4.5-with-an-absurdly-long-deployment-suffix", + "4213 ms", + ), + Row::fact("workspace", "/Users/somebody/Documents/work/monorepo/services/billing"), + Row::caveat( + "four calls used a model with no rate in the price table, so the cost above \ + is a floor and not a total", + ), + Row::Blank, + ]; + + for width in 20..=120u16 { + let lines = page::commit("cost", &rows, &theme, width); + // The first and last rows are the rules. They carry the title rather than + // content and are deliberately not folded — a rule broken across two rows + // would stop being an edge — so the claim is about everything between them. + for line in &lines[1..lines.len() - 1] { + let drawn: usize = line + .spans + .iter() + .map(|span| span.content.chars().count()) + .sum(); + assert!( + drawn <= width as usize, + "a page row overflowed {width} columns: {line:?}", + ); + } + } +} + +// --------------------------------------------------------------------------- +// The de-duplication itself +// --------------------------------------------------------------------------- + +/// `status::folded` exactly as 0.21.0 shipped it, before the four surfaces shared +/// one. +/// +/// Verbatim, comments and all. It is here to disagree, so it is not tidied, not +/// renamed and not improved — a copy that had been cleaned up would be a third +/// implementation and would prove nothing about the second. +fn folded_0_21_0(text: &str, width: usize, first: usize, rest: usize) -> Vec { + let mut rows: Vec = Vec::new(); + let mut indent = first; + let mut row = " ".repeat(indent); + let mut used = 0usize; + for word in text.split_whitespace() { + let mut word = word; + loop { + let room = width.saturating_sub(indent).max(1); + let space = usize::from(used > 0); + let length = word.chars().count(); + if used + space + length <= room { + if space == 1 { + row.push(' '); + } + row.push_str(word); + used += space + length; + break; + } + if used > 0 { + rows.push(std::mem::take(&mut row)); + indent = rest; + row = " ".repeat(indent); + used = 0; + continue; + } + let head: String = word.chars().take(room).collect(); + word = &word[head.len()..]; + row.push_str(&head); + rows.push(std::mem::take(&mut row)); + indent = rest; + row = " ".repeat(indent); + } + } + if used > 0 || rows.is_empty() { + rows.push(row); + } + rows +} + +/// `context::wrapped` exactly as 0.21.0 shipped it. +/// +/// The same twenty lines with the two indents written in rather than taken as +/// arguments, which is the whole of what separated the two copies. Kept whole for +/// the same reason as above: a paraphrase would be testing the paraphrase. +fn wrapped_0_21_0(text: &str, width: usize) -> Vec { + let mut rows: Vec = Vec::new(); + let mut indent = 2usize; + let mut row = " ".repeat(indent); + let mut used = 0usize; + for word in text.split_whitespace() { + let mut word = word; + loop { + let room = width.saturating_sub(indent).max(1); + let space = usize::from(used > 0); + let length = word.chars().count(); + if used + space + length <= room { + if space == 1 { + row.push(' '); + } + row.push_str(word); + used += space + length; + break; + } + if used > 0 { + rows.push(std::mem::take(&mut row)); + indent = 4; + row = " ".repeat(indent); + used = 0; + continue; + } + let head: String = word.chars().take(room).collect(); + word = &word[head.len()..]; + row.push_str(&head); + rows.push(std::mem::take(&mut row)); + indent = 4; + row = " ".repeat(indent); + } + } + if used > 0 || rows.is_empty() { + rows.push(row); + } + rows +} + +/// **The shared folder is the two it replaced, row for row, at every width.** +/// +/// A de-duplication is a claim about behaviour that no user-facing test states: +/// `/status` and `/context` each have a suite full of `contains` assertions, and +/// not one of them would notice a fold that broke one column earlier — the words +/// are all still on the page, in order, in the same rows plus or minus one. The +/// only way to state "nothing moved" is to keep the thing it moved from. +/// +/// The sweep runs over the fixtures that reach the split arm as well as the ones +/// that do not, because that arm is the one a rewrite gets wrong: it is reached +/// only when a single word cannot fit a row of its own, which on a real `/status` +/// page happens at eighty columns for a deep workspace path and never in a test +/// written against a short one. +/// +/// `wrapped` is compared at its own two indents, since that is the only shape it +/// had; anything else would be comparing the new function against a version of the +/// old one that never existed. +/// +/// Sabotage: change `used + space + length <= room` to `<`, which is the classic +/// off-by-one and is invisible on any page whose longest row is not exactly full — +/// under which this test fails at a handful of widths per fixture and every other +/// test in the repository stays green. +#[test] +fn the_shared_folder_is_the_two_copies_it_replaced() { + for text in texts() { + for width in 1..=120usize { + for (first, rest) in [(0usize, 0usize), (2, 4), (0, 2), (4, 2), (8, 8)] { + assert_eq!( + page::folded(&text, width, first, rest), + folded_0_21_0(&text, width, first, rest), + "the shared folder differs from `status::folded` at {width} columns, \ + indents {first}/{rest}, on {text:?}", + ); + } + assert_eq!( + page::folded(&text, width, 2, 4), + wrapped_0_21_0(&text, width), + "the shared folder differs from `context::wrapped` at {width} columns \ + on {text:?}", + ); + } + } +} diff --git a/tests/palette.rs b/tests/palette.rs index a1d6fbe..fbbde92 100644 --- a/tests/palette.rs +++ b/tests/palette.rs @@ -304,12 +304,13 @@ fn f1_an_exact_name_outranks_a_prefix_which_outranks_a_scattered_match() { #[test] fn f1_equal_scores_keep_the_first_row_still_between_keystrokes() { - // `c` matches EIGHT rows since 0.17.0: `copy`, `copy diff`, `contain`, - // `clear`, `config`, `context` and `compact` all begin with it, and `mcp` - // merely holds one — which is the point of the arm, since a scattered match - // ranks below every prefix. `attach`, which also merely held one, stopped - // being a command in 0.13.1. `co` narrows to the six that begin with `co`: - // `clear` has no `o` after its `c`, and neither does `mcp`. + // `c` matches NINE rows since 0.22.0: `copy`, `copy diff`, `contain`, + // `clear`, `config`, `context`, `compact` and now `cost` all begin with it, + // and `mcp` merely holds one — which is the point of the arm, since a + // scattered match ranks below every prefix. `attach`, which also merely held + // one, stopped being a command in 0.13.1. `co` narrows to the seven that + // begin with `co`: `clear` has no `o` after its `c`, and neither does `mcp`. + // (`stats`, the other command 0.22.0 added, holds no `c` at all.) // The three score the same and the tie-break is the order they were handed // in. The defect: an unstable sort swaps them on a keystroke that did not // change the result, and `Enter` takes a row nobody chose. @@ -320,14 +321,14 @@ fn f1_equal_scores_keep_the_first_row_still_between_keystrokes() { // quietly stopped happening. let mut picker = palette(); type_at(&mut picker, "c"); - assert_eq!(picker.matching(), 8); + assert_eq!(picker.matching(), 9); // `clear` and not `copy` since the list became grouped: the tie-break is the // order the rows were handed in, and the session group now leads. The // PROPERTY is unchanged and is what this test is about — the marked row must // not move on a keystroke that did not change the result. assert_eq!(marked(&picker), "clear"); type_at(&mut picker, "o"); - assert_eq!(picker.matching(), 6); + assert_eq!(picker.matching(), 7); assert_eq!( marked(&picker), "contain", diff --git a/tests/prices.rs b/tests/prices.rs new file mode 100644 index 0000000..fd03743 --- /dev/null +++ b/tests/prices.rs @@ -0,0 +1,1066 @@ +//! Where a price comes from, what it is spelled as, and what it refuses. +//! +//! **Four separate claims live in `src/prices.rs` and only one of them is +//! arithmetic.** The date conversion is a pure function of a number and can be +//! checked against a calendar; the rest are claims about *naming* — which id a +//! rate is filed under, whose catalogue it is attributed to, and whether the row +//! survives a trip through a real configuration file and back out of io-harness. +//! A test that only exercised `Catalogue::of` in memory would pass on every one +//! of those and still ship a release that writes nothing an operator can read. +//! +//! So the end of this file is deliberately not a unit test. It takes the edits +//! the module produces, runs them through `io_cli::edit::apply` over a +//! hand-written `io.toml`, hands the result to `io_harness::Config::from_toml`, +//! and asks the resulting `PriceTable` what a call on that model costs. That +//! round trip is the property — every step before it is a step towards it, and a +//! module that got the spelling wrong is a module whose prices exist in a file +//! nothing can read. + +use io_cli::edit; +use io_cli::prices::{self, Catalogue, DEFAULT_REFERENCE_URL}; +use io_harness::pricing::{Price, PriceTable}; +use io_harness::{Config, ModelInfo, PriceSource, ProviderSpec}; + +// --------------------------------------------------------------------------- +// Fixtures +// --------------------------------------------------------------------------- + +/// A configuration naming one provider, held so a `&ProviderSpec` can be taken +/// off it. +/// +/// **Built through io-harness's own parser rather than as a struct literal**, and +/// that is not ceremony. `ProviderSpec` is `#[non_exhaustive]` and its +/// `Compatible` variant carries seven fields; a literal here would be a second +/// spelling of a shape the harness owns, and it would stop compiling the release +/// the harness widens it — which is the wrong test failing for the wrong reason. +/// A `[[provider]]` table is also exactly what an operator writes, so the fixture +/// and the field agree by construction. +fn configured(body: &str) -> Config { + Config::from_toml(body).expect("the fixture is a configuration io-harness accepts") +} + +fn openrouter() -> Config { + configured("[[provider]]\nkind = \"openrouter\"\nmodel = \"anthropic/claude-sonnet-4.5\"\n") +} + +fn anthropic() -> Config { + configured("[[provider]]\nkind = \"anthropic\"\nmodel = \"claude-sonnet-4.5\"\n") +} + +fn openai() -> Config { + configured("[[provider]]\nkind = \"openai\"\nmodel = \"gpt-4.1\"\n") +} + +/// A provider the reference catalogue has never heard of and cannot speak for. +fn local() -> Config { + configured( + "[[provider]]\nkind = \"compatible\"\nbase_url = \"http://localhost:11434/v1\"\n\ + model = \"llama3.2\"\nauth = \"none\"\n", + ) +} + +fn spec(config: &Config) -> &ProviderSpec { + config.provider_spec().expect("the fixture names a provider") +} + +/// One catalogue row the vendor put a rate on. +fn priced(id: &str, input: u64, output: u64) -> ModelInfo { + ModelInfo { + id: id.to_string(), + price: Some(Price { + input, + output, + ..Price::ZERO + }), + price_source: Some(PriceSource::Vendor), + ..Default::default() + } +} + +/// One catalogue row the vendor served and put no rate on. +/// +/// **`price: None`, never `Some(Price::ZERO)`**, which is io-harness's own rule +/// stated on `ModelInfo::price` in as many words: zero is a rate and unknown is +/// not one, and a catalogue that conflated them would report a real bill as free. +fn unpriced(id: &str) -> ModelInfo { + ModelInfo { + id: id.to_string(), + price: None, + price_source: None, + ..Default::default() + } +} + +// --------------------------------------------------------------------------- +// `date` +// --------------------------------------------------------------------------- + +/// Whether `year` carries a twenty-ninth of February, by the rule rather than by +/// the shortcut. +/// +/// Divisible by four, except centuries, except centuries divisible by four +/// hundred. The third clause is the one a naive `% 4` gets wrong, and 2100 is the +/// first year the epoch can express where it matters — which is why the sweep +/// below runs past it. +fn leap(year: i64) -> bool { + year % 4 == 0 && (year % 100 != 0 || year % 400 == 0) +} + +fn days_in(year: i64, month: i64) -> i64 { + match month { + 1 | 3 | 5 | 7 | 8 | 10 | 12 => 31, + 4 | 6 | 9 | 11 => 30, + _ => { + if leap(year) { + 29 + } else { + 28 + } + } + } +} + +/// **The date is the civil calendar, checked against a second calendar rather +/// than against itself.** +/// +/// `prices::date` is Howard Hinnant's `civil_from_days`: an era of four hundred +/// years, a year-of-era, a day-of-year and two integer divisions that reconstruct +/// the month with no lookup table. It is correct and it is completely opaque, and +/// a test that asserted `date(1_772_150_400) == "2026-02-27"` would be asserting +/// that somebody once ran the function and wrote down what it said. +/// +/// So there are two halves here. The first is a handful of dates an ordinary +/// person can verify by counting — the epoch itself, a leap day with the day +/// either side of it, a year boundary with the day either side of *it*, and +/// 2100-03-01, which follows a February the century rule shortens back to +/// twenty-eight. The second is a walk: an independent day-by-day calendar, +/// written out above with the leap rule spelled in full, stepped forward from the +/// epoch for a hundred and ten years and compared against `date` at every single +/// step. The two implementations share nothing but the definition of a day. +/// +/// Sabotage: drop the `|| year % 400 == 0` clause from Hinnant's era arithmetic — +/// which is to say make 2000 an ordinary century — and the walk fails on +/// 2000-02-29 while every hand-written date before it still passes. Or drop the +/// `if month <= 2 { year + 1 }` correction, and every January and February in the +/// range moves back a year while the whole of March to December stays right. +#[test] +fn the_date_is_the_civil_calendar_and_not_a_transcription_of_the_function() { + // Counted by hand. 1970-01-01 is day zero; 2000-01-01 is thirty years of + // three hundred and sixty-five days plus the seven leap days of 1972 to + // 1996, which is 10,957 days, which is 946,684,800 seconds. + for (secs, expected) in [ + (0u64, "1970-01-01"), + // The last second of the first day. A conversion that divided wrongly + // would tip over here rather than at midnight. + (86_399, "1970-01-01"), + (86_400, "1970-01-02"), + // A year boundary, with the day either side of it. + (946_598_400, "1999-12-31"), + (946_684_800, "2000-01-01"), + // A leap day, with the day either side of it. 2000 is the century that + // *is* a leap year, which is the rule's hardest case in this range. + (951_696_000, "2000-02-28"), + (951_782_400, "2000-02-29"), + (951_868_800, "2000-03-01"), + // And the century that is not. 2100 is divisible by four and has no + // twenty-ninth of February, so the first of March follows the + // twenty-eighth directly. + (4_107_456_000, "2100-02-28"), + (4_107_542_400, "2100-03-01"), + ] { + assert_eq!( + prices::date(secs), + expected, + "{secs} seconds after the epoch is {expected}", + ); + } + + // The seconds inside a day do not reach the answer. Asserted rather than + // assumed, because the whole point of taking a `u64` of seconds is that the + // caller hands over a clock reading and not a date. + for offset in [0u64, 1, 3_600, 43_200, 86_399] { + assert_eq!( + prices::date(951_782_400 + offset), + "2000-02-29", + "a time of day moved the date", + ); + } + + // The walk. Forty thousand days is 1970-01-01 through the middle of 2079, + // which covers twenty-seven leap years, one leap century and — with the + // hand-written pair above — the century that is not one. + let (mut year, mut month, mut day) = (1970i64, 1i64, 1i64); + for index in 0..40_000u64 { + assert_eq!( + prices::date(index * 86_400), + format!("{year:04}-{month:02}-{day:02}"), + "the two calendars disagree {index} days after the epoch", + ); + day += 1; + if day > days_in(year, month) { + day = 1; + month += 1; + } + if month > 12 { + month = 1; + year += 1; + } + } +} + +// --------------------------------------------------------------------------- +// `Catalogue::of` +// --------------------------------------------------------------------------- + +/// **On OpenRouter the reference catalogue is the vendor, and saying otherwise +/// would understate it.** +/// +/// The catalogue io-cli reads *is* OpenRouter's own `/api/v1/models`, so an +/// OpenRouter operator is being quoted the provider's published rate by the +/// provider. Calling that a reference would attach a hedge to the one case where +/// no hedge is owed. The ids are also kept whole — `anthropic/claude-sonnet-4.5` +/// is how OpenRouter names the model, and it is therefore what io-harness records +/// on the provider call and what the price has to be filed under. +/// +/// Sabotage: strip the vendor prefix for OpenRouter too, which reads as a tidy-up +/// and is the one change that makes every OpenRouter row unfindable — the key in +/// the table becomes `claude-sonnet-4.5` and the `model` on every call stays +/// `anthropic/claude-sonnet-4.5`, so `Spend::unpriced_calls` counts the lot and +/// `/cost` reports a floor of zero without a single thing failing. +#[test] +fn openrouter_keeps_the_catalogues_own_spelling_and_is_quoted_as_the_vendor() { + let config = openrouter(); + let catalogue = Catalogue::of( + spec(&config), + vec![ + priced("anthropic/claude-sonnet-4.5", 3_000_000, 15_000_000), + priced("openai/gpt-4.1", 2_000_000, 8_000_000), + ], + "2026-08-27", + ); + + assert_eq!(catalogue.as_of, "2026-08-27"); + assert_eq!(catalogue.served, 2); + assert_eq!( + catalogue.source, + PriceSource::Vendor, + "the catalogue is OpenRouter's own, so for OpenRouter it is not a reference", + ); + let ids: Vec<&str> = catalogue.rows.iter().map(|(id, _)| id.as_str()).collect(); + assert_eq!( + ids, + ["anthropic/claude-sonnet-4.5", "openai/gpt-4.1"], + "an id was re-spelled, so the key will not match the model on a call", + ); +} + +/// **Anthropic and OpenAI lose the prefix and gain the hedge, and both halves +/// matter.** +/// +/// Neither vendor publishes a price on any endpoint, so the numbers come from a +/// third party's catalogue and every surface that draws money has to say so. The +/// spelling moves in the opposite direction from the attribution: the catalogue +/// namespaces the model (`anthropic/claude-sonnet-4.5`) and the operator's own +/// configuration does not (`claude-sonnet-4.5`), and it is the operator's +/// spelling that io-harness records on the call. +/// +/// The rows a prefix does not match are dropped entirely rather than kept under +/// their catalogue name. An OpenAI operator has no use for a row keyed +/// `anthropic/claude-sonnet-4.5`; it can never match a call they make, and it +/// would inflate the model count `/cost` reports and the count `too_short` +/// measures a later read against. +/// +/// Sabotage: keep the whole catalogue for these two providers rather than +/// filtering it, and this test fails on the row count while every other test in +/// this file — including the round trip — still passes. +#[test] +fn anthropic_and_openai_lose_their_prefix_and_are_quoted_as_a_reference() { + let served = || { + vec![ + priced("anthropic/claude-sonnet-4.5", 3_000_000, 15_000_000), + priced("anthropic/claude-opus-4.1", 15_000_000, 75_000_000), + // Served by this provider, and priced by nobody. It is what makes + // `served` and `rows.len()` different numbers below. + unpriced("anthropic/claude-experimental-preview"), + priced("openai/gpt-4.1", 2_000_000, 8_000_000), + priced("google/gemini-2.5-pro", 1_250_000, 10_000_000), + ] + }; + + let config = anthropic(); + let catalogue = Catalogue::of(spec(&config), served(), "2026-08-27"); + let ids: Vec<&str> = catalogue.rows.iter().map(|(id, _)| id.as_str()).collect(); + assert_eq!( + ids, + ["claude-opus-4.1", "claude-sonnet-4.5"], + "the prefix survived, or a model this provider does not serve did", + ); + assert_eq!( + catalogue.source, + PriceSource::Reference(DEFAULT_REFERENCE_URL.to_string()), + "Anthropic publishes no prices, so a rate about Anthropic is not from Anthropic", + ); + // **`served` counts what THIS provider serves, and `rows` counts what of it + // carries a rate.** The two are deliberately different numbers, and the gap is + // the sentence `fill_prices` writes: "priced two of the three models the + // reference catalogue serves". Counting the reference's whole four-hundred-row + // answer here would tell an Anthropic operator their rates cover two of four + // hundred, which is true of the catalogue and meaningless about them. + assert_eq!(catalogue.served, 3, "three of the fixture are Anthropic's"); + assert_eq!(catalogue.rows.len(), 2, "two of those three carry a rate"); + + let config = openai(); + let catalogue = Catalogue::of(spec(&config), served(), "2026-08-27"); + let ids: Vec<&str> = catalogue.rows.iter().map(|(id, _)| id.as_str()).collect(); + assert_eq!(ids, ["gpt-4.1"]); + assert_eq!( + catalogue.source, + PriceSource::Reference(DEFAULT_REFERENCE_URL.to_string()), + ); +} + +/// **A provider the reference cannot speak for gets no rows at all.** +/// +/// A `compatible` endpoint is a URL and a model slug: a proxy, a gateway, a +/// runtime on a port. The reference catalogue has never heard of it, and the only +/// thing it could offer is a row whose slug happens to collide — `llama3.2` on +/// somebody's laptop is not `meta-llama/llama-3.2-3b-instruct` on OpenRouter, and +/// pricing a local run at a hosted rate would be an invented bill. +/// +/// `served` is zero for the same reason `rows` is: the count is of what *this* +/// provider serves, and the reference cannot say that a local endpoint serves +/// anything at all. Reporting the reference's own four hundred here would be the +/// same confident wrong answer in a different column. +/// +/// Sabotage: make the `_` arm of `verify::named` return `models` unchanged, and a +/// local endpoint gains a full price table for models it does not run. +#[test] +fn a_provider_the_reference_cannot_speak_for_yields_no_rows() { + let config = local(); + let catalogue = Catalogue::of( + spec(&config), + vec![ + priced("meta-llama/llama-3.2-3b-instruct", 20_000, 20_000), + priced("llama3.2", 20_000, 20_000), + ], + "2026-08-27", + ); + + assert!( + catalogue.rows.is_empty(), + "a local endpoint was priced from a catalogue that has never seen it: {:?}", + catalogue.rows, + ); + assert_eq!( + catalogue.served, 0, + "the reference claimed to know what this endpoint serves", + ); +} + +/// **A model the catalogue served with no price is ABSENT, never entered at +/// zero.** +/// +/// This is the single most consequential line in the module and it is one that +/// leaves no trace when it goes wrong. `PriceTable::price` returning `None` is +/// what makes `Spend::unpriced_calls` count the call, which is what makes +/// `Total::is_floor` true, which is what makes `/cost` say the number it just +/// drew is a floor rather than a total. A row entered as `Price::ZERO` satisfies +/// every one of those checks — the model *is* priced, at nothing — so the call is +/// counted as free, the total is reported as complete, and every surface agrees +/// with every other surface about a number that is wrong. +/// +/// io-harness says the same thing on `ModelInfo::price` itself: "never +/// `Price::ZERO` to mean unknown". This asserts io-cli honours it. +/// +/// Sabotage: `.map(|m| (m.id, m.price.unwrap_or(Price::ZERO)))` in +/// `verify::priced` — five characters that read as a tidy-up of an `Option`, and +/// under which this is the only test in the repository that fails. +#[test] +fn a_model_the_catalogue_priced_at_nothing_is_absent_rather_than_zero() { + let config = openrouter(); + let catalogue = Catalogue::of( + spec(&config), + vec![ + priced("anthropic/claude-sonnet-4.5", 3_000_000, 15_000_000), + unpriced("some-lab/experimental-preview"), + unpriced("another-lab/unreleased"), + ], + "2026-08-27", + ); + + assert_eq!(catalogue.served, 3, "all three were served"); + let ids: Vec<&str> = catalogue.rows.iter().map(|(id, _)| id.as_str()).collect(); + assert_eq!( + ids, + ["anthropic/claude-sonnet-4.5"], + "an unpriced model reached the table, where it will be counted as free", + ); + // And the proof that absence is what the table sees: the row that is missing + // is missing from the `PriceTable` a write of these rows would produce. + let table = catalogue + .rows + .iter() + .fold( + PriceTable::new(catalogue.as_of.clone()), + |table, (id, price)| table.with(id.clone(), *price), + ); + assert_eq!(table.price("some-lab/experimental-preview"), None); + assert!(table.price("anthropic/claude-sonnet-4.5").is_some()); +} + +// --------------------------------------------------------------------------- +// `Catalogue::too_short` +// --------------------------------------------------------------------------- + +/// **The refusal that stops a truncated read from shrinking a bill in silence.** +/// +/// Every other failure in this area is loud: a catalogue that cannot be reached +/// leaves the operator with no prices and a sentence saying so. A catalogue that +/// answers with four rows where four hundred were expected is the quiet one — the +/// write succeeds, the file parses, `/cost` draws a smaller number, and nothing +/// anywhere says the table lost three hundred and ninety-six models. +/// +/// The rule has three arms and each is a different judgement. A first fill has +/// nothing to lose and is never refused, whatever it holds. An empty answer +/// against a table that has rows is refused outright — a replacement that prices +/// nothing is not a price change. And an answer under half the size of what it +/// would replace is refused as a truncation, while an answer merely *smaller* is +/// allowed through, because vendors do retire models and a rule that refused +/// every shrinkage would freeze the table the first time one did. +/// +/// Sabotage: return `false` for the empty case — under which the emptiest possible +/// answer, the one a half-read socket produces, is the one answer that is always +/// accepted. +#[test] +fn a_first_fill_is_never_refused_and_a_truncated_answer_always_is() { + let config = openrouter(); + let of = |count: usize| { + Catalogue::of( + spec(&config), + (0..count) + .map(|index| priced(&format!("vendor/model-{index}"), 1_000_000, 2_000_000)) + .collect(), + "2026-08-27", + ) + }; + + // A first fill. Nothing to compare against, so nothing is refused — including + // the empty answer, which the caller handles separately by saying no prices + // were written rather than by keeping a table that does not exist. + assert!(!of(0).too_short(0), "a first fill has nothing to lose"); + assert!(!of(1).too_short(0)); + assert!(!of(400).too_short(0)); + + // An empty answer against a table with rows in it. Refused at every size, + // including one, which is the case a `> existing / 2` rule would let through. + assert!(of(0).too_short(1), "an empty answer replaced a real table"); + assert!(of(0).too_short(400)); + + // Drastically shorter: refused. Similar: not. The line sits at half, and + // both sides of it are asserted so that moving it fails this rather than + // passing quietly in whichever direction it moved. + assert!(of(4).too_short(400), "a truncated read replaced a full table"); + assert!(of(199).too_short(400), "under half is a truncation"); + assert!( + !of(200).too_short(400), + "exactly half is allowed: vendors do retire models", + ); + assert!(!of(380).too_short(400), "a slightly smaller answer is a normal one"); + assert!(!of(420).too_short(400), "a larger answer is never a truncation"); +} + +// --------------------------------------------------------------------------- +// `Catalogue::edits` — the round trip +// --------------------------------------------------------------------------- + +/// The `[prices]` section a file already carrying prices has. +const EXISTING: &str = "\ +[[provider]] +kind = \"openai\" +model = \"gpt-4.1\" + +[prices] +as_of = \"2026-01-01\" + +[prices.models] +\"gpt-4o\" = { input = 2500000, output = 10000000 } +"; + +/// A file written by the wizard: a provider, io-cli's own section, and no prices +/// at all. This is what a first fill is applied to. +const FRESH: &str = "\ +[[provider]] +kind = \"anthropic\" +model = \"claude-sonnet-4.5\" + +[app.io-cli] +theme = \"dark\" +"; + +/// **A model id with a dot in it survives the write, and the proof is that +/// io-harness prices a call with it afterwards.** +/// +/// A bare TOML key is `A-Za-z0-9_-` and nothing else. `gpt-4.1` written bare is +/// not a key with a dot in it — it is two path segments, `gpt-4` and `1`, and the +/// rate lands in a table nothing reads. Nothing fails: the file parses, the +/// section exists, `Config::prices()` returns a table, and `price("gpt-4.1")` +/// answers `None` forever. +/// +/// So the assertion deliberately does not stop at the text. It applies the edits +/// to a real file, parses the result with io-harness's own loader, and asks the +/// resulting `PriceTable` what the model costs — because the text containing +/// `"gpt-4.1"` and the table answering to `gpt-4.1` are two different claims and +/// only the second one is the feature. +/// +/// Sabotage: drop the `quoted` call in `Catalogue::edits` and spell the path with +/// `format!("prices.models.{model}")`. The file still parses, this test fails on +/// `price`, and nothing else in the repository notices. +#[test] +fn a_dotted_model_id_survives_the_write_and_prices_a_call() { + let config = openai(); + let catalogue = Catalogue::of( + spec(&config), + vec![ + priced("openai/gpt-4.1", 2_000_000, 8_000_000), + priced("openai/gpt-4.1-mini", 400_000, 1_600_000), + priced("openai/o3", 2_000_000, 8_000_000), + ], + "2026-08-27", + ); + + // The flag comes from the file rather than from a literal, because that pair + // is the feature: `has_models_section` asks the document and `edits` chooses + // its shape from the answer, and a test that hard-coded `true` here would pass + // against a reader that always said so. + assert!( + prices::has_models_section(EXISTING), + "the fixture is meant to be the refresh case", + ); + let written = edit::apply(EXISTING, &catalogue.edits(prices::has_models_section(EXISTING))) + .expect("the edits produce a configuration file that parses"); + + // The spelling, so a failure here names the cause rather than the symptom. + assert!( + written.contains("\"gpt-4.1\""), + "the dotted id was written bare, which is two keys and not one:\n{written}", + ); + + // The property. io-harness reads the file it would really read, and the table + // it builds answers to the id io-harness would really record on a call. + let reloaded = Config::from_toml(&written).expect("io-harness accepts the written file"); + let table = reloaded.prices().expect("the file carries a price table"); + assert_eq!(table.as_of(), "2026-08-27", "the date did not move"); + assert_eq!( + table.price("gpt-4.1"), + Some(Price { + input: 2_000_000, + output: 8_000_000, + ..Price::ZERO + }), + "the dotted id reached a key io-harness cannot find:\n{written}", + ); + assert!(table.price("gpt-4.1-mini").is_some()); + assert!(table.price("o3").is_some()); + + // **A row the catalogue no longer serves is left alone**, which is the module's + // own rule and not an accident of the splice: io-harness prices a call by the + // model name recorded on it, so the old row is what prices an old run + // correctly and `/cost` reports history as well as today. + assert_eq!( + table.price("gpt-4o"), + Some(Price { + input: 2_500_000, + output: 10_000_000, + ..Price::ZERO + }), + "a model the catalogue stopped serving lost the rate that prices its history", + ); +} + +/// **The first fill: the case every new install takes, against a file that has no +/// `[prices]` section at all.** +/// +/// The test above starts from a file that already carries `[prices.models]`, so +/// every model edit finds a section to insert a key into. That is the *second* +/// fill. The first one — the wizard writes `io.toml`, the credential check +/// passes, `fill_prices` reads the catalogue and writes it — starts from a file +/// with neither `[prices]` nor `[prices.models]` in it, and it is the only path +/// an operator takes on day one. +/// +/// **Every edit in a batch is resolved against the document as it was BEFORE the +/// batch**, which is why the two cases cannot share a shape. `edit::apply` walks +/// the file's headers once and every `set` is answered from that walk — so four +/// hundred `set`s into a `[prices.models]` that does not exist yet each fall +/// through to the append arm and each emit their own `[prices.models]` header. The +/// file gains four hundred definitions of one table, TOML refuses the lot, and the +/// only thing an operator sees is "no prices were written". The first fill could +/// not have worked at all. +/// +/// So a first fill writes the section once, whole, and this test is the one that +/// says so: exactly one header, every model in it, and the whole thing readable +/// back through io-harness. The header count is asserted directly rather than left +/// to the parse, because a duplicate is only *usually* a parse error — a second +/// header carrying keys the first did not would parse on some shapes and silently +/// split the table on others. +/// +/// Sabotage: pass `true` here, or make `has_models_section` answer on the parent +/// `[prices]` rather than on the child — under which this fails with a duplicate +/// table and every other test in this file still passes. +#[test] +fn a_first_fill_writes_every_model_into_a_file_that_has_no_prices_section() { + let config = anthropic(); + let catalogue = Catalogue::of( + spec(&config), + vec![ + priced("anthropic/claude-sonnet-4.5", 3_000_000, 15_000_000), + priced("anthropic/claude-opus-4.1", 15_000_000, 75_000_000), + priced("anthropic/claude-haiku-4.5", 1_000_000, 5_000_000), + ], + "2026-08-27", + ); + assert_eq!(catalogue.rows.len(), 3, "the fixture prices three models"); + assert!( + !prices::has_models_section(FRESH), + "the fixture is meant to be the first-fill case", + ); + + let edits = catalogue.edits(prices::has_models_section(FRESH)); + assert_eq!( + edits.len(), + 2, + "a first fill is the date and the section, however many models it holds", + ); + + let written = edit::apply(FRESH, &edits).unwrap_or_else(|error| { + panic!( + "a first fill of three models produced a file that does not parse, so an operator's \ + very first `/cost` reports tokens and no money: {error}" + ) + }); + + assert_eq!( + written.matches("[prices.models]").count(), + 1, + "the fill wrote the table more than once:\n{written}", + ); + + let reloaded = Config::from_toml(&written).expect("io-harness accepts the written file"); + let table = reloaded.prices().expect("the file carries a price table"); + assert_eq!(table.as_of(), "2026-08-27"); + for (model, price) in &catalogue.rows { + assert_eq!( + table.price(model).as_ref(), + Some(price), + "`{model}` is not in the table the first fill wrote:\n{written}", + ); + } + // Nothing else in the file moved. The provider the operator just configured + // is the one thing a broken splice is most likely to take with it. + assert_eq!( + reloaded.provider_spec(), + config.provider_spec(), + "the write disturbed the provider:\n{written}", + ); + + // **And it is not a special case that only works at three.** Four hundred is + // what an OpenRouter operator's first fill really is, and it is the size at + // which the old shape produced four hundred headers rather than two. + let config = openrouter(); + let many = Catalogue::of( + spec(&config), + (0..400) + .map(|index| priced(&format!("vendor-{index}/model-4.1"), 1_000_000, 2_000_000)) + .collect(), + "2026-08-27", + ); + let written = edit::apply(FRESH, &many.edits(false)) + .expect("four hundred models are one section, not four hundred"); + assert_eq!(written.matches("[prices.models]").count(), 1); + let table = Config::from_toml(&written) + .expect("io-harness accepts it") + .prices() + .expect("the file carries a table"); + assert!( + (0..400).all(|index| table.price(&format!("vendor-{index}/model-4.1")).is_some()), + "a model was lost from a four-hundred-row first fill", + ); +} + +/// **`has_models_section` asks the file, and asks about the child table rather +/// than the parent.** +/// +/// The question `edits` needs answered is whether a `set` will find somewhere to +/// insert a key, and that is a question about `[prices.models]` and nothing else. +/// A `[prices]` carrying an `as_of` and no models is a real state — it is exactly +/// what a refresh whose catalogue priced nothing leaves behind — and answering +/// `true` for it sends the next fill down the `set` path into a section that is +/// not there. +/// +/// The two substitutes that look equivalent are both wrong, and the file is the +/// only thing that is not: io-cli's own record of how many models it last wrote +/// is zero for a `[prices]` an operator typed by hand, and asking the loaded +/// `PriceTable` whether it prices anything this catalogue serves answers `false` +/// for a real section whose models the provider has since replaced. +/// +/// Sabotage: drop the `path.len() == 2` test and match on any path beginning +/// `prices.models` — under which a file whose models are written as +/// `[prices.models."gpt-4.1"]` sub-tables answers `true`, takes the `set` path, +/// and lands its rows in a section that does not exist. +#[test] +fn the_models_section_is_looked_for_in_the_file_and_not_inferred() { + assert!(prices::has_models_section(EXISTING)); + assert!(!prices::has_models_section(FRESH)); + + // A `[prices]` with a date and no models: the state a refresh that priced + // nothing leaves, and the one an inference off the parent gets wrong. + assert!( + !prices::has_models_section("[prices]\nas_of = \"2026-01-01\"\n"), + "a `[prices]` with no models was read as a models section", + ); + + // An unparseable file answers `false`, which sends the caller down the create + // path — where `apply` refuses it with a complaint about the file rather than + // about the prices, which is the truthful one. + assert!(!prices::has_models_section("[prices\nas_of =")); +} + +/// **On a refresh it is one edit per model, because a rate is something an +/// operator is invited to correct by hand.** +/// +/// The alternative — one edit rewriting `[prices.models]` whole, which is what the +/// first fill does because it must — produces the same file and a very different +/// consequence. `Edit::section` replaces nothing and refuses a section that +/// exists, precisely so that a refresh cannot take this path: a whole-section +/// write discards every row nobody named, which on a refresh is every model the +/// catalogue stopped serving *and* every rate the operator corrected by hand. +/// io-harness prices a call by the model name recorded on it, so those old rows +/// are exactly what price an old run correctly, and `/cost` reports history as +/// well as today. +/// +/// The unpriced model is the other half: it contributes no edit at all, so a +/// model the catalogue served without a rate never reaches the file as a zero. +/// +/// Sabotage: collect the rows into one `Edit::section` on the refresh path too — +/// under which the first `/config` refresh silently deletes every hand-corrected +/// rate and every model the provider retired, and the file still parses. +#[test] +fn a_refresh_writes_one_row_per_model_and_leaves_the_rest_alone() { + let config = openrouter(); + let catalogue = Catalogue::of( + spec(&config), + vec![ + priced("a/one", 1_000_000, 2_000_000), + priced("a/two", 1_000_000, 2_000_000), + unpriced("a/three"), + ], + "2026-08-27", + ); + + let edits = catalogue.edits(true); + assert_eq!( + edits.len(), + 1 + catalogue.rows.len(), + "the date plus one row per PRICED model, and nothing for the unpriced one", + ); + assert_eq!(edits[0].path(), "prices.as_of"); + assert!( + edits[1..] + .iter() + .all(|edit| edit.path().starts_with("prices.models.")), + "a refresh edit addresses something other than one model's row", + ); + + // And `Edit::section` refuses the file that already has one, which is what + // stops the two paths being interchangeable by accident. + let refused = edit::apply(EXISTING, &catalogue.edits(false)) + .expect_err("a whole-section write over an existing section must be refused"); + assert!( + refused.contains("already in this file"), + "the refusal does not say why it refused: {refused}", + ); + + // A model with every dimension free still round-trips as a `Price` rather + // than being dropped: `{}` is the inline table that says exactly that, and a + // free model is a fact the table can state where an unpriced one is not. + let free = Catalogue::of( + spec(&config), + vec![ModelInfo { + id: "a/free".into(), + price: Some(Price::ZERO), + price_source: Some(PriceSource::Vendor), + ..Default::default() + }], + "2026-08-27", + ); + let written = edit::apply(EXISTING, &free.edits(true)).expect("a free model is writable"); + let table = Config::from_toml(&written) + .expect("io-harness accepts it") + .prices() + .expect("the file carries a table"); + assert_eq!( + table.price("a/free"), + Some(Price::ZERO), + "a model that costs nothing is priced at nothing, which is not the same as unpriced", + ); +} + +// --------------------------------------------------------------------------- +// `changes` +// --------------------------------------------------------------------------- + +/// **A refresh lists what moved and nothing else, and a model with no row yet is +/// `was: None` rather than a change from zero.** +/// +/// The distinction is the difference between two sentences an operator reads +/// differently: "this got cheaper, from nothing" is nonsense, and "this is new" +/// is the answer. A `was` defaulted to `Price::ZERO` would render every arrival +/// as a price rise from free, which on a four-hundred-model catalogue turns the +/// first refresh after an upstream addition into four hundred rows of noise with +/// the real change buried in it. +/// +/// The other half is the silence. A rate that has not moved is not a change, so a +/// refresh that found nothing says so in one line rather than reprinting the +/// whole table — and an operator who has been shown four hundred unchanged rows +/// once will not read the list the time it matters. +/// +/// Sabotage: compare on the model id rather than on the price, and every refresh +/// lists every model forever. +#[test] +fn changes_lists_only_what_moved_and_calls_an_unpriced_model_new() { + let config = openrouter(); + let fresh = Catalogue::of( + spec(&config), + vec![ + // Moved. + priced("a/dearer", 4_000_000, 16_000_000), + // Unchanged, to the micro-unit. + priced("a/steady", 1_000_000, 2_000_000), + // Not in the table at all. + priced("a/arrived", 500_000, 1_500_000), + ], + "2026-08-27", + ); + + let existing = PriceTable::new("2026-01-01") + .with( + "a/dearer", + Price { + input: 3_000_000, + output: 12_000_000, + ..Price::ZERO + }, + ) + .with( + "a/steady", + Price { + input: 1_000_000, + output: 2_000_000, + ..Price::ZERO + }, + ); + + let moved = prices::changes(Some(&existing), &fresh); + let names: Vec<&str> = moved.iter().map(|change| change.model.as_str()).collect(); + // **Alphabetical, because `Catalogue::rows` is and `changes` preserves it** — + // which is a property worth pinning rather than working around. A refresh + // report an operator reads twice should list the same rates in the same order + // both times, and catalogue order is whatever a remote server felt like. + assert_eq!( + names, + ["a/arrived", "a/dearer"], + "an unchanged rate was listed, a changed one was not, or the order is not stable", + ); + + // Found by name rather than by index, so a future ordering change fails the + // assertion above — which is about order — instead of silently moving these, + // which are about values. + let by = |model: &str| { + moved + .iter() + .find(|change| change.model == model) + .unwrap_or_else(|| panic!("`{model}` is not in {names:?}")) + }; + + let dearer = by("a/dearer"); + assert_eq!(dearer.was.map(|price| price.input), Some(3_000_000)); + assert_eq!(dearer.now.input, 4_000_000); + + let arrived = by("a/arrived"); + assert_eq!( + arrived.was, None, + "a model the table has never priced is new, not a change from zero", + ); + assert_eq!(arrived.now.output, 1_500_000); + + // Nothing moved: nothing listed. Including the case an operator hits most + // often, which is running `/config` twice in a day. + assert!( + prices::changes(Some(&existing), &Catalogue::of(spec(&config), vec![ + priced("a/steady", 1_000_000, 2_000_000) + ], "2026-08-27")) + .is_empty(), + "an unchanged table reported a change", + ); + + // No table at all: every row is new. The first fill, from the other side. + let first = prices::changes(None, &fresh); + assert_eq!(first.len(), 3); + assert!( + first.iter().all(|change| change.was.is_none()), + "a model was reported as having had a rate in a table that does not exist", + ); +} + +// --------------------------------------------------------------------------- +// `source_word` +// --------------------------------------------------------------------------- + +/// **The sentence under every `$` on every page, and it names the catalogue +/// rather than the connected provider.** +/// +/// An operator on Anthropic is being shown numbers about Anthropic that Anthropic +/// never published. Flattening that into "Anthropic's prices" would attribute a +/// figure to a vendor who did not state it, which is the one claim this whole +/// module exists to avoid making. So the reference form carries the URL — the +/// operator can go and read the source — and the vendor form does not, because +/// there is nothing to point at that is not the provider they are already +/// connected to. +/// +/// The `_` arm is required rather than defensive: `PriceSource` is +/// `#[non_exhaustive]`, so io-harness can add a variant this release has never +/// seen, and the honest answer for one is that io-cli does not recognise it — not +/// a name io-cli invented for it. +/// +/// Sabotage: make the `_` arm say "the provider's own catalogue", and a future +/// harness variant is silently attributed to the vendor. +#[test] +fn source_word_names_the_catalogue_and_never_the_connected_provider() { + let vendor = prices::source_word(&PriceSource::Vendor); + assert_eq!(vendor, "the provider's own catalogue"); + assert!( + !vendor.contains("http"), + "the vendor form points at a URL the operator did not need", + ); + + let reference = prices::source_word(&PriceSource::Reference( + DEFAULT_REFERENCE_URL.to_string(), + )); + assert!( + reference.contains(DEFAULT_REFERENCE_URL), + "the reference form does not say which reference: {reference}", + ); + assert!( + reference.contains("reference"), + "the hedge is the word, not the URL: {reference}", + ); + + // The URL is io-harness's own constant re-exported, not a second copy of it + // in this repository — which is what makes it move when the harness moves it. + assert_eq!( + DEFAULT_REFERENCE_URL, + io_harness::provider::catalog::DEFAULT_REFERENCE_URL, + ); +} + +// --------------------------------------------------------------------------- +// The filter, and why it is applied exactly once +// --------------------------------------------------------------------------- + +/// **`verify::named` is not idempotent, and that is why `priced` no longer +/// filters.** +/// +/// The two used to be one call each, and both filtered: `Catalogue::of` handed the +/// raw catalogue to `named` to get its count and the raw catalogue to `priced` to +/// get its rows, and `priced` filtered again internally. Applied twice to an +/// Anthropic catalogue the second pass finds no `anthropic/` prefix left to strip +/// — every id has already lost it — and `strip`'s `strip_prefix(prefix)?` discards +/// the row rather than keeping it. The result is a table with nothing in it, from +/// a catalogue that priced fifteen models, with nothing failing anywhere. +/// +/// This test states the non-idempotence directly rather than trusting the call +/// sites, because the property is what makes the call sites' shape mandatory: a +/// future edit that "tidies up" by filtering inside `priced` again is not a +/// duplicate of work already done, it is an erasure. +/// +/// The OpenRouter arm is the control. Its filter is the identity, so applying it +/// twice changes nothing — which is exactly why a test written only against +/// OpenRouter would have passed against the broken shape. +/// +/// Sabotage: put the `named` call back inside `verify::priced` — under which this +/// fails on the Anthropic arm and `Catalogue::of` starts returning an empty table +/// for both providers whose prefix it strips. +#[test] +fn the_catalogue_filter_is_applied_exactly_once_because_it_is_not_idempotent() { + let config = anthropic(); + let raw = vec![ + priced("anthropic/claude-sonnet-4.5", 3_000_000, 15_000_000), + priced("openai/gpt-4.1", 2_000_000, 8_000_000), + ]; + + let once = io_cli::verify::named(spec(&config), raw.clone()); + assert_eq!(once.len(), 1, "one of the two is Anthropic's"); + assert_eq!(once[0].id, "claude-sonnet-4.5", "the prefix is stripped"); + + let twice = io_cli::verify::named(spec(&config), once.clone()); + assert!( + twice.is_empty(), + "the filter is idempotent after all, and this test is asserting nothing: {:?}", + twice.iter().map(|model| &model.id).collect::>(), + ); + + // So `priced` takes rows that have already been named and does not filter + // again — which is the only shape under which `Catalogue::of` can both count + // and price the same set. + let rows = io_cli::verify::priced(once); + assert_eq!(rows.len(), 1, "pricing filtered a second time"); + assert_eq!(rows[0].0, "claude-sonnet-4.5"); + + // The control. OpenRouter's filter is the identity, so it survives being + // applied twice — and a test written only against it would have passed + // against the shape that lost every Anthropic row. + let config = openrouter(); + let once = io_cli::verify::named(spec(&config), raw.clone()); + let twice = io_cli::verify::named(spec(&config), once.clone()); + assert_eq!( + once.len(), + twice.len(), + "OpenRouter's filter is the identity and has to survive being applied twice", + ); + assert_eq!(twice.len(), 2); +} + +/// **`quoted` is the module's only speller, and it escapes rather than decorates.** +/// +/// Model ids carry dots, slashes and colons, and a dot in a TOML key can only be +/// spelled quoted — a bare key is `A-Za-z0-9_-` and nothing else. So this is not +/// presentation: an unquoted `gpt-4.1` is two path segments and reaches the wrong +/// place, or no place. It is spelled through the `toml` crate rather than with a +/// `format!` for the case a hand-written quote gets wrong, which is a value +/// containing a quote or a backslash — and a model id from a catalogue is +/// somebody else's string. +/// +/// Sabotage: `format!("\"{text}\"")` — which passes for every id anyone has ever +/// seen and produces a file that does not parse the first time a catalogue serves +/// one with a quote in it. +#[test] +fn a_model_id_is_spelled_by_the_toml_crate_and_not_by_a_format_string() { + assert_eq!(prices::quoted("gpt-4.1"), "\"gpt-4.1\""); + assert_eq!(prices::quoted("anthropic/claude-sonnet-4.5"), "\"anthropic/claude-sonnet-4.5\""); + + // The two a hand-written pair of quotes gets wrong. Both round-trip through + // the parser rather than being compared to a spelling typed here, because the + // escape rules are the `toml` crate's and this file should not have a second + // opinion about them. + for id in ["a\"b", "back\\slash", "new\nline"] { + let spelled = prices::quoted(id); + let parsed = toml::from_str::(&format!("probe = {spelled}")) + .unwrap_or_else(|error| { + panic!("`{id}` was spelled as {spelled}, which is not TOML: {error}") + }); + assert_eq!( + parsed.get("probe").and_then(|value| value.as_str()), + Some(id), + "`{id}` was spelled as {spelled}, which parses back as something else", + ); + } +} diff --git a/tests/stats.rs b/tests/stats.rs new file mode 100644 index 0000000..15d3ee9 --- /dev/null +++ b/tests/stats.rs @@ -0,0 +1,638 @@ +//! How the work has gone: `/stats`, and the four places it refuses to average. +//! +//! **Every figure on this page is a `Store` call io-cli had never made.** The +//! aggregates have been in io-harness since 0.30.0 — `runs_by_outcome`, +//! `runs_by_day`, `first_try`, `gate_failures_by_phase`, `recovery` and +//! `store_size` — and the interface over them reported none of it. Reading them is +//! the easy half; the hard half is the four questions the harness deliberately +//! refuses to answer, because each of them is a place where a page can quietly +//! answer a different question from the one its heading asks. +//! +//! `FirstTry` is three counts and not a rate, because the denominator is a +//! choice. `gate_failures_by_phase` covers one of two gate mechanisms, whose phase +//! vocabularies do not overlap. `ttft_ms` is `None` for a call nothing measured, +//! never zero. And `StoreSize::tables` is empty on a SQLite build without +//! `dbstat`, which is a build that cannot answer rather than a database whose +//! tables are all empty. Every one of those has an obvious wrong rendering that +//! reads perfectly well, and this file is mostly about those four. +//! +//! The store is real and driven through io-harness's own recording calls. A page +//! asserted against hand-built aggregate structs would be asserting its own format +//! strings; here the numbers come out of SQL that ran. + +use io_harness::{ + CheckpointEvent, ContextEvent, GateOutcome, ProviderCall, SandboxEvent, Store, Usage, +}; + +use io_cli::glyphs::ASCII; +use io_cli::stats::{self, RECENT_RUNS}; +use io_cli::theme::{Theme, DARK}; + +// --------------------------------------------------------------------------- +// Fixtures +// --------------------------------------------------------------------------- + +/// Wide enough that nothing folds, so a row lookup finds a whole row. +const ROOMY: u16 = 200; + +fn ascii() -> Theme { + DARK.with_glyphs(ASCII) +} + +fn call(model: &str, latency_ms: u64, ttft_ms: Option) -> ProviderCall { + ProviderCall { + step: 1, + provider: "anthropic".into(), + model: Some(model.to_string()), + usage: Some(Usage { + prompt_tokens: 900, + completion_tokens: 100, + total_tokens: 1_000, + ..Default::default() + }), + latency_ms, + ttft_ms, + ..Default::default() + } +} + +/// A store with a history: runs that ended four ways, both gate mechanisms +/// having failed, all three recovery mechanisms having fired, and provider calls +/// with a mixture of measured and unmeasured first tokens. +/// +/// Everything is recorded through the call io-harness's own documentation shows, +/// so the aggregates below are the aggregates a real run would produce rather +/// than rows this file inserted a shape it invented. +fn history() -> Store { + let store = Store::memory().expect("an in-memory store"); + + // Three runs that succeeded, one of which failed a gate on the way, and one + // that stalled. `first_try` is finished AND successful AND carrying no + // `gate_phase_failed` event, so this fixture is 5 finished, 3 succeeded, + // 2 first try — three different numbers, which is what makes a page that + // picked the wrong one visible. + for outcome in ["success", "success", "stalled", "step_cap_reached"] { + let run = store.start_run("summarise the module", "/repo").expect("a run"); + store.finish_run(run, outcome).expect("the run finishes"); + } + + let retried = store.start_run("port the parser", "/repo").expect("a run"); + // **The sandbox gate's vocabulary.** Four phases, none of which overlaps the + // review/command/contains set below. + store + .record_sandbox_event(&SandboxEvent::gate_phase_failed(retried, 2, "test-run")) + .expect("the phase failure is recorded"); + store + .record_sandbox_event(&SandboxEvent::gate_phase_failed( + retried, + 4, + "criterion-compile", + )) + .expect("the phase failure is recorded"); + // **The other gate's vocabulary, recorded in another table entirely.** These + // must not reach the phase list: they are a different mechanism, and merging + // them would produce a list whose categories mean two things. + for phase in ["review", "command", "contains"] { + store + .put_gate_attempt(retried, 3, phase, GateOutcome::Failed, "did not hold") + .expect("the gate attempt is recorded"); + } + store.finish_run(retried, "success").expect("it finishes"); + + // All three recovery mechanisms, one each, so a page that reported the wrong + // counter cannot pass by coincidence. + store + .record_context_event(retried, &ContextEvent::served(1, "anthropic")) + .expect("the fallback is recorded"); + store + .record_context_event(retried, &ContextEvent::replan(3, "no progress")) + .expect("the replan is recorded"); + store + .record_checkpoint_event(&CheckpointEvent::resume(retried, 4, "after a crash")) + .expect("the resume is recorded"); + + // Five calls, three of which measured a first token. The two that did not are + // the whole of the `None`-is-not-zero case. + for (model, latency, ttft) in [ + ("claude-sonnet-4.5", 1_200u64, Some(300u64)), + ("claude-sonnet-4.5", 4_213, Some(900)), + ("claude-opus-4.1", 800, Some(100)), + ("claude-opus-4.1", 2_500, None), + ("claude-haiku-4.5", 150, None), + ] { + store + .record_provider_call(retried, &call(model, latency, ttft)) + .expect("the call is recorded"); + } + + store +} + +/// The `/stats` page as a reader sees it: every row, spans concatenated. +fn page(store: &Store) -> Vec { + stats::committed(store, &ascii(), ROOMY) + .expect("the page draws") + .iter() + .map(|line| { + line.spans + .iter() + .map(|span| span.content.as_ref()) + .collect::() + }) + .collect() +} + +fn text(rows: &[String]) -> String { + rows.join("\n") +} + +/// The rows of one section: everything after its heading, up to the blank row +/// that separates it from the next. +/// +/// **Sections are the whole point of `Row::Heading` existing**, and a test that +/// searched the whole page for a word could not tell a figure under the right +/// heading from the same figure under the wrong one — which is exactly the +/// failure the two gate vocabularies would produce. +fn section<'a>(rows: &'a [String], heading: &str) -> &'a [String] { + let at = rows + .iter() + .position(|row| row == heading) + .unwrap_or_else(|| panic!("no `{heading}` heading on the page:\n{}", text(rows))); + let rest = &rows[at + 1..]; + let end = rest + .iter() + .position(|row| row.is_empty()) + .unwrap_or(rest.len()); + &rest[..end] +} + +/// The value of the row reading `label: …` inside `rows`. +fn field<'a>(rows: &'a [String], label: &str) -> &'a str { + let prefix = format!("{label}: "); + rows.iter() + .find_map(|row| row.trim_start().strip_prefix(&prefix)) + .unwrap_or_else(|| panic!("no `{label}` row here:\n{}", text(rows))) +} + +/// Every heading the page draws, in the order it draws them. +/// +/// A heading is indented to zero and every other row to two, which is what makes +/// this readable off the rendered rows rather than off an internal structure — and +/// it is also the property a reader in `--plain` depends on. +fn headings(rows: &[String]) -> Vec<&str> { + rows.iter() + .skip(1) + .take(rows.len().saturating_sub(2)) + .filter(|row| !row.is_empty() && !row.starts_with(' ')) + .map(String::as_str) + .collect() +} + +// --------------------------------------------------------------------------- +// The page +// --------------------------------------------------------------------------- + +/// **Every section draws, over a store with a real history in it.** +/// +/// Eight lists on one page, and each is a separate `Store` read that can fail on +/// its own. A page that lost one would still render, still be edged, and still +/// answer seven questions — so the assertion is on the whole list of headings in +/// order rather than on any one of them, because that is the only form that +/// notices a section quietly leaving. +/// +/// The sample bound is in the heading and not merely applied. `/cost` could not +/// bound its grouped reads honestly and says so in a comment; this page can, +/// because `Store::runs` returns ids and io-cli takes the tail itself — so the +/// sample really is the last N runs and the heading says which N. That is the +/// difference between a bound and a truncation: "the last 200 runs" is a true +/// statement about a subset where "by model" over a truncated read would be a +/// false statement about the whole. +/// +/// Sabotage: drop any one `rows.push(Row::heading(…))` — under which the figures +/// beneath it join the section above and read as answers to its question. +#[test] +fn every_section_of_the_page_draws_with_its_scope_in_the_heading() { + let store = history(); + let rows = page(&store); + + assert_eq!( + headings(&rows), + [ + "runs by outcome", + "runs by day", + "finished on the first try", + "gate failures by phase", + "recovery", + // Five runs in the store, all of them inside the bound. + "slowest calls, of the last 5 runs", + "time to first token", + "what the store holds", + ], + "a section left the page, or arrived under a name that does not say its \ + scope:\n{}", + text(&rows), + ); + + // The bound the heading names is the module's own constant, so a release that + // moves it moves both together and this stays a claim about honesty rather + // than about the number two hundred. + assert_eq!(RECENT_RUNS, 200); + + // Outcomes are the raw strings io-harness recorded, not a success/failure + // collapse: "stalled" and "step_cap_reached" are different endings and the + // distinction is the reason an operator is reading this page. + let outcomes = section(&rows, "runs by outcome"); + assert_eq!(field(outcomes, "success"), "3"); + assert_eq!(field(outcomes, "stalled"), "1"); + assert_eq!(field(outcomes, "step_cap_reached"), "1"); + + // Recovery: three counters, one each, so a page reading the wrong one cannot + // pass by coincidence. + let recovery = section(&rows, "recovery"); + assert_eq!(field(recovery, "fallbacks"), "1"); + assert_eq!(field(recovery, "replans"), "1"); + assert_eq!(field(recovery, "resumes"), "1"); + + // The slowest call is named by its model and its latency, in that order, + // because the question is which model is slow. + let slowest = section(&rows, "slowest calls, of the last 5 runs"); + assert_eq!(field(slowest, "claude-sonnet-4.5"), "4213 ms"); + assert!( + slowest.len() <= 5, + "the slow list is unbounded: {slowest:?}", + ); +} + +/// **The first-try share names its denominator in the row, rather than printing a +/// bare percentage.** +/// +/// io-harness returns three counts and deliberately not a rate, and it says why: +/// *first_try / succeeded* is "when we got there, how often first time" and +/// *first_try / runs* is "how often does this work at all", and both are +/// legitimate questions with different answers. Returning one number would be +/// picking for the reader and hiding which was picked. +/// +/// So the page picks one and says which. A row reading `73%` is a number two +/// readers reconstruct differently — one of them will read it as a share of every +/// run, including the ones still going — and they will disagree about whether the +/// agent got better this week. +/// +/// The fixture makes the three counts three different numbers on purpose. A +/// fixture where every run succeeded first time would give the same percentage +/// under either denominator, and the test would pass against the wrong one. +/// +/// Sabotage: `format!("{}%", first.first_try * 100 / first.runs)` with the label +/// left as it is — under which the row says "share of successful runs" and shows +/// a share of every run, which is the failure a bare percentage makes invisible. +#[test] +fn the_first_try_share_names_its_denominator_rather_than_printing_a_bare_percentage() { + let store = history(); + let rows = page(&store); + let first = section(&rows, "finished on the first try"); + + // The three counts io-harness gives, each under a label that says what it + // counts. Five runs finished, three of them succeeded, two of those had no + // gate failure — three different numbers, from one fixture. + assert_eq!(field(first, "runs finished"), "5"); + assert_eq!(field(first, "of those, succeeded"), "3"); + assert_eq!(field(first, "of those, with no gate failure"), "2"); + + // The share, and the label carries the denominator rather than the reader + // having to supply one. Two of three is sixty-six percent; two of five would + // be forty, which is the number the wrong denominator produces. + assert_eq!( + field(first, "first-try share of successful runs"), + "66%", + "the share is not two of the three runs that succeeded:\n{}", + text(&rows), + ); + assert!( + !first.iter().any(|row| row.trim() == "66%"), + "the percentage is drawn without a label saying what it is a share of: {first:?}", + ); + + // A store where nothing has succeeded yet has no share to draw, and draws + // none rather than dividing by zero or reporting a confident nought percent. + let empty = Store::memory().expect("an in-memory store"); + let run = empty.start_run("summarise", "/repo").expect("a run"); + empty.finish_run(run, "stalled").expect("it finishes"); + let rows = page(&empty); + let first = section(&rows, "finished on the first try"); + assert_eq!(field(first, "of those, succeeded"), "0"); + assert!( + !first.iter().any(|row| row.contains('%')), + "a share was drawn over a denominator of zero: {first:?}", + ); +} + +/// **The two gate vocabularies stay under separate headings, because they are two +/// mechanisms and not two spellings.** +/// +/// io-harness records gate failures twice, in two tables, with two phase +/// vocabularies that do not overlap. `SandboxEvent` phases are `subject-compile`, +/// `criterion-compile`, `test-run` and `subject-emptied`; `GateAttempt` phases are +/// `review`, `command` and `contains`. Merged into one list they would produce a +/// chart whose categories mean two different things, and the operator reading +/// "gate failures by phase" would be adding a compile failure to a review +/// disagreement. +/// +/// The fixture has both, failing, at the same time — which is the only way to +/// state that the page keeps them apart rather than that the page has only ever +/// seen one of them. The note is the other half: a reader who came looking for +/// their failing `review` gate has to be told where it is not, or they will read +/// its absence as "the review gate has never failed". +/// +/// Sabotage: `UNION ALL` the two tables behind `gate_failures_by_phase`, or add a +/// second `tallies` call over the gate attempts under the same heading — under +/// which the counts still add up, the page still renders, and this is the only +/// test that fails. +#[test] +fn the_two_gate_vocabularies_stay_under_separate_headings() { + let store = history(); + let rows = page(&store); + let gates = section(&rows, "gate failures by phase"); + + assert_eq!(field(gates, "test-run"), "1"); + assert_eq!(field(gates, "criterion-compile"), "1"); + + // The other mechanism's phases are not rows here. Checked as rows rather than + // as words, because the note below deliberately names all three in prose. + for phase in ["review", "command", "contains"] { + assert!( + gates + .iter() + .all(|row| !row.trim_start().starts_with(format!("{phase}: ").as_str())), + "`{phase}` is a gate attempt and is counted under the sandbox gate's \ + phases: {gates:?}", + ); + } + + // And the reader is told where it went, in the section itself rather than in + // a footnote somewhere else — because the question is asked while looking at + // this list. + let note = text(gates); + assert!( + note.contains("review, command and contains"), + "the section does not say which gate's phases these are not:\n{note}", + ); + assert!( + note.contains("not merged into this list"), + "the section does not say the other gate records separately:\n{note}", + ); +} + +/// **An unmeasured time to first token is `None`, and it is counted apart rather +/// than averaged in as a zero.** +/// +/// A zero would drag every figure on the section toward an instant nothing ever +/// took, and it would do it worst on exactly the paths that do not stream — which +/// are the paths an operator is most likely to be diagnosing when they open this +/// page. So the unmeasured calls are excluded from the statistics and the count of +/// what was measured is drawn beside them, which is what makes the median +/// interpretable at all. +/// +/// The median is computed here from the seeded latencies rather than read off the +/// page and asserted against itself. +/// +/// Sabotage: `call.ttft_ms.unwrap_or(0)` in the collector — under which the median +/// of this fixture drops from 300 ms to 100 ms, the slowest is unchanged, and +/// nothing says two of the five calls were never measured. +#[test] +fn an_unmeasured_time_to_first_token_is_counted_apart_and_never_averaged_as_zero() { + let store = history(); + let rows = page(&store); + let ttft = section(&rows, "time to first token"); + + // The three measured values, sorted, and the median io-harness's own data + // would give: the middle of what was measured, not the middle of what was + // called. + let measured = [100u64, 300, 900]; + assert_eq!( + field(ttft, "median"), + format!("{} ms", measured[measured.len() / 2]), + "the median counted the calls that measured nothing:\n{}", + text(&rows), + ); + assert_eq!( + field(ttft, "slowest"), + format!("{} ms", measured[measured.len() - 1]), + ); + assert_eq!( + field(ttft, "measured on"), + "3 of 5 calls", + "the page does not say how much of the sample it is speaking for", + ); + // **A whole figure of `0 ms`, not the substring.** `text(ttft).contains("0 ms")` + // was the first spelling of this and it is a false positive on every measured + // value that happens to end in a zero: `300 ms` and `100 ms` both contain it. + // The claim is that no *figure* is zero, so the test is on the value of a row. + assert!( + !ttft + .iter() + .any(|row| row.trim_end().ends_with(": 0 ms")), + "an unmeasured first token reached the figures as an instant one:\n{}", + text(ttft), + ); + + // A store whose calls measured nothing says so in words rather than drawing a + // median of nought. + let store = Store::memory().expect("an in-memory store"); + let run = store.start_run("summarise", "/repo").expect("a run"); + store + .record_provider_call(run, &call("claude-sonnet-4.5", 900, None)) + .expect("recorded"); + let rows = page(&store); + let ttft = section(&rows, "time to first token"); + assert!( + text(ttft).contains("unmeasured rather than instant"), + "a sample that measured nothing drew a figure anyway: {ttft:?}", + ); + assert!( + !text(ttft).contains("median"), + "a median was drawn over nothing: {ttft:?}", + ); +} + +/// **A SQLite build that cannot break the file down by table says so, rather than +/// drawing the breakdown as zeroes or omitting it in silence.** +/// +/// `dbstat` is a virtual table over the b-tree pages and it is compiled in or it +/// is not. io-harness returns an empty table list on a build without it, and it +/// says why in a comment: that is a build that cannot answer the question, not a +/// database whose tables are all empty. The two are indistinguishable in the data +/// and completely different facts. +/// +/// **Which branch runs here is a property of the SQLite this crate happens to +/// link, not of io-cli**, so the test asserts the mapping in both directions +/// rather than picking one and pretending the other cannot happen: a breakdown +/// present means no excuse drawn, and a breakdown absent means the excuse drawn. +/// Asserting one branch would go green on the developer's machine and say nothing +/// about CI — where the SQLite is built by a different job on a different +/// platform, which is exactly where this would differ. +/// +/// The file's own figures stand either way, which is the other half of io-harness's +/// argument for not failing the whole size call over it. +/// +/// Sabotage: draw the table rows unconditionally — under which a build without +/// `dbstat` shows a `what the store holds` section with the size, the free space +/// and nothing at all where the breakdown should be, and no sentence saying which +/// of "there are no tables" and "I cannot see the tables" is true. +#[test] +fn an_absent_table_breakdown_reads_as_cannot_break_down_rather_than_as_zeroes() { + let store = history(); + let size = store.store_size().expect("the store reports its size"); + let rows = page(&store); + let held = section(&rows, "what the store holds"); + let drawn = text(held); + + // The figures that do not depend on `dbstat` are there either way. + assert_eq!(field(held, "sessions"), "0", "no session was opened"); + assert_eq!(field(held, "runs"), "5"); + assert!( + !field(held, "file").starts_with("0 "), + "a store with five runs in it reported no file at all: {drawn}", + ); + + let excuse = "cannot break the file down by table"; + if size.tables.is_empty() { + assert!( + drawn.contains(excuse), + "this build has no `dbstat` and the page does not say so:\n{drawn}", + ); + assert!( + !held.iter().any(|row| row.contains("provider_calls")), + "a breakdown was drawn from a table list that is empty: {held:?}", + ); + } else { + assert!( + !drawn.contains(excuse), + "this build has `dbstat` and the page claims it does not:\n{drawn}", + ); + // Largest first, and no more than five, so the section stays a summary. + let (largest, _) = &size.tables[0]; + assert!( + drawn.contains(largest.as_str()), + "the largest table is not in the breakdown:\n{drawn}", + ); + assert!( + size.tables + .iter() + .take(5) + .all(|(name, _)| drawn.contains(name.as_str())), + "the breakdown is shorter than the five rows it promises:\n{drawn}", + ); + } +} + +/// **Nothing on this page reclaims anything, and the free figure says so.** +/// +/// `Store::compact` is a full `VACUUM`: it needs free disk roughly equal to the +/// database file, it rewrites the whole thing, and it is not quick. A page that +/// reported free space and then reclaimed it on its own would be a page that +/// surprised somebody mid-session — and this one is opened while a turn may be in +/// flight. +/// +/// The sentence is only drawn when there is something to reclaim, because a page +/// that explained a decision about nothing every time it was opened would be a +/// page nobody finishes reading. +/// +/// Sabotage: call `Store::compact` from this page, or drop the sentence and leave +/// the figure — under which an operator reads a number with no idea whether it is +/// about to move. +#[test] +fn the_page_reports_free_space_and_does_not_offer_to_reclaim_it() { + let store = history(); + let before = store.store_size().expect("a size"); + let rows = page(&store); + let held = section(&rows, "what the store holds"); + + // **Named units, and the disjunction that used to stand here was no test at + // all**: `ends_with(..) || !is_empty()` is satisfied by the second half for + // every non-empty string, so a figure rendered as a bare byte count would have + // passed it. `stats::bytes` renders exactly four units and this asserts on + // that set, so a figure that lost its suffix fails here rather than on a + // reader's screen. + let free = field(held, "free inside it"); + assert!( + ["B", "KiB", "MiB", "GiB"] + .iter() + .any(|unit| free.ends_with(unit)), + "the free figure has no unit: {free:?} in {held:?}", + ); + if before.free_bytes > 0 { + assert!( + text(held).contains("is not done from this page"), + "free space was reported with no word about what would reclaim it: {held:?}", + ); + } + + // Reading the page changed nothing about the store. The figures a second read + // gives are the figures the first one gave. + let after = store.store_size().expect("a size"); + assert_eq!( + (before.file_bytes, before.free_bytes, before.runs), + (after.file_bytes, after.free_bytes, after.runs), + "drawing the page moved the database it was reporting on", + ); +} + +/// **An empty store draws every section and reports nothing, rather than a row of +/// zeroes.** +/// +/// The distinction every counter in this product is held to: "no run has been +/// recorded" and "every run scored zero" are different facts, and a page that +/// drew the second when the first was true would be reporting a corpus that does +/// not exist. It is also the state a brand new install is in, which is to say the +/// first thing anybody sees. +/// +/// The page still has all eight sections. An empty state that quietly dropped its +/// headings would leave a first-time reader with no idea what the page is for. +/// +/// Sabotage: return `Vec::new()` from `tallies` on an empty list rather than a +/// note — under which the headings sit above nothing at all and read as sections +/// that failed rather than as questions with no answer yet. +#[test] +fn an_empty_store_says_nothing_is_recorded_rather_than_drawing_zeroes() { + let store = Store::memory().expect("an in-memory store"); + let rows = page(&store); + let drawn = text(&rows); + + assert_eq!( + headings(&rows).len(), + 8, + "an empty store lost a section:\n{drawn}", + ); + // Zero runs, so the sample is zero runs and the heading says so in the + // singular-aware form rather than reading "the last 0 run". + assert!( + headings(&rows).contains(&"slowest calls, of the last 0 runs"), + "the sample heading does not survive an empty store: {:?}", + headings(&rows), + ); + + for heading in ["runs by outcome", "runs by day"] { + let empty = section(&rows, heading); + assert_eq!(empty.len(), 1, "`{heading}` drew more than a note: {empty:?}"); + assert_eq!( + empty[0], " no runs recorded", + "`{heading}` drew a zero where it has nothing to report", + ); + } + assert!( + text(section(&rows, "gate failures by phase")).contains("no gate has failed"), + "an empty store reported gate failures:\n{drawn}", + ); + assert!( + text(section(&rows, "slowest calls, of the last 0 runs")) + .contains("no provider calls in those runs"), + "an empty store drew a slow list:\n{drawn}", + ); + + // The counters that genuinely are zero are still drawn as zero: nobody has + // fallen back, replanned or resumed, and that is a measured nought rather + // than an absence. + let recovery = section(&rows, "recovery"); + assert_eq!(field(recovery, "fallbacks"), "0"); + assert_eq!(field(recovery, "replans"), "0"); + assert_eq!(field(recovery, "resumes"), "0"); +} diff --git a/tests/status.rs b/tests/status.rs index f00dfb3..2cf5edb 100644 --- a/tests/status.rs +++ b/tests/status.rs @@ -2072,17 +2072,25 @@ fn f4_a_full_counts_row_drops_a_counter_and_not_the_planning_phase() { text.contains("workspace-write/macos-sandbox-exec") && text.contains("read-only"), "the phase took the rest of its group with it: {text:?}", ); - // What paid for it, and which end it was taken off. The key hint is the - // rightmost counter and the least of them — `/help` is the help screen — - // and the fields left are the ones the row is ordered to keep. + // What paid for it, and which end it was taken off. `ctx 23%` is the + // rightmost *counter* and is what a crowded row gives up first. assert!( - !text.contains("/ for commands"), + !text.contains("ctx 23%"), "nothing was dropped, so the row cannot have fitted: {text:?}", ); assert!( - text.contains("6 steps") && text.contains("26.9k tok") && text.contains("ctx 23%"), + text.contains("6 steps") && text.contains("26.9k tok"), "counters came off the wrong end: {text:?}", ); + // **And the key hint is not a counter and is not droppable.** It was the + // rightmost item on the row and the narrowing took it first, which meant a + // full row lost `esc stops` mid-turn — the only place the footer says how to + // interrupt what is running. It is held out of the narrowing and appended + // after it, so it survives every width the row is drawn at. + assert!( + text.contains("/ for commands"), + "the interrupt hint was dropped to make room for a counter: {text:?}", + ); } /// **F4 — and it holds with one more counter on the row, at every width the @@ -2097,11 +2105,16 @@ fn f4_a_full_counts_row_drops_a_counter_and_not_the_planning_phase() { /// /// `plan 2/5` stands in for that next counter: it is a real member of the same /// row, it is eight characters plus a separator, and with it the row no longer -/// fits at any width in the sweep on the first trim. The sweep starts at seventy -/// because a right-hand group of fifty-seven plus the leftmost counter is -/// sixty-four columns, and below that there is no row that holds both — the -/// trim stops at one counter deliberately, since a row that had given up every -/// number would be reporting the mode by erasing the session. +/// fits at any width in the sweep on the first trim. +/// +/// **The sweep starts at seventy-four, and the number is the whole of what the +/// key hint being undroppable costs.** The right-hand group is fifty-seven +/// columns; `/ for commands` is fourteen and its separator three; fifty-seven +/// plus seventeen is seventy-four, and that is the narrowest row that can carry +/// the standing mode at all now that the hint is not a counter and cannot be +/// taken to make room. Below it the group goes and the hint stays — which is the +/// right way round: a mode an operator cannot see is bad, and a running turn an +/// operator cannot find the key to stop is worse. /// /// Sabotage: pop from the front of `counts` instead of the back — under which /// this test still passes at every width and the one above fails on `6 steps`, @@ -2111,7 +2124,7 @@ fn f4_the_planning_phase_holds_at_every_width_that_can_hold_the_group() { let mut status = full_row(); status.plan = Some((2, 5)); - for width in 70..=160u16 { + for width in 74..=160u16 { let text = footed(&status, width); assert!( text.contains("planning"), @@ -2132,3 +2145,182 @@ fn f4_the_planning_phase_holds_at_every_width_that_can_hold_the_group() { } } } + +// --------------------------------------------------------------------------- +// 0.22.0 — the cost field +// --------------------------------------------------------------------------- + +/// A store with one run, and one provider call per model named. +/// +/// Real rows through io-harness's own recorder rather than a `Total` built here, +/// because `Status::note_cost_from` reads the store: the split a price needs — +/// fresh prompt against cache read against cache write against completion — lives +/// only on the `provider_calls` row, and `EventKind::Step` carries a scalar token +/// count that cannot be priced at all. A fixture that handed the figure in would +/// be asserting the format string. +fn run_calling(models: &[&str]) -> (io_harness::Store, i64) { + let store = io_harness::Store::memory().expect("an in-memory store"); + let run = store.start_run("summarise the module", "/repo").expect("a run"); + for model in models { + store + .record_provider_call( + run, + &io_harness::ProviderCall { + step: 1, + provider: "anthropic".into(), + model: Some((*model).to_string()), + usage: Some(io_harness::Usage { + prompt_tokens: 10_000, + completion_tokens: 2_000, + total_tokens: 12_000, + ..Default::default() + }), + latency_ms: 1_200, + ..Default::default() + }, + ) + .expect("the call is recorded"); + } + (store, run) +} + +/// The rates in force, pricing exactly one model. +fn rates() -> io_harness::pricing::PriceTable { + io_harness::pricing::PriceTable::new("2026-08-27").with( + "anthropic/claude-sonnet-4.5", + io_harness::pricing::Price { + input: 3_000_000, + output: 15_000_000, + ..io_harness::pricing::Price::ZERO + }, + ) +} + +/// **The cost field is absent rather than zero in all three ways of having no +/// answer, and they are three different states of an install.** +/// +/// An operator with no `[prices]` section, an operator whose table does not price +/// the model this run is using, and a session that has not called anything yet. +/// The status line has no room to tell them apart and does not try — `/cost` is +/// one keystroke away and tells all three — but it must not answer any of them +/// with `$0`, because `$0` is a measured figure and none of these is measured. +/// +/// This is the same rule the rest of the line already keeps: every counter is +/// absent until there is something to count, so a session that has run nothing +/// carries an almost empty row rather than a row of zeroes. The money field is the +/// one where getting it wrong is not merely untidy — a run whose models are all +/// outside the table has a real cost this program cannot state, and stating it as +/// nothing is the invented number the whole of `/cost` is built to avoid. +/// +/// Sabotage: `self.cost = Some(total.micros)` — three characters, reads as a +/// simplification of a `then_some`, and under it every unpriced session reports +/// `$0` for a turn that cost real money. Nothing else in the repository fails. +#[test] +fn the_cost_field_is_absent_rather_than_zero_when_nothing_can_be_priced() { + // 1. No prices configured at all. An empty table is what `cost::table` hands + // back for a file with no `[prices]`, so this is the ordinary state of a + // fresh install rather than an edge case. + let (store, run) = run_calling(&["anthropic/claude-sonnet-4.5"]); + let mut status = Status::new("anthropic/claude-sonnet-4.5"); + status.note_cost_from(&store, run, &io_harness::pricing::PriceTable::new("")); + assert_eq!(status.cost, None, "an empty table produced a figure"); + for (renderer, text) in both_renderers(&status) { + assert!( + !text.contains('$'), + "{renderer} drew money with no prices configured: {text:?}", + ); + } + + // 2. Prices configured, and none of them for the model this run used. The + // table is real, the date is real, and the answer is still unavailable. + let (store, run) = run_calling(&["some-lab/experimental-preview"]); + let mut status = Status::new("some-lab/experimental-preview"); + status.note_cost_from(&store, run, &rates()); + assert_eq!( + status.cost, None, + "a model outside the table was priced at nothing", + ); + for (renderer, text) in both_renderers(&status) { + assert!( + !text.contains('$'), + "{renderer} priced a model the table does not price: {text:?}", + ); + } + + // 3. A run that has called nothing. Not zero: nothing. + let (store, run) = run_calling(&[]); + let mut status = Status::new("anthropic/claude-sonnet-4.5"); + status.note_cost_from(&store, run, &rates()); + assert_eq!(status.cost, None, "a run with no calls reported a cost"); + for (renderer, text) in both_renderers(&status) { + assert!(!text.contains('$'), "{renderer}: {text:?}"); + } + + // And a run id nothing ever recorded reads back as no calls at all — which + // is no figure, not a zero one, and is not an error either: a notice about a + // failed read of a decorative field would be worse than the field's absence, + // which is why `note_cost_from` is silent on every failure. + let mut status = Status::new("anthropic/claude-sonnet-4.5"); + status.note_cost_from(&store, 9_999, &rates()); + assert_eq!(status.cost, None, "a run id that does not exist invented a figure"); +} + +/// **When there is a figure it is on BOTH renderers, out of one method.** +/// +/// `Status` renders two ways: `line` is the one-row form, which has exactly one +/// production caller — the fallback for a terminal under seven rows — and `footer` +/// is the three-row form `Status::render` takes on everything taller, which is to +/// say on every real terminal anybody uses. **This file has shipped a field into +/// one of them twice**: 0.8.0's spend field and 0.12.0's planning phase were each +/// added to `line`, asserted against `line`, went green, and were nowhere on +/// screen in a live capture of the running binary. +/// +/// So the assertion is over the pair, and the money is one method — `cost_field` +/// — that both renderers extend from, exactly as `budgets_left` and `queued_left` +/// already are. A test that read one renderer is a test that can pass while the +/// operator has an invisible field, which is the whole of what this test is +/// against. +/// +/// The figure itself is checked against `cost::money` of what io-harness says the +/// call cost, rather than against a string, so the two surfaces are proved to be +/// drawing the same number and not merely both drawing a dollar sign. +/// +/// Sabotage: push `self.cost_field()` into `Status::fields` alone and drop the +/// `counts.extend(self.cost_field())` from the footer — under which the one-row +/// arm of this test passes and the binary shows nothing. +#[test] +fn the_cost_field_is_drawn_by_the_line_and_by_the_footer_alike() { + let (store, run) = run_calling(&["anthropic/claude-sonnet-4.5"]); + let table = rates(); + + let mut status = Status::new("anthropic/claude-sonnet-4.5"); + status.note_cost_from(&store, run, &table); + + // What the harness says the run cost, from the rows the harness stored. + let calls = store.provider_calls(run).expect("the calls are readable"); + let expected = io_cli::cost::Total::of(&calls, &table).micros; + assert!(expected > 0, "a fixture that costs nothing proves nothing"); + assert_eq!(status.cost, Some(expected)); + + let drawn = io_cli::cost::money(expected); + for (renderer, text) in both_renderers(&status) { + assert!( + text.contains(&drawn), + "{renderer} does not carry the cost {drawn}: {text:?}", + ); + } + + // **Run-scoped, and forgotten with the run.** A cost that outlived its turn + // would be a figure that stopped moving while a new turn spent money beside + // it — the same argument the module makes for clearing the step count and the + // provider, and the opposite of the one it makes for keeping the planning + // phase. + status.forget_run(); + assert_eq!(status.cost, None, "the cost outlived the run that incurred it"); + for (renderer, text) in both_renderers(&status) { + assert!( + !text.contains('$'), + "{renderer} still shows the last run's bill: {text:?}", + ); + } +} From 7de95a5afa11c9b1bbad1526cbe3c71df4a5bfc3 Mon Sep 17 00:00:00 2001 From: aakashpawar1999 Date: Thu, 27 Aug 2026 22:02:39 +0530 Subject: [PATCH 6/7] fix: the three clippy findings and the five the rustdoc gate caught MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The nightly rustdoc gate is not in `ci.yml`, so it is run by hand at the seal — and this is the fifth release in six where it has found something no other gate can see. Four of the five are this release's: a doc link to `PriceTable` with no path, a public `spell` linking to the private `segments` it is the inverse of, and two `[`array`]` links that are ambiguous against the primitive type. The fifth predates it — `crate::attach::unquote` has never resolved — and is corrected here rather than carried. Clippy: a same-type cast in the date arithmetic, a manual checked division of the first-try share, and a `sort_by` that says `sort_by_key` more clearly. --- src/composer.rs | 2 +- src/edit.rs | 6 +++--- src/prices.rs | 4 ++-- src/stats.rs | 7 +++--- tests/cost.rs | 52 +++++++++++++++++++++++++++----------------- tests/edit.rs | 4 +++- tests/page.rs | 15 ++++++++++--- tests/prices.rs | 57 ++++++++++++++++++++++++++++++++----------------- tests/stats.rs | 14 +++++++----- tests/status.rs | 14 +++++++++--- 10 files changed, 115 insertions(+), 60 deletions(-) diff --git a/src/composer.rs b/src/composer.rs index 09a0895..862b450 100644 --- a/src/composer.rs +++ b/src/composer.rs @@ -227,7 +227,7 @@ pub fn pasted_path(text: &str) -> Option { /// /// The mark is chosen rather than escaped into: a path carrying a double quote is /// wrapped in single quotes, one carrying both is left bare. That keeps this the -/// exact inverse of [`crate::attach::unquote`], which takes off one matching pair +/// exact inverse of `crate::attach`'s own `unquote`, which takes off one matching pair /// and knows nothing about escapes — two halves that agree because neither of /// them has anything to agree about. fn quoted(path: &str) -> String { diff --git a/src/edit.rs b/src/edit.rs index 4539531..4061a58 100644 --- a/src/edit.rs +++ b/src/edit.rs @@ -629,7 +629,7 @@ fn split_path(path: &str) -> Result<(TablePath, String), String> { /// whose name is `a"b`, and trimming strips repeated quotes off both ends and /// resolves no escape at all. A literal string `'...'` takes no escapes, so a /// `'` can never appear inside one and finding its end is a search. The decoding -/// is `toml`'s own for the reason [`array`] spells through `toml` rather than a +/// is `toml`'s own for the reason [`array()`] spells through `toml` rather than a /// format string — the escape rules belong to the format, not to this file. /// [`spell`] is the inverse, for the two places a segment goes back into a /// document. @@ -697,7 +697,7 @@ fn segments(path: &str) -> Result, String> { /// Spell one decoded segment the way a document has to carry it. /// -/// The inverse of the decoding [`segments`] does, and it exists because that +/// The inverse of the decoding `segments` does, and it exists because that /// decoding created the need: a segment in hand is now a NAME, and a name is not /// always spellable as itself. A bare key is `A-Za-z0-9_-` and nothing else, so /// anything holding a dot, a space, a slash or a quote has to go back into the @@ -706,7 +706,7 @@ fn segments(path: &str) -> Result, String> { /// still parses. /// /// The quoting goes through `toml`'s own serializer for the same reason -/// [`array`] does: a format string cannot escape a `"` or a backslash correctly +/// [`array()`] does: a format string cannot escape a `"` or a backslash correctly /// and would produce either a parse error or a different name. #[must_use] pub fn spell(segment: &str) -> String { diff --git a/src/prices.rs b/src/prices.rs index 66bae22..8693c1a 100644 --- a/src/prices.rs +++ b/src/prices.rs @@ -1,7 +1,7 @@ //! Where a price comes from, what it claims, and when it was read. //! //! **io-cli compiles no prices in, and this module is the reason it does not have -//! to.** io-harness prices a call from a [`PriceTable`] the operator's +//! to.** io-harness prices a call from a [`io_harness::pricing::PriceTable`] the operator's //! configuration carries, and that table has to be filled from somewhere. Filling //! it from a list baked into the binary would be a promise the binary cannot keep: //! providers move prices without announcing it, a release cadence is not a pricing @@ -589,7 +589,7 @@ pub fn date(secs: u64) -> String { let days = (secs / 86_400) as i64; let z = days + 719_468; let era = if z >= 0 { z } else { z - 146_096 } / 146_097; - let doe = (z - era * 146_097) as i64; + let doe = z - era * 146_097; let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; let year = yoe + era * 400; let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); diff --git a/src/stats.rs b/src/stats.rs index 9b8da7a..26d5e3f 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -101,10 +101,10 @@ pub fn committed( // io-harness gives three counts and refuses to pick one for exactly this // reason; a bare `73%` would be a number whose meaning the reader has to // reconstruct, and two readers would reconstruct it differently. - if first.succeeded > 0 { + if let Some(share) = (first.first_try * 100).checked_div(first.succeeded) { rows.push(Row::fact( "first-try share of successful runs", - format!("{}%", first.first_try * 100 / first.succeeded), + format!("{share}%"), )); } @@ -154,7 +154,8 @@ pub fn committed( ) }) .collect(); - slow.sort_by(|a, b| b.0.cmp(&a.0)); + // Descending, so `take(SLOWEST)` below is the slowest and not the fastest. + slow.sort_by_key(|(latency, _)| std::cmp::Reverse(*latency)); if slow.is_empty() { rows.push(Row::note("no provider calls in those runs")); } else { diff --git a/tests/cost.rs b/tests/cost.rs index 0429302..20d79e2 100644 --- a/tests/cost.rs +++ b/tests/cost.rs @@ -105,7 +105,9 @@ struct Seeded { fn seeded(calls: &[ProviderCall]) -> Seeded { let store = Store::memory().expect("an in-memory store"); - let run = store.start_run("summarise the module", "/repo").expect("a run"); + let run = store + .start_run("summarise the module", "/repo") + .expect("a run"); let session = store.create_session("/repo").expect("a session"); store .record_turn(session, None, run, "summarise the module") @@ -258,7 +260,10 @@ fn the_total_is_the_rates_times_the_split_and_not_a_figure_typed_here() { assert_eq!(three.micros, expected * 3); assert_eq!(three.usage.prompt_tokens, usage.prompt_tokens * 3); assert_eq!(three.usage.cache_read_tokens, usage.cache_read_tokens * 3); - assert_eq!(three.usage.server_tool_requests, usage.server_tool_requests * 3); + assert_eq!( + three.usage.server_tool_requests, + usage.server_tool_requests * 3 + ); } /// **A call the provider said nothing about is unknown, and unknown is not @@ -289,7 +294,10 @@ fn a_call_the_provider_said_nothing_about_is_unknown_and_never_free() { ]; let total = Total::of(&calls, &table()); - assert_eq!(total.calls, 3, "a call that reported nothing still happened"); + assert_eq!( + total.calls, 3, + "a call that reported nothing still happened" + ); assert_eq!(total.unknown, 2); assert_eq!( total.unpriced, 0, @@ -352,7 +360,10 @@ fn a_model_the_table_does_not_price_is_a_floor_and_says_so() { assert_eq!(total.calls, 3); assert_eq!(total.unknown, 0, "all three reported their usage"); assert_eq!(total.unpriced, 2, "two of the three cannot be priced"); - assert!(total.is_floor(), "a total missing two calls' money is a floor"); + assert!( + total.is_floor(), + "a total missing two calls' money is a floor" + ); assert_eq!( total.usage.prompt_tokens, split().prompt_tokens * 3, @@ -360,7 +371,9 @@ fn a_model_the_table_does_not_price_is_a_floor_and_says_so() { ); assert_eq!( total.micros, - table().cost_micros(PRICED, &split()).expect("the priced one"), + table() + .cost_micros(PRICED, &split()) + .expect("the priced one"), "the money is exactly the one call the table can price", ); @@ -522,7 +535,7 @@ fn money_is_integer_arithmetic_at_every_magnitude() { // floor: a figure that rounded to `$2.00` from `$1.999999` would report more // than the store can account for. assert_eq!(cost::money(1_999_999), "$1.99"); - assert_eq!(cost::money(999_99), "$0.0999"); + assert_eq!(cost::money(99_999), "$0.0999"); // The rendering never loses a digit or gains one: every sub-dollar figure it // can express has four decimals, at every magnitude in the sweep. The @@ -605,7 +618,10 @@ fn an_empty_price_table_draws_tokens_and_no_money() { spelled(split().prompt_tokens * 2), "the token figures went missing with the prices:\n{drawn}", ); - assert_eq!(field(&rows, "completion"), spelled(split().completion_tokens * 2)); + assert_eq!( + field(&rows, "completion"), + spelled(split().completion_tokens * 2) + ); // And no currency anywhere. Not a zero, not a floor of zero, not a dollar // sign on a grouped row: there is no rate on this install and every figure @@ -633,16 +649,8 @@ fn an_empty_price_table_draws_tokens_and_no_money() { #[test] fn a_session_that_has_run_nothing_says_so_rather_than_drawing_zeroes() { let store = Store::memory().expect("an in-memory store"); - let lines = cost::committed( - &store, - &table(), - &provenance(), - None, - None, - &ascii(), - ROOMY, - ) - .expect("the page draws with nothing in the store"); + let lines = cost::committed(&store, &table(), &provenance(), None, None, &ascii(), ROOMY) + .expect("the page draws with nothing in the store"); let drawn: String = lines .iter() .map(|line| { @@ -701,7 +709,9 @@ fn the_session_total_is_every_turn_and_not_the_run_in_flight() { } let table = table(); - let one = table.cost_micros(PRICED, &split()).expect("one call's cost"); + let one = table + .cost_micros(PRICED, &split()) + .expect("one call's cost"); let lines = cost::committed( &store, &table, @@ -792,7 +802,11 @@ fn the_footer_names_the_catalogue_and_the_date_or_admits_it_cannot() { "a table with no recorded source got one invented for it:\n{hand_written}", ); - let none = text(&page(&fixture, &PriceTable::new(""), &Provenance::default())); + let none = text(&page( + &fixture, + &PriceTable::new(""), + &Provenance::default(), + )); assert!( none.contains("prices: none configured"), "an install with no prices does not say so at the foot of the page:\n{none}", diff --git a/tests/edit.rs b/tests/edit.rs index dd7fe26..8dd680a 100644 --- a/tests/edit.rs +++ b/tests/edit.rs @@ -684,7 +684,9 @@ fn f1_a_whole_section_is_written_once_and_refused_when_it_is_already_there() { assert_eq!(parsed["prices"]["as_of"].as_str(), Some("2026-08-27")); for model in ["gpt-4.1", "gpt-4o", "o3"] { assert!( - parsed["prices"]["models"][model]["input"].as_integer().is_some(), + parsed["prices"]["models"][model]["input"] + .as_integer() + .is_some(), "`{model}` did not land as a key of the section:\n{after}" ); } diff --git a/tests/page.rs b/tests/page.rs index 75d186c..fa5e2a7 100644 --- a/tests/page.rs +++ b/tests/page.rs @@ -173,7 +173,10 @@ fn a_word_longer_than_its_room_is_split_and_one_that_merely_does_not_fit_is_not( let rows = page::folded(sentence, 80, 2, 4); assert!(rows.len() > 1, "the fixture has to fold to say anything"); let words: Vec<&str> = sentence.split_whitespace().collect(); - let folded_words: Vec<&str> = rows.iter().flat_map(|line| line.split_whitespace()).collect(); + let folded_words: Vec<&str> = rows + .iter() + .flat_map(|line| line.split_whitespace()) + .collect(); assert_eq!( words, folded_words, "a word was broken across two rows when a fresh row would have held it: {rows:?}", @@ -222,7 +225,10 @@ fn the_hanging_indent_marks_a_continuation_from_the_second_row() { // what a reader under `NO_COLOR` or `--plain` has to go on. let heading = page::folded("slowest calls, of the last two hundred runs", 20, 0, 2); assert!(heading.len() > 1); - assert!(!heading[0].starts_with(' '), "a heading is indented to zero"); + assert!( + !heading[0].starts_with(' '), + "a heading is indented to zero" + ); assert!(heading[1].starts_with(" ")); } @@ -352,7 +358,10 @@ fn no_row_of_a_committed_page_runs_past_the_terminal() { "anthropic/claude-sonnet-4.5-with-an-absurdly-long-deployment-suffix", "4213 ms", ), - Row::fact("workspace", "/Users/somebody/Documents/work/monorepo/services/billing"), + Row::fact( + "workspace", + "/Users/somebody/Documents/work/monorepo/services/billing", + ), Row::caveat( "four calls used a model with no rate in the price table, so the cost above \ is a floor and not a total", diff --git a/tests/prices.rs b/tests/prices.rs index fd03743..90f21bd 100644 --- a/tests/prices.rs +++ b/tests/prices.rs @@ -60,7 +60,9 @@ fn local() -> Config { } fn spec(config: &Config) -> &ProviderSpec { - config.provider_spec().expect("the fixture names a provider") + config + .provider_spec() + .expect("the fixture names a provider") } /// One catalogue row the vendor put a rate on. @@ -397,13 +399,10 @@ fn a_model_the_catalogue_priced_at_nothing_is_absent_rather_than_zero() { ); // And the proof that absence is what the table sees: the row that is missing // is missing from the `PriceTable` a write of these rows would produce. - let table = catalogue - .rows - .iter() - .fold( - PriceTable::new(catalogue.as_of.clone()), - |table, (id, price)| table.with(id.clone(), *price), - ); + let table = catalogue.rows.iter().fold( + PriceTable::new(catalogue.as_of.clone()), + |table, (id, price)| table.with(id.clone(), *price), + ); assert_eq!(table.price("some-lab/experimental-preview"), None); assert!(table.price("anthropic/claude-sonnet-4.5").is_some()); } @@ -459,14 +458,23 @@ fn a_first_fill_is_never_refused_and_a_truncated_answer_always_is() { // Drastically shorter: refused. Similar: not. The line sits at half, and // both sides of it are asserted so that moving it fails this rather than // passing quietly in whichever direction it moved. - assert!(of(4).too_short(400), "a truncated read replaced a full table"); + assert!( + of(4).too_short(400), + "a truncated read replaced a full table" + ); assert!(of(199).too_short(400), "under half is a truncation"); assert!( !of(200).too_short(400), "exactly half is allowed: vendors do retire models", ); - assert!(!of(380).too_short(400), "a slightly smaller answer is a normal one"); - assert!(!of(420).too_short(400), "a larger answer is never a truncation"); + assert!( + !of(380).too_short(400), + "a slightly smaller answer is a normal one" + ); + assert!( + !of(420).too_short(400), + "a larger answer is never a truncation" + ); } // --------------------------------------------------------------------------- @@ -536,8 +544,11 @@ fn a_dotted_model_id_survives_the_write_and_prices_a_call() { prices::has_models_section(EXISTING), "the fixture is meant to be the refresh case", ); - let written = edit::apply(EXISTING, &catalogue.edits(prices::has_models_section(EXISTING))) - .expect("the edits produce a configuration file that parses"); + let written = edit::apply( + EXISTING, + &catalogue.edits(prices::has_models_section(EXISTING)), + ) + .expect("the edits produce a configuration file that parses"); // The spelling, so a failure here names the cause rather than the symptom. assert!( @@ -895,9 +906,14 @@ fn changes_lists_only_what_moved_and_calls_an_unpriced_model_new() { // Nothing moved: nothing listed. Including the case an operator hits most // often, which is running `/config` twice in a day. assert!( - prices::changes(Some(&existing), &Catalogue::of(spec(&config), vec![ - priced("a/steady", 1_000_000, 2_000_000) - ], "2026-08-27")) + prices::changes( + Some(&existing), + &Catalogue::of( + spec(&config), + vec![priced("a/steady", 1_000_000, 2_000_000)], + "2026-08-27" + ) + ) .is_empty(), "an unchanged table reported a change", ); @@ -942,9 +958,7 @@ fn source_word_names_the_catalogue_and_never_the_connected_provider() { "the vendor form points at a URL the operator did not need", ); - let reference = prices::source_word(&PriceSource::Reference( - DEFAULT_REFERENCE_URL.to_string(), - )); + let reference = prices::source_word(&PriceSource::Reference(DEFAULT_REFERENCE_URL.to_string())); assert!( reference.contains(DEFAULT_REFERENCE_URL), "the reference form does not say which reference: {reference}", @@ -1045,7 +1059,10 @@ fn the_catalogue_filter_is_applied_exactly_once_because_it_is_not_idempotent() { #[test] fn a_model_id_is_spelled_by_the_toml_crate_and_not_by_a_format_string() { assert_eq!(prices::quoted("gpt-4.1"), "\"gpt-4.1\""); - assert_eq!(prices::quoted("anthropic/claude-sonnet-4.5"), "\"anthropic/claude-sonnet-4.5\""); + assert_eq!( + prices::quoted("anthropic/claude-sonnet-4.5"), + "\"anthropic/claude-sonnet-4.5\"" + ); // The two a hand-written pair of quotes gets wrong. Both round-trip through // the parser rather than being compared to a spelling typed here, because the diff --git a/tests/stats.rs b/tests/stats.rs index 15d3ee9..2d00cff 100644 --- a/tests/stats.rs +++ b/tests/stats.rs @@ -72,7 +72,9 @@ fn history() -> Store { // 2 first try — three different numbers, which is what makes a page that // picked the wrong one visible. for outcome in ["success", "success", "stalled", "step_cap_reached"] { - let run = store.start_run("summarise the module", "/repo").expect("a run"); + let run = store + .start_run("summarise the module", "/repo") + .expect("a run"); store.finish_run(run, outcome).expect("the run finishes"); } @@ -427,9 +429,7 @@ fn an_unmeasured_time_to_first_token_is_counted_apart_and_never_averaged_as_zero // value that happens to end in a zero: `300 ms` and `100 ms` both contain it. // The claim is that no *figure* is zero, so the test is on the value of a row. assert!( - !ttft - .iter() - .any(|row| row.trim_end().ends_with(": 0 ms")), + !ttft.iter().any(|row| row.trim_end().ends_with(": 0 ms")), "an unmeasured first token reached the figures as an instant one:\n{}", text(ttft), ); @@ -612,7 +612,11 @@ fn an_empty_store_says_nothing_is_recorded_rather_than_drawing_zeroes() { for heading in ["runs by outcome", "runs by day"] { let empty = section(&rows, heading); - assert_eq!(empty.len(), 1, "`{heading}` drew more than a note: {empty:?}"); + assert_eq!( + empty.len(), + 1, + "`{heading}` drew more than a note: {empty:?}" + ); assert_eq!( empty[0], " no runs recorded", "`{heading}` drew a zero where it has nothing to report", diff --git a/tests/status.rs b/tests/status.rs index 2cf5edb..6947ecb 100644 --- a/tests/status.rs +++ b/tests/status.rs @@ -2160,7 +2160,9 @@ fn f4_the_planning_phase_holds_at_every_width_that_can_hold_the_group() { /// be asserting the format string. fn run_calling(models: &[&str]) -> (io_harness::Store, i64) { let store = io_harness::Store::memory().expect("an in-memory store"); - let run = store.start_run("summarise the module", "/repo").expect("a run"); + let run = store + .start_run("summarise the module", "/repo") + .expect("a run"); for model in models { store .record_provider_call( @@ -2262,7 +2264,10 @@ fn the_cost_field_is_absent_rather_than_zero_when_nothing_can_be_priced() { // which is why `note_cost_from` is silent on every failure. let mut status = Status::new("anthropic/claude-sonnet-4.5"); status.note_cost_from(&store, 9_999, &rates()); - assert_eq!(status.cost, None, "a run id that does not exist invented a figure"); + assert_eq!( + status.cost, None, + "a run id that does not exist invented a figure" + ); } /// **When there is a figure it is on BOTH renderers, out of one method.** @@ -2316,7 +2321,10 @@ fn the_cost_field_is_drawn_by_the_line_and_by_the_footer_alike() { // provider, and the opposite of the one it makes for keeping the planning // phase. status.forget_run(); - assert_eq!(status.cost, None, "the cost outlived the run that incurred it"); + assert_eq!( + status.cost, None, + "the cost outlived the run that incurred it" + ); for (renderer, text) in both_renderers(&status) { assert!( !text.contains('$'), From 2ac64db315ce0addc0334b0e49558b69c141cee5 Mon Sep 17 00:00:00 2001 From: aakashpawar1999 Date: Thu, 27 Aug 2026 22:17:05 +0530 Subject: [PATCH 7/7] test: the source_url path the review found inert now has coverage `app.io-cli.prices.source_url` is documented as the only way an operator on a self-hosted or compatible endpoint gets prices, and the only test either path had was the one asserting the empty result the defect produced. This is its companion: the same endpoint with a catalogue the operator named, taken whole, attributed to its URL rather than to the vendor, and carried through `edit::apply` into a `PriceTable` that answers. --- tests/prices.rs | 81 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/tests/prices.rs b/tests/prices.rs index 90f21bd..1b25cc5 100644 --- a/tests/prices.rs +++ b/tests/prices.rs @@ -359,6 +359,87 @@ fn a_provider_the_reference_cannot_speak_for_yields_no_rows() { ); } +/// **The same endpoint, with a catalogue the operator named — and this is the +/// half that was inert.** +/// +/// `app.io-cli.prices.source_url` is documented in `src/settings.rs` and in the +/// README as the only way an operator on a self-hosted or `compatible` endpoint +/// gets prices at all. It could not work: the read fetched the operator's own +/// catalogue and then put it through the same narrowing as the default one, whose +/// `_` arm discards every row for a `compatible` provider. So io-cli fetched the +/// right list and threw all of it away, and the test above — asserting the empty +/// result — was the only coverage either path had. +/// +/// The narrowing exists to cut one vendor's view of the whole field down to the +/// provider in force. An operator who set `source_url` has already answered that +/// question: they pointed io-cli at the list their own endpoint publishes, so +/// every row of it is theirs and the ids are already spelled the way their +/// provider names them. A named catalogue is therefore taken whole. +/// +/// The source is still recorded as the URL and not as the vendor, because it is +/// still not the vendor speaking — it is a list the operator chose. +/// +/// Sabotage: route `Catalogue::named` through the same filter as +/// `Catalogue::of` — under which only this test fails, and it fails by returning +/// the release to the state where the key exists, is documented, and does nothing. +#[test] +fn a_catalogue_the_operator_named_is_taken_whole_and_attributed_to_its_url() { + const MINE: &str = "http://localhost:11434/v1/models"; + let config = local(); + let catalogue = Catalogue::named( + spec(&config), + vec![ + priced("llama3.2", 20_000, 20_000), + priced("qwen3-coder", 30_000, 30_000), + unpriced("nomic-embed-text"), + ], + "2026-08-27", + MINE, + ); + + let ids: Vec<&str> = catalogue + .rows + .iter() + .map(|(model, _)| model.as_str()) + .collect(); + assert_eq!( + ids, + ["llama3.2", "qwen3-coder"], + "the operator's own catalogue was narrowed by a filter built for somebody else's", + ); + // Served counts what the endpoint offers, priced or not — the third row is a + // model it serves and put no rate on, which is a different fact from a model + // it does not serve. + assert_eq!(catalogue.served, 3); + assert_eq!( + catalogue.source, + PriceSource::Reference(MINE.to_string()), + "a list the operator chose was attributed to the vendor", + ); + assert!( + prices::source_word(&catalogue.source).contains(MINE), + "the page would not say which catalogue these rates came from", + ); + + // And it survives the write, which is the only thing that makes any of it + // reach a `/cost` page. + let written = edit::apply(FRESH, &catalogue.edits(prices::has_models_section(FRESH))) + .expect("the edits produce a file that parses"); + let table = Config::from_toml(&written) + .expect("io-harness accepts it") + .prices() + .expect("the file carries a table"); + assert_eq!( + table.price("llama3.2").map(|price| price.input), + Some(20_000) + ); + assert_eq!( + table.price("nomic-embed-text"), + None, + "a model served with no rate was entered at zero", + ); +} + /// **A model the catalogue served with no price is ABSENT, never entered at /// zero.** ///